diff --git a/.github/workflows/change-log-check.yml b/.github/workflows/change-log-check.yml index 1e5374179f..6fe9ac2a9f 100644 --- a/.github/workflows/change-log-check.yml +++ b/.github/workflows/change-log-check.yml @@ -17,7 +17,7 @@ jobs: if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-changelog') }} run: | git fetch origin ${{ github.base_ref }} - CHANGELOG_DIFF=$(git diff origin/${{ github.base_ref }} origin/${{ github.head_ref }} -- changelog.md) + CHANGELOG_DIFF=$(git diff ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} -- changelog.md) echo "${CHANGELOG_DIFF}" if [ -z "$CHANGELOG_DIFF" ]; then echo "ERROR: No changes detected in CHANGELOG.md. Please update the changelog." diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 079b8dfa92..936823a405 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,6 +1,6 @@ name: "Pull Request Labeler" on: - pull_request: + pull_request_target: types: - opened - edited diff --git a/.gitignore b/.gitignore index d53a2695bd..6a53c9be7b 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,11 @@ localnet/ganache/storage localnet/zetachain-monorepo /standalone-network/authz/tx.json /zetanode.log + +# Blockscout Files +contrib/localnet/blockscout/services/blockscout-db-data/* +contrib/localnet/blockscout/services/logs/* +contrib/localnet/blockscout/services/redis-data/* +contrib/localnet/blockscout/services/stats-db-data/* +contrib/localnet/grafana/addresses.txt +contrib/localnet/addresses.txt \ No newline at end of file diff --git a/Makefile b/Makefile index dd47b6a541..9bc41f0451 100644 --- a/Makefile +++ b/Makefile @@ -247,6 +247,19 @@ stateful-upgrade-source: stop-stateful-upgrade: cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-stateful.yml down --remove-orphans +############################################################################### +### Monitoring ### +############################################################################### + +start-monitoring: + @echo "Starting monitoring services" + cd contrib/localnet/grafana/ && ./get-tss-address.sh + cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-monitoring.yml up -d + +stop-monitoring: + @echo "Stopping monitoring services" + cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-monitoring.yml down + ############################################################################### ### GoReleaser ### ############################################################################### diff --git a/changelog.md b/changelog.md index 17b3450361..f2ec931d7a 100644 --- a/changelog.md +++ b/changelog.md @@ -11,7 +11,7 @@ * ChainNoncesAll :Changed from `/zeta-chain/observer/chainNonces` to `/zeta-chain/observer/chainNonces` . It returns all the chain nonces for all chains. This returns the current nonce of the TSS address for all chains. ### Features - +* [1498](https://github.com/zeta-chain/node/pull/1498) - Add monitoring(grafana, prometheus, ethbalance) for localnet testing * [1395](https://github.com/zeta-chain/node/pull/1395) - Add state variable to track aborted zeta amount * [1410](https://github.com/zeta-chain/node/pull/1410) - `snapshots` commands * enable zetaclients to use dynamic gas price on zetachain - enables >0 min_gas_price in feemarket module @@ -20,6 +20,7 @@ ### Fixes +* [1530](https://github.com/zeta-chain/node/pull/1530) - Outbound tx confirmation/inclusion enhancement * [1496](https://github.com/zeta-chain/node/issues/1496) - post block header for enabled EVM chains only * [1518](https://github.com/zeta-chain/node/pull/1518) - Avoid duplicate keysign if an outTx is already pending * fix Code4rena issue - zetaclients potentially miss inTx when PostSend (or other RPC) fails diff --git a/contrib/localnet/README.md b/contrib/localnet/README.md index 467e0bc877..96143b06d5 100644 --- a/contrib/localnet/README.md +++ b/contrib/localnet/README.md @@ -76,6 +76,30 @@ which does the following docker compose command: # in zeta-node/contrib/localnet/orchestrator $ docker compose down --remove-orphans ``` +### Run monitoring setup +Before starting the monitoring setup, make sure the Zetacore API is up at http://localhost:1317. +You can also add any additional ETH addresses to monitor in zeta-node/contrib/localnet/grafana/addresses.txt file +```bash +# in zeta-node/ +make start-monitoring +``` +which does the following docker compose command: +```bash +# in zeta-node/contrib/localnet/ +$ docker compose -f docker-compose-monitoring.yml up -d +``` +### Grafana credentials and dashboards +The Grafana default credentials are admin:admin. The dashboards are located at http://localhost:3000. +### Stop monitoring setup +```bash +# in zeta-node/ +make stop-monitoring +``` +which does the following docker compose command: +```bash +# in zeta-node/contrib/localnet/ +$ docker compose -f docker-compose-monitoring.yml down --remove-orphans +``` ## Useful data diff --git a/contrib/localnet/docker-compose-monitoring.yml b/contrib/localnet/docker-compose-monitoring.yml new file mode 100644 index 0000000000..ad66f828ec --- /dev/null +++ b/contrib/localnet/docker-compose-monitoring.yml @@ -0,0 +1,47 @@ +version: '3' + +services: + grafana: + image: grafana/grafana:latest + container_name: grafana + hostname: grafana + volumes: + - ./grafana/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml + - ./grafana/dashboards/:/etc/grafana/provisioning/dashboards + ports: + - "3000:3000" + networks: + mynetwork: + ipv4_address: 172.20.0.30 + + prometheus: + image: prom/prometheus:latest + container_name: prometheus + hostname: prometheus + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + ports: + - "9090:9090" + networks: + mynetwork: + ipv4_address: 172.20.0.31 + + zetachain-exporter: + image: zetachain/zetachain-exporter:latest + container_name: zetachain-exporter + hostname: zetachain-exporter + volumes: + - ./grafana/addresses.txt:/app/addresses.txt + ports: + - "9015:9015" + networks: + mynetwork: + ipv4_address: 172.20.0.32 + environment: + - GETH=http://eth:8545 + +networks: + mynetwork: + ipam: + config: + - subnet: 172.20.0.0/24 \ No newline at end of file diff --git a/contrib/localnet/grafana/dashboards/cosmos_dashboard.json b/contrib/localnet/grafana/dashboards/cosmos_dashboard.json new file mode 100644 index 0000000000..85ca855f22 --- /dev/null +++ b/contrib/localnet/grafana/dashboards/cosmos_dashboard.json @@ -0,0 +1,1458 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A Grafana dashboard compatible with all the cosmos-sdk and tendermint based blockchains.", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 11036, + "graphTooltip": 0, + "id": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 64, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "$chain_id overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 62, + "links": [], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "", + "mode": "html" + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "transparent": true, + "type": "text" + }, + { + "datasource": { + "uid": "$DS" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 3 + }, + "hideTimeOverride": false, + "id": 4, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_consensus_height{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "30s", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Block Height", + "type": "stat" + }, + { + "datasource": { + "uid": "$DS" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 3 + }, + "hideTimeOverride": false, + "id": 40, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_consensus_total_txs{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "interval": "30s", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Total Transactions", + "type": "stat" + }, + { + "datasource": { + "uid": "$DS" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 3 + }, + "hideTimeOverride": true, + "id": 39, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_consensus_block_interval_seconds{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "interval": "30s", + "intervalFactor": 1, + "refId": "A" + } + ], + "timeFrom": "1h", + "title": "Avg Block Time", + "type": "stat" + }, + { + "datasource": { + "uid": "$DS" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 3 + }, + "hideTimeOverride": false, + "id": 47, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_consensus_validators_power{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "30s", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Bonded Tokens", + "type": "stat" + }, + { + "aliasColors": { + "Height for last 3 hours": "#447ebc", + "Total Transactions for last 3 hours": "#ef843c" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$DS" + }, + "decimals": 0, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 7 + }, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_consensus_validators{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Active", + "refId": "A" + }, + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_consensus_missing_validators{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Missing", + "refId": "B" + }, + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_consensus_byzantine_validators{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Byzantine", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Validators", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "locale", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "none", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Height for last 3 hours": "#447ebc", + "Total Transactions for last 3 hours": "#ef843c" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$DS" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 7 + }, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": 350, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_consensus_validators_power{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Online", + "refId": "A" + }, + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_consensus_missing_validators_power{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Missing", + "refId": "B" + }, + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_consensus_byzantine_validators_power{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Byzantine", + "refId": "C" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Voting Power", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "none", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Height for last 3 hours": "#447ebc", + "Total Transactions for last 3 hours": "#ef843c" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$DS" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 16 + }, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 49, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_consensus_block_size_bytes{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Block Size", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Block Size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 2, + "format": "bytes", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "none", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": { + "Height for last 3 hours": "#447ebc", + "Total Transactions for last 3 hours": "#ef843c" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$DS" + }, + "decimals": 0, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 3, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 16 + }, + "hiddenSeries": false, + "hideTimeOverride": false, + "id": 50, + "legend": { + "alignAsTable": false, + "avg": true, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_consensus_num_txs{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Transactions", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Transactions", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "decimals": 0, + "format": "short", + "label": "", + "logBase": 1, + "min": "0", + "show": true + }, + { + "format": "none", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 55, + "panels": [], + "repeat": "instance", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "refId": "A" + } + ], + "title": "instance overview: $instance", + "type": "row" + }, + { + "datasource": { + "uid": "$DS" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "max": 20, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#e24d42", + "value": null + }, + { + "color": "#ef843c", + "value": 2 + }, + { + "color": "#7eb26d", + "value": 5 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 22 + }, + "hideTimeOverride": true, + "id": 53, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_p2p_peers{chain_id=\"$chain_id\", instance=~\"$instance\"}", + "format": "time_series", + "instant": true, + "interval": "30s", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Connected Peers", + "type": "gauge" + }, + { + "datasource": { + "uid": "$DS" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "max": 50, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#7eb26d", + "value": null + }, + { + "color": "#ef843c", + "value": 10 + }, + { + "color": "#e24d42", + "value": 20 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 22 + }, + "hideTimeOverride": true, + "id": 56, + "links": [], + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_mempool_size{chain_id=\"$chain_id\", instance=~\"$instance\"}", + "format": "time_series", + "instant": true, + "interval": "30s", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Unconfirmed Transactions", + "type": "gauge" + }, + { + "datasource": { + "uid": "$DS" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 22 + }, + "hideTimeOverride": true, + "id": 60, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_mempool_failed_txs{chain_id=\"$chain_id\", instance=~\"$instance\"}", + "format": "time_series", + "instant": true, + "interval": "30s", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Failed Transactions", + "type": "stat" + }, + { + "datasource": { + "uid": "$DS" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 22 + }, + "hideTimeOverride": true, + "id": 61, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_mempool_recheck_times{chain_id=\"$chain_id\", instance=~\"$instance\"}", + "format": "time_series", + "instant": true, + "interval": "30s", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Recheck Times", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$DS" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 26 + }, + "hiddenSeries": false, + "id": 59, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_p2p_peer_receive_bytes_total{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{peer_id}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Network Input", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "series", + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$DS" + }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 26 + }, + "hiddenSeries": false, + "id": 58, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$DS" + }, + "expr": "tendermint_p2p_peer_send_bytes_total{chain_id=\"$chain_id\", instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{peer_id}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Total Network Output", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "series", + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [ + "Blockchain", + "Cosmos" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "athens_101-1", + "value": "athens_101-1" + }, + "datasource": { + "uid": "$DS" + }, + "definition": "label_values(tendermint_consensus_height, chain_id)", + "hide": 0, + "includeAll": false, + "label": "Chain ID", + "multi": false, + "name": "chain_id", + "options": [], + "query": "label_values(tendermint_consensus_height, chain_id)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": "", + "current": { + "selected": false, + "text": "172.20.0.11:26660", + "value": "172.20.0.11:26660" + }, + "datasource": { + "uid": "$DS" + }, + "definition": "label_values(tendermint_consensus_height{chain_id=\"$chain_id\"}, instance)", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(tendermint_consensus_height{chain_id=\"$chain_id\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 5, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now/d", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Cosmos Dashboard", + "uid": "UJyurCTWz", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/contrib/localnet/grafana/dashboards/default.yaml b/contrib/localnet/grafana/dashboards/default.yaml new file mode 100644 index 0000000000..581a362406 --- /dev/null +++ b/contrib/localnet/grafana/dashboards/default.yaml @@ -0,0 +1,8 @@ +apiVersion: 1 + +providers: + - name: Default # A uniquely identifiable name for the provider + folder: Dashboards # The folder where to place the dashboards + type: file + options: + path: /etc/grafana/provisioning/dashboards # Path to dashboard files on disk (required) \ No newline at end of file diff --git a/contrib/localnet/grafana/dashboards/eth_balance.json b/contrib/localnet/grafana/dashboards/eth_balance.json new file mode 100644 index 0000000000..7cafe4c4d9 --- /dev/null +++ b/contrib/localnet/grafana/dashboards/eth_balance.json @@ -0,0 +1,955 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Graph Ethereum Wallet Balances", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 6970, + "graphTooltip": 0, + "id": 4, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 0 + }, + "id": 3, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "eth_total_addresses", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Total Addresses Logged", + "type": "stat" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 2, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 16, + "x": 4, + "y": 0 + }, + "hiddenSeries": false, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "topk(5, eth_balance)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{name}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Top 5 Balances", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "#fce2de", + "value": 0 + }, + { + "color": "#d44a3a", + "value": 100 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 0 + }, + "id": 11, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "(eth_balance_total - eth_balance_total offset 1m) / eth_balance_total", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Total ETH 1 Hour Change", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 2, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "eth_load_seconds", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Total Load Time", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "#fce2de", + "value": 0 + }, + { + "color": "#d44a3a", + "value": 100 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 4 + }, + "id": 12, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "(eth_balance_total - eth_balance_total offset 24h) / eth_balance_total", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Total ETH 24 Hour Change", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 14, + "links": [], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
${name}
", + "mode": "html" + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "refId": "A" + } + ], + "type": "text" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fill": 7, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 11 + }, + "hiddenSeries": false, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.2.2", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "eth_balance{name=\"${name}\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "${name} ETH", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "${name} Balance", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "locale", + "label": "ETH", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 14, + "y": 11 + }, + "id": 7, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "eth_balance{name=\"${name}\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "${name} ETH", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "#fce2de", + "value": 0 + }, + { + "color": "#d44a3a", + "value": 100 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 11 + }, + "id": 8, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "(eth_balance{name=\"${name}\"} - eth_balance{name=\"${name}\"} offset 1h) / eth_balance{name=\"${name}\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "${name} ETH 1 Hour Change", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "#fce2de", + "value": 0 + }, + { + "color": "#d44a3a", + "value": 100 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 14, + "y": 15 + }, + "id": 10, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "(eth_balance{name=\"${name}\"} - eth_balance{name=\"${name}\"} offset 24h) / eth_balance{name=\"${name}\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "${name} ETH 24 Hour Change", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "#fce2de", + "value": 0 + }, + { + "color": "#d44a3a", + "value": 100 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 15 + }, + "id": 9, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "expr": "(eth_balance{name=\"${name}\"} - eth_balance{name=\"${name}\"} offset 7d) / eth_balance{name=\"${name}\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "${name} ETH Week Change", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "ethTSS", + "value": "ethTSS" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "", + "hide": 0, + "includeAll": false, + "label": "name", + "multi": false, + "name": "name", + "options": [], + "query": "label_values(eth_balance, name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Ethereum Wallet Monitor", + "uid": "pgGHUOdmv", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/contrib/localnet/grafana/dashboards/zetaclient_dashboard.json b/contrib/localnet/grafana/dashboards/zetaclient_dashboard.json new file mode 100644 index 0000000000..68194ee1f9 --- /dev/null +++ b/contrib/localnet/grafana/dashboards/zetaclient_dashboard.json @@ -0,0 +1,551 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard for Zetaclient Metrics", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 6, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "title": "Zetaclient Dashboard", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "zetaclient_pending_txs_goerli_localnet", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Pending Transaction Goerli Localnet", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "zetaclient_hotkey_burn_rate", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Hotkey Burn Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "zetaclient_pending_txs_btc_regtest", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Pending Transactions BTC", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "zetaclient_Outbound_tx_sign_count{instance=\"172.20.0.21:8886\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Outbound Transaction Sign Count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "zetaclient_rpc_getBlockByNumber_count_goerli_localnet", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "BlockByNo Count Goerli/Localnet", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 38, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Zetaclient Dashboard", + "uid": "efac1ac6-2adb-4123-8833-a8634c5fa64d", + "version": 9, + "weekStart": "" +} \ No newline at end of file diff --git a/contrib/localnet/grafana/datasource.yaml b/contrib/localnet/grafana/datasource.yaml new file mode 100644 index 0000000000..59cd333a94 --- /dev/null +++ b/contrib/localnet/grafana/datasource.yaml @@ -0,0 +1,16 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + # Access mode - proxy (server in the UI) or direct (browser in the UI). + url: http://prometheus:9090 + jsonData: + httpMethod: POST + manageAlerts: true + prometheusType: Prometheus + prometheusVersion: 2.48.1 + cacheLevel: 'High' + disableRecordingRules: false + incrementalQueryOverlapWindow: 10m \ No newline at end of file diff --git a/contrib/localnet/grafana/get-tss-address.sh b/contrib/localnet/grafana/get-tss-address.sh new file mode 100755 index 0000000000..eae4b031af --- /dev/null +++ b/contrib/localnet/grafana/get-tss-address.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +/usr/sbin/sshd + +#retrieve value of ETH TSS Address from localnet +ethTSS_address=$(curl 'http://localhost:1317/zeta-chain/observer/get_tss_address' | jq -r '.eth') + +#write value of ETH TSS Address to addresses.txt file +printf "ethTSS:$ethTSS_address\n" > addresses.txt \ No newline at end of file diff --git a/contrib/localnet/prometheus/prometheus.yml b/contrib/localnet/prometheus/prometheus.yml new file mode 100644 index 0000000000..3c73f90385 --- /dev/null +++ b/contrib/localnet/prometheus/prometheus.yml @@ -0,0 +1,39 @@ +# my global config +global: + scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + # - alertmanager:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: +# - "first_rules.yml" +# - "second_rules.yml" + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + # The job name is added as a label `job=` to any timeseries scraped from this config. + - job_name: "zetacore" + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + static_configs: + - targets: ["172.20.0.11:26660"] + + - job_name: "zetaclient" + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + static_configs: + - targets: ["172.20.0.21:8886"] + + - job_name: "ethbalance" + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + static_configs: + - targets: ["172.20.0.32:9015"] \ No newline at end of file diff --git a/contrib/localnet/zetacored/common/app.toml b/contrib/localnet/zetacored/common/app.toml index 624e89d40c..7bbf48cf96 100644 --- a/contrib/localnet/zetacored/common/app.toml +++ b/contrib/localnet/zetacored/common/app.toml @@ -80,7 +80,7 @@ service-name = "" # Enabled enables the application telemetry functionality. When enabled, # an in-memory sink is also enabled by default. Operators may also enabled # other sinks such as Prometheus. -enabled = false +enabled = true # Enable prefixing gauge values with hostname. enable-hostname = false diff --git a/contrib/localnet/zetacored/common/config.toml b/contrib/localnet/zetacored/common/config.toml index 29b433e824..c0f955ea4b 100644 --- a/contrib/localnet/zetacored/common/config.toml +++ b/contrib/localnet/zetacored/common/config.toml @@ -451,7 +451,7 @@ psql-conn = "" # When true, Prometheus metrics are served under /metrics on # PrometheusListenAddr. # Check out the documentation for the list of available metrics. -prometheus = false +prometheus = true # Address to listen for Prometheus collector(s) connections prometheus_listen_addr = ":26660" diff --git a/go.mod b/go.mod index a8dd44e772..b583a97b16 100644 --- a/go.mod +++ b/go.mod @@ -52,7 +52,7 @@ require ( require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect - github.com/mattn/go-sqlite3 v1.14.16 // indirect + github.com/mattn/go-sqlite3 v1.14.19 // indirect gorm.io/driver/sqlite v1.4.4 gorm.io/gorm v1.24.6 ) diff --git a/go.sum b/go.sum index 238beaf820..da78e7283c 100644 --- a/go.sum +++ b/go.sum @@ -2154,8 +2154,8 @@ github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOq github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI= +github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= diff --git a/zetaclient/bitcoin_client.go b/zetaclient/bitcoin_client.go index 4e0651b45e..2b9f242b56 100644 --- a/zetaclient/bitcoin_client.go +++ b/zetaclient/bitcoin_client.go @@ -57,9 +57,9 @@ type BitcoinChainClient struct { Mu *sync.Mutex // lock for all the maps, utxos and core params pendingNonce uint64 - includedTxHashes map[string]uint64 // key: tx hash - includedTxResults map[string]btcjson.GetTransactionResult // key: chain-tss-nonce - broadcastedTx map[string]string // key: chain-tss-nonce, value: outTx hash + includedTxHashes map[string]uint64 // key: tx hash + includedTxResults map[string]*btcjson.GetTransactionResult // key: chain-tss-nonce + broadcastedTx map[string]string // key: chain-tss-nonce, value: outTx hash utxos []btcjson.ListUnspentResult params observertypes.ChainParams @@ -148,7 +148,7 @@ func NewBitcoinClient( ob.zetaClient = bridge ob.Tss = tss ob.includedTxHashes = make(map[string]uint64) - ob.includedTxResults = make(map[string]btcjson.GetTransactionResult) + ob.includedTxResults = make(map[string]*btcjson.GetTransactionResult) ob.broadcastedTx = make(map[string]string) ob.params = btcCfg.ChainParams @@ -442,24 +442,23 @@ func (ob *BitcoinChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64 } // Try including this outTx broadcasted by myself - inMempool, err := ob.checkNSaveIncludedTx(txnHash, params) - if err != nil { - ob.logger.ObserveOutTx.Error().Err(err).Msg("IsSendOutTxProcessed: checkNSaveIncludedTx failed") + txResult, inMempool := ob.checkIncludedTx(txnHash, params) + if txResult == nil { + ob.logger.ObserveOutTx.Error().Err(err).Msg("IsSendOutTxProcessed: checkIncludedTx failed") return false, false, err - } - if inMempool { // to avoid unnecessary Tss keysign + } else if inMempool { // still in mempool (should avoid unnecessary Tss keysign) ob.logger.ObserveOutTx.Info().Msgf("IsSendOutTxProcessed: outTx %s is still in mempool", outTxID) return true, false, nil + } else { // included + ob.setIncludedTx(nonce, txResult) } // Get tx result again in case it is just included - ob.Mu.Lock() - res, included = ob.includedTxResults[outTxID] - ob.Mu.Unlock() - if !included { + res = ob.getIncludedTx(nonce) + if res == nil { return false, false, nil } - ob.logger.ObserveOutTx.Info().Msgf("IsSendOutTxProcessed: checkNSaveIncludedTx succeeded for outTx %s", outTxID) + ob.logger.ObserveOutTx.Info().Msgf("IsSendOutTxProcessed: setIncludedTx succeeded for outTx %s", outTxID) } // It's safe to use cctx's amount to post confirmation because it has already been verified in observeOutTx() @@ -830,14 +829,11 @@ func (ob *BitcoinChainClient) refreshPendingNonce() { } func (ob *BitcoinChainClient) getOutTxidByNonce(nonce uint64, test bool) (string, error) { - ob.Mu.Lock() - res, included := ob.includedTxResults[ob.GetTxID(nonce)] - ob.Mu.Unlock() // There are 2 types of txids an observer can trust // 1. The ones had been verified and saved by observer self. // 2. The ones had been finalized in zetacore based on majority vote. - if included { + if res := ob.getIncludedTx(nonce); res != nil { return res.TxID, nil } if !test { // if not unit test, get cctx from zetacore @@ -1020,13 +1016,28 @@ func (ob *BitcoinChainClient) observeOutTx() { if len(tracker.HashList) > 1 { ob.logger.ObserveOutTx.Warn().Msgf("observeOutTx: oops, outTxID %s got multiple (%d) outTx hashes", outTxID, len(tracker.HashList)) } - // verify outTx hashes + // iterate over all txHashes to find the truly included one. + // we do it this (inefficient) way because we don't rely on the first one as it may be a false positive (for unknown reason). + txCount := 0 + var txResult *btcjson.GetTransactionResult for _, txHash := range tracker.HashList { - _, err := ob.checkNSaveIncludedTx(txHash.TxHash, params) - if err != nil { - ob.logger.ObserveOutTx.Error().Err(err).Msg("observeOutTx: checkNSaveIncludedTx failed") + result, inMempool := ob.checkIncludedTx(txHash.TxHash, params) + if result != nil && !inMempool { // included + txCount++ + txResult = result + ob.logger.ObserveOutTx.Info().Msgf("observeOutTx: included outTx %s for chain %d nonce %d", txHash.TxHash, ob.chain.ChainId, tracker.Nonce) + if txCount > 1 { + ob.logger.ObserveOutTx.Error().Msgf( + "observeOutTx: checkIncludedTx passed, txCount %d chain %d nonce %d result %v", txCount, ob.chain.ChainId, tracker.Nonce, result) + } } } + if txCount == 1 { // should be only one txHash included for each nonce + ob.setIncludedTx(tracker.Nonce, txResult) + } else if txCount > 1 { + ob.removeIncludedTx(tracker.Nonce) // we can't tell which txHash is true, so we remove all (if any) to be safe + ob.logger.ObserveOutTx.Error().Msgf("observeOutTx: included multiple (%d) outTx for chain %d nonce %d", txCount, ob.chain.ChainId, tracker.Nonce) + } } ticker.UpdateInterval(ob.GetChainParams().OutTxTicker, ob.logger.ObserveOutTx) case <-ob.stop: @@ -1036,50 +1047,69 @@ func (ob *BitcoinChainClient) observeOutTx() { } } -// checkNSaveIncludedTx either includes a new outTx or update an existing outTx result. -// Returns inMempool, error -func (ob *BitcoinChainClient) checkNSaveIncludedTx(txHash string, params types.OutboundTxParams) (bool, error) { +// checkIncludedTx checks if a txHash is included and returns (txResult, inMempool) +// Note: if txResult is nil, then inMempool flag should be ignored. +func (ob *BitcoinChainClient) checkIncludedTx(txHash string, params types.OutboundTxParams) (*btcjson.GetTransactionResult, bool) { outTxID := ob.GetTxID(params.OutboundTxTssNonce) hash, getTxResult, err := ob.GetTxResultByHash(txHash) if err != nil { - return false, errors.Wrapf(err, "checkNSaveIncludedTx: error GetTxResultByHash: %s", txHash) + ob.logger.ObserveOutTx.Error().Err(err).Msgf("checkIncludedTx: error GetTxResultByHash: %s", txHash) + return nil, false + } + if txHash != getTxResult.TxID { // just in case, we'll use getTxResult.TxID later + ob.logger.ObserveOutTx.Error().Msgf("checkIncludedTx: inconsistent txHash %s and getTxResult.TxID %s", txHash, getTxResult.TxID) + return nil, false } if getTxResult.Confirmations >= 0 { // check included tx only err = ob.checkTssOutTxResult(hash, getTxResult, params, params.OutboundTxTssNonce) if err != nil { - return false, errors.Wrapf(err, "checkNSaveIncludedTx: error verify bitcoin outTx %s outTxID %s", txHash, outTxID) + ob.logger.ObserveOutTx.Error().Err(err).Msgf("checkIncludedTx: error verify bitcoin outTx %s outTxID %s", txHash, outTxID) + return nil, false } + return getTxResult, false // included + } + return getTxResult, true // in mempool +} - ob.Mu.Lock() - defer ob.Mu.Unlock() - nonce, foundHash := ob.includedTxHashes[txHash] - res, foundRes := ob.includedTxResults[outTxID] - - // include new outTx and enforce rigid 1-to-1 mapping: outTxID(nonce) <===> txHash - if !foundHash && !foundRes { - ob.includedTxHashes[txHash] = params.OutboundTxTssNonce - ob.includedTxResults[outTxID] = *getTxResult - if params.OutboundTxTssNonce >= ob.pendingNonce { // try increasing pending nonce on every newly included outTx - ob.pendingNonce = params.OutboundTxTssNonce + 1 - } - ob.logger.ObserveOutTx.Info().Msgf("checkNSaveIncludedTx: included new bitcoin outTx %s outTxID %s pending nonce %d", txHash, outTxID, ob.pendingNonce) - } - // update saved tx result as confirmations may increase - if foundHash && foundRes { - ob.includedTxResults[outTxID] = *getTxResult - if getTxResult.Confirmations > res.Confirmations { - ob.logger.ObserveOutTx.Info().Msgf("checkNSaveIncludedTx: bitcoin outTx %s got confirmations %d", txHash, getTxResult.Confirmations) - } - } - if !foundHash && foundRes { // be alert for duplicate payment!!! As we got a new hash paying same cctx. It might happen (e.g. majority of signers get crupted) - ob.logger.ObserveOutTx.Error().Msgf("checkNSaveIncludedTx: duplicate payment by bitcoin outTx %s outTxID %s, prior result %v, current result %v", txHash, outTxID, res, *getTxResult) +// setIncludedTx saves included tx result in memory +func (ob *BitcoinChainClient) setIncludedTx(nonce uint64, getTxResult *btcjson.GetTransactionResult) { + txHash := getTxResult.TxID + outTxID := ob.GetTxID(nonce) + + ob.Mu.Lock() + defer ob.Mu.Unlock() + res, found := ob.includedTxResults[outTxID] + + if !found { // not found. + ob.includedTxResults[outTxID] = getTxResult // include new outTx and enforce rigid 1-to-1 mapping: nonce <===> txHash + if nonce >= ob.pendingNonce { // try increasing pending nonce on every newly included outTx + ob.pendingNonce = nonce + 1 } - if foundHash && !foundRes { - ob.logger.ObserveOutTx.Error().Msgf("checkNSaveIncludedTx: unreachable code path! outTx %s outTxID %s, prior nonce %d, current nonce %d", txHash, outTxID, nonce, params.OutboundTxTssNonce) + ob.logger.ObserveOutTx.Info().Msgf("setIncludedTx: included new bitcoin outTx %s outTxID %s pending nonce %d", txHash, outTxID, ob.pendingNonce) + } else if txHash == res.TxID { // found same hash. + ob.includedTxResults[outTxID] = getTxResult // update tx result as confirmations may increase + if getTxResult.Confirmations > res.Confirmations { + ob.logger.ObserveOutTx.Info().Msgf("setIncludedTx: bitcoin outTx %s got confirmations %d", txHash, getTxResult.Confirmations) } - return false, nil + } else { // found other hash. + // be alert for duplicate payment!!! As we got a new hash paying same cctx (for whatever reason). + delete(ob.includedTxResults, outTxID) // we can't tell which txHash is true, so we remove all to be safe + ob.logger.ObserveOutTx.Error().Msgf("setIncludedTx: duplicate payment by bitcoin outTx %s outTxID %s, prior outTx %s", txHash, outTxID, res.TxID) } - return true, nil // in mempool +} + +// getIncludedTx gets the receipt and transaction from memory +func (ob *BitcoinChainClient) getIncludedTx(nonce uint64) *btcjson.GetTransactionResult { + ob.Mu.Lock() + defer ob.Mu.Unlock() + return ob.includedTxResults[ob.GetTxID(nonce)] +} + +// removeIncludedTx removes included tx's result from memory +func (ob *BitcoinChainClient) removeIncludedTx(nonce uint64) { + ob.Mu.Lock() + defer ob.Mu.Unlock() + delete(ob.includedTxResults, ob.GetTxID(nonce)) } // Basic TSS outTX checks: diff --git a/zetaclient/btc_signer_test.go b/zetaclient/btc_signer_test.go index 8bee941a98..5c12e67876 100644 --- a/zetaclient/btc_signer_test.go +++ b/zetaclient/btc_signer_test.go @@ -398,7 +398,7 @@ func createTestClient(t *testing.T) *BitcoinChainClient { client := &BitcoinChainClient{ Tss: tss, Mu: &sync.Mutex{}, - includedTxResults: make(map[string]btcjson.GetTransactionResult), + includedTxResults: make(map[string]*btcjson.GetTransactionResult), } // Create 10 dummy UTXOs (22.44 BTC in total) @@ -413,7 +413,7 @@ func createTestClient(t *testing.T) *BitcoinChainClient { func mineTxNSetNonceMark(ob *BitcoinChainClient, nonce uint64, txid string, preMarkIndex int) { // Mine transaction outTxID := ob.GetTxID(nonce) - ob.includedTxResults[outTxID] = btcjson.GetTransactionResult{TxID: txid} + ob.includedTxResults[outTxID] = &btcjson.GetTransactionResult{TxID: txid} // Set nonce mark tssAddress := ob.Tss.BTCAddressWitnessPubkeyHash().EncodeAddress() diff --git a/zetaclient/evm_client.go b/zetaclient/evm_client.go index 005fa10260..badce4eca6 100644 --- a/zetaclient/evm_client.go +++ b/zetaclient/evm_client.go @@ -296,11 +296,11 @@ func (ob *EVMChainClient) Stop() { // returns: isIncluded, isConfirmed, Error // If isConfirmed, it also post to ZetaCore func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, cointype common.CoinType, logger zerolog.Logger) (bool, bool, error) { - if !ob.isTxConfirmed(nonce) { - return false, false, nil - } params := ob.GetChainParams() receipt, transaction := ob.GetTxNReceipt(nonce) + if receipt == nil || transaction == nil { // not confirmed yet + return false, false, nil + } sendID := fmt.Sprintf("%s-%d", ob.chain.String(), nonce) logger = logger.With().Str("sendID", sendID).Logger() @@ -543,13 +543,6 @@ func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, co return false, false, nil } -// The lowest nonce we observe outTx for each chain -var lowestOutTxNonceToObserve = map[int64]uint64{ - 5: 113000, // Goerli - 97: 102600, // BSC testnet - 80001: 154500, // Mumbai -} - // FIXME: there's a chance that a txhash in OutTxChan may not deliver when Stop() is called // observeOutTx periodically checks all the txhash in potential outbound txs func (ob *EVMChainClient) observeOutTx() { @@ -558,7 +551,7 @@ func (ob *EVMChainClient) observeOutTx() { if err != nil || timeoutNonce <= 0 { timeoutNonce = 100 * 3 // process up to 100 hashes } - ob.logger.ObserveOutTx.Info().Msgf("observeOutTx using timeoutNonce %d seconds", timeoutNonce) + ob.logger.ObserveOutTx.Info().Msgf("observeOutTx: using timeoutNonce %d seconds", timeoutNonce) ticker, err := NewDynamicTicker(fmt.Sprintf("EVM_observeOutTx_%d", ob.chain.ChainId), ob.GetChainParams().OutTxTicker) if err != nil { @@ -577,28 +570,37 @@ func (ob *EVMChainClient) observeOutTx() { //FIXME: remove this timeout here to ensure that all trackers are queried outTimeout := time.After(time.Duration(timeoutNonce) * time.Second) TRACKERLOOP: - // Skip old gabbage trackers as we spent too much time on querying them for _, tracker := range trackers { nonceInt := tracker.Nonce - if nonceInt < lowestOutTxNonceToObserve[ob.chain.ChainId] { - continue - } if ob.isTxConfirmed(nonceInt) { // Go to next tracker if this one already has a confirmed tx continue } + txCount := 0 + var receipt *ethtypes.Receipt + var transaction *ethtypes.Transaction for _, txHash := range tracker.HashList { select { case <-outTimeout: - ob.logger.ObserveOutTx.Warn().Msgf("observeOutTx timeout on chain %d nonce %d", ob.chain.ChainId, nonceInt) + ob.logger.ObserveOutTx.Warn().Msgf("observeOutTx: timeout on chain %d nonce %d", ob.chain.ChainId, nonceInt) break TRACKERLOOP default: - if ob.confirmTxByHash(txHash.TxHash, nonceInt) { - ob.logger.ObserveOutTx.Info().Msgf("observeOutTx confirmed outTx %s for chain %d nonce %d", txHash.TxHash, ob.chain.ChainId, nonceInt) - break + if recpt, tx, ok := ob.checkConfirmedTx(txHash.TxHash, nonceInt); ok { + txCount++ + receipt = recpt + transaction = tx + ob.logger.ObserveOutTx.Info().Msgf("observeOutTx: confirmed outTx %s for chain %d nonce %d", txHash.TxHash, ob.chain.ChainId, nonceInt) + if txCount > 1 { + ob.logger.ObserveOutTx.Error().Msgf( + "observeOutTx: checkConfirmedTx passed, txCount %d chain %d nonce %d receipt %v transaction %v", txCount, ob.chain.ChainId, nonceInt, receipt, transaction) + } } - ob.logger.ObserveOutTx.Debug().Msgf("observeOutTx outTx %s for chain %d nonce %d not confirmed yet", txHash.TxHash, ob.chain.ChainId, nonceInt) } } + if txCount == 1 { // should be only one txHash confirmed for each nonce. + ob.SetTxNReceipt(nonceInt, receipt, transaction) + } else if txCount > 1 { // should not happen. We can't tell which txHash is true. It might happen (e.g. glitchy/hacked endpoint) + ob.logger.ObserveOutTx.Error().Msgf("observeOutTx: confirmed multiple (%d) outTx for chain %d nonce %d", txCount, ob.chain.ChainId, nonceInt) + } } ticker.UpdateInterval(ob.GetChainParams().OutTxTicker, ob.logger.ObserveOutTx) case <-ob.stop: @@ -611,47 +613,45 @@ func (ob *EVMChainClient) observeOutTx() { // SetPendingTx sets the pending transaction in memory func (ob *EVMChainClient) SetPendingTx(nonce uint64, transaction *ethtypes.Transaction) { ob.Mu.Lock() + defer ob.Mu.Unlock() ob.outTxPendingTransactions[ob.GetTxID(nonce)] = transaction - ob.Mu.Unlock() } // GetPendingTx gets the pending transaction from memory func (ob *EVMChainClient) GetPendingTx(nonce uint64) *ethtypes.Transaction { ob.Mu.Lock() - transaction := ob.outTxPendingTransactions[ob.GetTxID(nonce)] - ob.Mu.Unlock() - return transaction + defer ob.Mu.Unlock() + return ob.outTxPendingTransactions[ob.GetTxID(nonce)] } // SetTxNReceipt sets the receipt and transaction in memory func (ob *EVMChainClient) SetTxNReceipt(nonce uint64, receipt *ethtypes.Receipt, transaction *ethtypes.Transaction) { ob.Mu.Lock() + defer ob.Mu.Unlock() delete(ob.outTxPendingTransactions, ob.GetTxID(nonce)) // remove pending transaction, if any ob.outTXConfirmedReceipts[ob.GetTxID(nonce)] = receipt ob.outTXConfirmedTransactions[ob.GetTxID(nonce)] = transaction - ob.Mu.Unlock() } -// getTxNReceipt gets the receipt and transaction from memory +// GetTxNReceipt gets the receipt and transaction from memory func (ob *EVMChainClient) GetTxNReceipt(nonce uint64) (*ethtypes.Receipt, *ethtypes.Transaction) { ob.Mu.Lock() + defer ob.Mu.Unlock() receipt := ob.outTXConfirmedReceipts[ob.GetTxID(nonce)] transaction := ob.outTXConfirmedTransactions[ob.GetTxID(nonce)] - ob.Mu.Unlock() return receipt, transaction } // isTxConfirmed returns true if there is a confirmed tx for 'nonce' func (ob *EVMChainClient) isTxConfirmed(nonce uint64) bool { ob.Mu.Lock() - confirmed := ob.outTXConfirmedReceipts[ob.GetTxID(nonce)] != nil && ob.outTXConfirmedTransactions[ob.GetTxID(nonce)] != nil - ob.Mu.Unlock() - return confirmed + defer ob.Mu.Unlock() + return ob.outTXConfirmedReceipts[ob.GetTxID(nonce)] != nil && ob.outTXConfirmedTransactions[ob.GetTxID(nonce)] != nil } -// confirmTxByHash checks if a txHash is confirmed and saves transaction and receipt in memory -// returns true if confirmed or false otherwise -func (ob *EVMChainClient) confirmTxByHash(txHash string, nonce uint64) bool { +// checkConfirmedTx checks if a txHash is confirmed +// returns (receipt, transaction, true) if confirmed or (nil, nil, false) otherwise +func (ob *EVMChainClient) checkConfirmedTx(txHash string, nonce uint64) (*ethtypes.Receipt, *ethtypes.Transaction, bool) { ctxt, cancel := context.WithTimeout(context.Background(), 3*time.Second) defer cancel() @@ -659,15 +659,34 @@ func (ob *EVMChainClient) confirmTxByHash(txHash string, nonce uint64) bool { transaction, isPending, err := ob.evmClient.TransactionByHash(ctxt, ethcommon.HexToHash(txHash)) if err != nil { log.Error().Err(err).Msgf("confirmTxByHash: TransactionByHash error, txHash %s nonce %d", txHash, nonce) - return false + return nil, nil, false } if transaction == nil { // should not happen log.Error().Msgf("confirmTxByHash: transaction is nil for txHash %s nonce %d", txHash, nonce) - return false + return nil, nil, false + } + + // check tx sender and nonce + signer := ethtypes.NewLondonSigner(big.NewInt(ob.chain.ChainId)) + from, err := signer.Sender(transaction) + if err != nil { + log.Error().Err(err).Msgf("confirmTxByHash: local recovery of sender address failed for txHash %s chain %d", transaction.Hash().Hex(), ob.chain.ChainId) + return nil, nil, false + } + if from != ob.Tss.EVMAddress() { // must be TSS address + log.Error().Msgf("confirmTxByHash: sender %s for txHash %s chain %d is not TSS address %s", + from.Hex(), transaction.Hash().Hex(), ob.chain.ChainId, ob.Tss.EVMAddress().Hex()) + return nil, nil, false } - if isPending { // save pending transaction + if transaction.Nonce() != nonce { // must match cctx nonce + log.Error().Msgf("confirmTxByHash: txHash %s nonce mismatch: wanted %d, got tx nonce %d", txHash, nonce, transaction.Nonce()) + return nil, nil, false + } + + // save pending transaction + if isPending { ob.SetPendingTx(nonce, transaction) - return false + return nil, nil, false } // query receipt @@ -676,33 +695,26 @@ func (ob *EVMChainClient) confirmTxByHash(txHash string, nonce uint64) bool { if err != ethereum.NotFound { log.Warn().Err(err).Msgf("confirmTxByHash: TransactionReceipt error, txHash %s nonce %d", txHash, nonce) } - return false + return nil, nil, false } if receipt == nil { // should not happen log.Error().Msgf("confirmTxByHash: receipt is nil for txHash %s nonce %d", txHash, nonce) - return false + return nil, nil, false } - // check nonce and confirmations - if transaction.Nonce() != nonce { - log.Error().Msgf("confirmTxByHash: txHash %s nonce mismatch: wanted %d, got tx nonce %d", txHash, nonce, transaction.Nonce()) - return false - } + // check confirmations confHeight := receipt.BlockNumber.Uint64() + ob.GetChainParams().ConfirmationCount if confHeight >= math.MaxInt64 { log.Error().Msgf("confirmTxByHash: confHeight is too large for txHash %s nonce %d", txHash, nonce) - return false + return nil, nil, false } if confHeight > ob.GetLastBlockHeight() { log.Info().Msgf("confirmTxByHash: txHash %s nonce %d included but not confirmed: receipt block %d, current block %d", txHash, nonce, receipt.BlockNumber, ob.GetLastBlockHeight()) - return false + return nil, nil, false } - // confirmed, save receipt and transaction - ob.SetTxNReceipt(nonce, receipt, transaction) - - return true + return receipt, transaction, true } // SetLastBlockHeightScanned set last block height scanned (not necessarily caught up with external block; could be slow/paused)