diff --git a/.github/workflows/change-log-check.yml b/.github/workflows/change-log-check.yml
index 1e5374179f..6fe9ac2a9f 100644
--- a/.github/workflows/change-log-check.yml
+++ b/.github/workflows/change-log-check.yml
@@ -17,7 +17,7 @@ jobs:
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-changelog') }}
run: |
git fetch origin ${{ github.base_ref }}
- CHANGELOG_DIFF=$(git diff origin/${{ github.base_ref }} origin/${{ github.head_ref }} -- changelog.md)
+ CHANGELOG_DIFF=$(git diff ${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }} -- changelog.md)
echo "${CHANGELOG_DIFF}"
if [ -z "$CHANGELOG_DIFF" ]; then
echo "ERROR: No changes detected in CHANGELOG.md. Please update the changelog."
diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml
index 079b8dfa92..936823a405 100644
--- a/.github/workflows/labeler.yml
+++ b/.github/workflows/labeler.yml
@@ -1,6 +1,6 @@
name: "Pull Request Labeler"
on:
- pull_request:
+ pull_request_target:
types:
- opened
- edited
diff --git a/.github/workflows/publish-typescript.yaml b/.github/workflows/publish-typescript.yaml
index a5906d5047..890f867d8b 100644
--- a/.github/workflows/publish-typescript.yaml
+++ b/.github/workflows/publish-typescript.yaml
@@ -16,16 +16,20 @@ jobs:
node-version: '20.x'
registry-url: 'https://registry.npmjs.org'
+ - name: Install buf
+ run: |
+ curl -sSL https://github.com/bufbuild/buf/releases/download/v1.28.1/buf-Linux-x86_64 -o /usr/local/bin/buf
+ chmod +x /usr/local/bin/buf
+
+ - name: Generate
+ run: |
+ make typescript
+
- name: Set Version
working-directory: typescript
run: |
- version=$(cat app/setup_handlers.go | grep "const releaseVersion" | cut -d ' ' -f4 | tr -d '"')
+ version=$(cat ../app/setup_handlers.go | grep "const releaseVersion" | cut -d ' ' -f4 | tr -d '"')
npm version ${version}
- sed -i 's/@zetachain\/blockchain-types/@zetachain\/node-types/' package.json
-
- - name: Install dependencies and build 🔧
- working-directory: typescript
- run: npm ci && npm run build
- name: Publish package on NPM 📦
run: npm publish
diff --git a/.gitignore b/.gitignore
index d53a2695bd..6a53c9be7b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -46,3 +46,11 @@ localnet/ganache/storage
localnet/zetachain-monorepo
/standalone-network/authz/tx.json
/zetanode.log
+
+# Blockscout Files
+contrib/localnet/blockscout/services/blockscout-db-data/*
+contrib/localnet/blockscout/services/logs/*
+contrib/localnet/blockscout/services/redis-data/*
+contrib/localnet/blockscout/services/stats-db-data/*
+contrib/localnet/grafana/addresses.txt
+contrib/localnet/addresses.txt
\ No newline at end of file
diff --git a/Makefile b/Makefile
index d6be4b1273..34862f40e3 100644
--- a/Makefile
+++ b/Makefile
@@ -247,6 +247,19 @@ stateful-upgrade-source:
stop-stateful-upgrade:
cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-stateful.yml down --remove-orphans
+###############################################################################
+### Monitoring ###
+###############################################################################
+
+start-monitoring:
+ @echo "Starting monitoring services"
+ cd contrib/localnet/grafana/ && ./get-tss-address.sh
+ cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-monitoring.yml up -d
+
+stop-monitoring:
+ @echo "Stopping monitoring services"
+ cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-monitoring.yml down
+
###############################################################################
### GoReleaser ###
###############################################################################
diff --git a/changelog.md b/changelog.md
index 53b2621d27..0498a869e1 100644
--- a/changelog.md
+++ b/changelog.md
@@ -11,7 +11,7 @@
* ChainNoncesAll :Changed from `/zeta-chain/observer/chainNonces` to `/zeta-chain/observer/chainNonces` . It returns all the chain nonces for all chains. This returns the current nonce of the TSS address for all chains.
### Features
-
+* [1498](https://github.com/zeta-chain/node/pull/1498) - Add monitoring(grafana, prometheus, ethbalance) for localnet testing
* [1395](https://github.com/zeta-chain/node/pull/1395) - Add state variable to track aborted zeta amount
* [1410](https://github.com/zeta-chain/node/pull/1410) - `snapshots` commands
* enable zetaclients to use dynamic gas price on zetachain - enables >0 min_gas_price in feemarket module
@@ -20,6 +20,7 @@
### Fixes
+* [1530](https://github.com/zeta-chain/node/pull/1530) - Outbound tx confirmation/inclusion enhancement
* [1496](https://github.com/zeta-chain/node/issues/1496) - post block header for enabled EVM chains only
* [1518](https://github.com/zeta-chain/node/pull/1518) - Avoid duplicate keysign if an outTx is already pending
* fix Code4rena issue - zetaclients potentially miss inTx when PostSend (or other RPC) fails
diff --git a/contrib/localnet/README.md b/contrib/localnet/README.md
index 467e0bc877..96143b06d5 100644
--- a/contrib/localnet/README.md
+++ b/contrib/localnet/README.md
@@ -76,6 +76,30 @@ which does the following docker compose command:
# in zeta-node/contrib/localnet/orchestrator
$ docker compose down --remove-orphans
```
+### Run monitoring setup
+Before starting the monitoring setup, make sure the Zetacore API is up at http://localhost:1317.
+You can also add any additional ETH addresses to monitor in zeta-node/contrib/localnet/grafana/addresses.txt file
+```bash
+# in zeta-node/
+make start-monitoring
+```
+which does the following docker compose command:
+```bash
+# in zeta-node/contrib/localnet/
+$ docker compose -f docker-compose-monitoring.yml up -d
+```
+### Grafana credentials and dashboards
+The Grafana default credentials are admin:admin. The dashboards are located at http://localhost:3000.
+### Stop monitoring setup
+```bash
+# in zeta-node/
+make stop-monitoring
+```
+which does the following docker compose command:
+```bash
+# in zeta-node/contrib/localnet/
+$ docker compose -f docker-compose-monitoring.yml down --remove-orphans
+```
## Useful data
diff --git a/contrib/localnet/docker-compose-monitoring.yml b/contrib/localnet/docker-compose-monitoring.yml
new file mode 100644
index 0000000000..ad66f828ec
--- /dev/null
+++ b/contrib/localnet/docker-compose-monitoring.yml
@@ -0,0 +1,47 @@
+version: '3'
+
+services:
+ grafana:
+ image: grafana/grafana:latest
+ container_name: grafana
+ hostname: grafana
+ volumes:
+ - ./grafana/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml
+ - ./grafana/dashboards/:/etc/grafana/provisioning/dashboards
+ ports:
+ - "3000:3000"
+ networks:
+ mynetwork:
+ ipv4_address: 172.20.0.30
+
+ prometheus:
+ image: prom/prometheus:latest
+ container_name: prometheus
+ hostname: prometheus
+ volumes:
+ - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+ ports:
+ - "9090:9090"
+ networks:
+ mynetwork:
+ ipv4_address: 172.20.0.31
+
+ zetachain-exporter:
+ image: zetachain/zetachain-exporter:latest
+ container_name: zetachain-exporter
+ hostname: zetachain-exporter
+ volumes:
+ - ./grafana/addresses.txt:/app/addresses.txt
+ ports:
+ - "9015:9015"
+ networks:
+ mynetwork:
+ ipv4_address: 172.20.0.32
+ environment:
+ - GETH=http://eth:8545
+
+networks:
+ mynetwork:
+ ipam:
+ config:
+ - subnet: 172.20.0.0/24
\ No newline at end of file
diff --git a/contrib/localnet/grafana/dashboards/cosmos_dashboard.json b/contrib/localnet/grafana/dashboards/cosmos_dashboard.json
new file mode 100644
index 0000000000..85ca855f22
--- /dev/null
+++ b/contrib/localnet/grafana/dashboards/cosmos_dashboard.json
@@ -0,0 +1,1458 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "A Grafana dashboard compatible with all the cosmos-sdk and tendermint based blockchains.",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "gnetId": 11036,
+ "graphTooltip": 0,
+ "id": 1,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 64,
+ "panels": [],
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "$chain_id overview",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "gridPos": {
+ "h": 2,
+ "w": 24,
+ "x": 0,
+ "y": 1
+ },
+ "id": 62,
+ "links": [],
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "",
+ "mode": "html"
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "refId": "A"
+ }
+ ],
+ "transparent": true,
+ "type": "text"
+ },
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 0,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "locale"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 0,
+ "y": 3
+ },
+ "hideTimeOverride": false,
+ "id": 4,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_consensus_height{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "hide": false,
+ "instant": true,
+ "interval": "30s",
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "Block Height",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 0,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "locale"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 6,
+ "y": 3
+ },
+ "hideTimeOverride": false,
+ "id": 40,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_consensus_total_txs{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "interval": "30s",
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "Total Transactions",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 2,
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 12,
+ "y": 3
+ },
+ "hideTimeOverride": true,
+ "id": 39,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "mean"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_consensus_block_interval_seconds{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "interval": "30s",
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "timeFrom": "1h",
+ "title": "Avg Block Time",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 0,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 18,
+ "y": 3
+ },
+ "hideTimeOverride": false,
+ "id": 47,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "area",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_consensus_validators_power{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "instant": false,
+ "interval": "30s",
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "Bonded Tokens",
+ "type": "stat"
+ },
+ {
+ "aliasColors": {
+ "Height for last 3 hours": "#447ebc",
+ "Total Transactions for last 3 hours": "#ef843c"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$DS"
+ },
+ "decimals": 0,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 3,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 7
+ },
+ "hiddenSeries": false,
+ "hideTimeOverride": false,
+ "id": 15,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": false,
+ "show": true,
+ "sideWidth": 350,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "10.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_consensus_validators{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Active",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_consensus_missing_validators{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Missing",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_consensus_byzantine_validators{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Byzantine",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Validators",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 0,
+ "format": "locale",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "none",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {
+ "Height for last 3 hours": "#447ebc",
+ "Total Transactions for last 3 hours": "#ef843c"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$DS"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 3,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 7
+ },
+ "hiddenSeries": false,
+ "hideTimeOverride": false,
+ "id": 48,
+ "legend": {
+ "alignAsTable": true,
+ "avg": false,
+ "current": true,
+ "max": true,
+ "min": true,
+ "rightSide": false,
+ "show": true,
+ "sideWidth": 350,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "10.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_consensus_validators_power{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Online",
+ "refId": "A"
+ },
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_consensus_missing_validators_power{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Missing",
+ "refId": "B"
+ },
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_consensus_byzantine_validators_power{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "Byzantine",
+ "refId": "C"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Voting Power",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 0,
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "none",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {
+ "Height for last 3 hours": "#447ebc",
+ "Total Transactions for last 3 hours": "#ef843c"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$DS"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 3,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 0,
+ "y": 16
+ },
+ "hiddenSeries": false,
+ "hideTimeOverride": false,
+ "id": 49,
+ "legend": {
+ "alignAsTable": false,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": false,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "10.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_consensus_block_size_bytes{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Block Size",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Block Size",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 2,
+ "format": "bytes",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "none",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {
+ "Height for last 3 hours": "#447ebc",
+ "Total Transactions for last 3 hours": "#ef843c"
+ },
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$DS"
+ },
+ "decimals": 0,
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 3,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 5,
+ "w": 12,
+ "x": 12,
+ "y": 16
+ },
+ "hiddenSeries": false,
+ "hideTimeOverride": false,
+ "id": 50,
+ "legend": {
+ "alignAsTable": false,
+ "avg": true,
+ "current": false,
+ "max": true,
+ "min": false,
+ "rightSide": false,
+ "show": true,
+ "total": true,
+ "values": true
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "10.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_consensus_num_txs{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "instant": false,
+ "interval": "",
+ "intervalFactor": 1,
+ "legendFormat": "Transactions",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Transactions",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "decimals": 0,
+ "format": "short",
+ "label": "",
+ "logBase": 1,
+ "min": "0",
+ "show": true
+ },
+ {
+ "format": "none",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "collapsed": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 21
+ },
+ "id": 55,
+ "panels": [],
+ "repeat": "instance",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "refId": "A"
+ }
+ ],
+ "title": "instance overview: $instance",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 0,
+ "mappings": [],
+ "max": 20,
+ "min": 0,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#e24d42",
+ "value": null
+ },
+ {
+ "color": "#ef843c",
+ "value": 2
+ },
+ {
+ "color": "#7eb26d",
+ "value": 5
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 0,
+ "y": 22
+ },
+ "hideTimeOverride": true,
+ "id": 53,
+ "links": [],
+ "options": {
+ "minVizHeight": 75,
+ "minVizWidth": 75,
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_p2p_peers{chain_id=\"$chain_id\", instance=~\"$instance\"}",
+ "format": "time_series",
+ "instant": true,
+ "interval": "30s",
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "Connected Peers",
+ "type": "gauge"
+ },
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 0,
+ "mappings": [],
+ "max": 50,
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#7eb26d",
+ "value": null
+ },
+ {
+ "color": "#ef843c",
+ "value": 10
+ },
+ {
+ "color": "#e24d42",
+ "value": 20
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 6,
+ "y": 22
+ },
+ "hideTimeOverride": true,
+ "id": 56,
+ "links": [],
+ "options": {
+ "minVizHeight": 75,
+ "minVizWidth": 75,
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "last"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "showThresholdLabels": false,
+ "showThresholdMarkers": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_mempool_size{chain_id=\"$chain_id\", instance=~\"$instance\"}",
+ "format": "time_series",
+ "instant": true,
+ "interval": "30s",
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "Unconfirmed Transactions",
+ "type": "gauge"
+ },
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 0,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 12,
+ "y": 22
+ },
+ "hideTimeOverride": true,
+ "id": 60,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_mempool_failed_txs{chain_id=\"$chain_id\", instance=~\"$instance\"}",
+ "format": "time_series",
+ "instant": true,
+ "interval": "30s",
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "Failed Transactions",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "decimals": 0,
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "locale"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 6,
+ "x": 18,
+ "y": 22
+ },
+ "hideTimeOverride": true,
+ "id": 61,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_mempool_recheck_times{chain_id=\"$chain_id\", instance=~\"$instance\"}",
+ "format": "time_series",
+ "instant": true,
+ "interval": "30s",
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "Recheck Times",
+ "type": "stat"
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$DS"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 0,
+ "y": 26
+ },
+ "hiddenSeries": false,
+ "id": 59,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "10.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_p2p_peer_receive_bytes_total{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{peer_id}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Total Network Input",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "series",
+ "show": false,
+ "values": [
+ "current"
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "aliasColors": {},
+ "bars": true,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "uid": "$DS"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "links": []
+ },
+ "overrides": []
+ },
+ "fill": 1,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 9,
+ "w": 12,
+ "x": 12,
+ "y": 26
+ },
+ "hiddenSeries": false,
+ "id": 58,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "rightSide": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": false,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "10.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "uid": "$DS"
+ },
+ "expr": "tendermint_p2p_peer_send_bytes_total{chain_id=\"$chain_id\", instance=\"$instance\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{peer_id}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Total Network Output",
+ "tooltip": {
+ "shared": false,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "series",
+ "show": false,
+ "values": [
+ "current"
+ ]
+ },
+ "yaxes": [
+ {
+ "format": "bytes",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": false
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 38,
+ "tags": [
+ "Blockchain",
+ "Cosmos"
+ ],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "Prometheus",
+ "value": "PBFA97CFB590B2093"
+ },
+ "hide": 0,
+ "includeAll": false,
+ "label": "Datasource",
+ "multi": false,
+ "name": "DS",
+ "options": [],
+ "query": "prometheus",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "type": "datasource"
+ },
+ {
+ "current": {
+ "selected": false,
+ "text": "athens_101-1",
+ "value": "athens_101-1"
+ },
+ "datasource": {
+ "uid": "$DS"
+ },
+ "definition": "label_values(tendermint_consensus_height, chain_id)",
+ "hide": 0,
+ "includeAll": false,
+ "label": "Chain ID",
+ "multi": false,
+ "name": "chain_id",
+ "options": [],
+ "query": "label_values(tendermint_consensus_height, chain_id)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 0,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ },
+ {
+ "allValue": "",
+ "current": {
+ "selected": false,
+ "text": "172.20.0.11:26660",
+ "value": "172.20.0.11:26660"
+ },
+ "datasource": {
+ "uid": "$DS"
+ },
+ "definition": "label_values(tendermint_consensus_height{chain_id=\"$chain_id\"}, instance)",
+ "hide": 0,
+ "includeAll": false,
+ "label": "Instance",
+ "multi": false,
+ "name": "instance",
+ "options": [],
+ "query": "label_values(tendermint_consensus_height{chain_id=\"$chain_id\"}, instance)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 5,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now/d",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Cosmos Dashboard",
+ "uid": "UJyurCTWz",
+ "version": 1,
+ "weekStart": ""
+}
\ No newline at end of file
diff --git a/contrib/localnet/grafana/dashboards/default.yaml b/contrib/localnet/grafana/dashboards/default.yaml
new file mode 100644
index 0000000000..581a362406
--- /dev/null
+++ b/contrib/localnet/grafana/dashboards/default.yaml
@@ -0,0 +1,8 @@
+apiVersion: 1
+
+providers:
+ - name: Default # A uniquely identifiable name for the provider
+ folder: Dashboards # The folder where to place the dashboards
+ type: file
+ options:
+ path: /etc/grafana/provisioning/dashboards # Path to dashboard files on disk (required)
\ No newline at end of file
diff --git a/contrib/localnet/grafana/dashboards/eth_balance.json b/contrib/localnet/grafana/dashboards/eth_balance.json
new file mode 100644
index 0000000000..7cafe4c4d9
--- /dev/null
+++ b/contrib/localnet/grafana/dashboards/eth_balance.json
@@ -0,0 +1,955 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Graph Ethereum Wallet Balances",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "gnetId": 6970,
+ "graphTooltip": 0,
+ "id": 4,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "none"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 0,
+ "y": 0
+ },
+ "id": 3,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "eth_total_addresses",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "Total Addresses Logged",
+ "type": "stat"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fill": 2,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 16,
+ "x": 4,
+ "y": 0
+ },
+ "hiddenSeries": false,
+ "id": 6,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": true,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "10.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "topk(5, eth_balance)",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "{{name}}",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "Top 5 Balances",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#299c46",
+ "value": null
+ },
+ {
+ "color": "#fce2de",
+ "value": 0
+ },
+ {
+ "color": "#d44a3a",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 20,
+ "y": 0
+ },
+ "id": 11,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "(eth_balance_total - eth_balance_total offset 1m) / eth_balance_total",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "Total ETH 1 Hour Change",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "dtdurations"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 0,
+ "y": 4
+ },
+ "id": 2,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "eth_load_seconds",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "Total Load Time",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#299c46",
+ "value": null
+ },
+ {
+ "color": "#fce2de",
+ "value": 0
+ },
+ {
+ "color": "#d44a3a",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 4,
+ "x": 20,
+ "y": 4
+ },
+ "id": 12,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "(eth_balance_total - eth_balance_total offset 24h) / eth_balance_total",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "Total ETH 24 Hour Change",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "gridPos": {
+ "h": 3,
+ "w": 24,
+ "x": 0,
+ "y": 8
+ },
+ "id": 14,
+ "links": [],
+ "options": {
+ "code": {
+ "language": "plaintext",
+ "showLineNumbers": false,
+ "showMiniMap": false
+ },
+ "content": "
${name}",
+ "mode": "html"
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "datasource",
+ "uid": "grafana"
+ },
+ "refId": "A"
+ }
+ ],
+ "type": "text"
+ },
+ {
+ "aliasColors": {},
+ "bars": false,
+ "dashLength": 10,
+ "dashes": false,
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fill": 7,
+ "fillGradient": 0,
+ "gridPos": {
+ "h": 8,
+ "w": 14,
+ "x": 0,
+ "y": 11
+ },
+ "hiddenSeries": false,
+ "id": 5,
+ "legend": {
+ "avg": false,
+ "current": false,
+ "max": false,
+ "min": false,
+ "show": false,
+ "total": false,
+ "values": false
+ },
+ "lines": true,
+ "linewidth": 1,
+ "links": [],
+ "nullPointMode": "null",
+ "options": {
+ "alertThreshold": true
+ },
+ "percentage": false,
+ "pluginVersion": "10.2.2",
+ "pointradius": 5,
+ "points": false,
+ "renderer": "flot",
+ "seriesOverrides": [],
+ "spaceLength": 10,
+ "stack": false,
+ "steppedLine": false,
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "eth_balance{name=\"${name}\"}",
+ "format": "time_series",
+ "intervalFactor": 1,
+ "legendFormat": "${name} ETH",
+ "refId": "A"
+ }
+ ],
+ "thresholds": [],
+ "timeRegions": [],
+ "title": "${name} Balance",
+ "tooltip": {
+ "shared": true,
+ "sort": 0,
+ "value_type": "individual"
+ },
+ "type": "graph",
+ "xaxis": {
+ "mode": "time",
+ "show": true,
+ "values": []
+ },
+ "yaxes": [
+ {
+ "format": "locale",
+ "label": "ETH",
+ "logBase": 1,
+ "show": true
+ },
+ {
+ "format": "short",
+ "logBase": 1,
+ "show": true
+ }
+ ],
+ "yaxis": {
+ "align": false
+ }
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ },
+ "unit": "locale"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 5,
+ "x": 14,
+ "y": 11
+ },
+ "id": 7,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "none",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "eth_balance{name=\"${name}\"}",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "${name} ETH",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#299c46",
+ "value": null
+ },
+ {
+ "color": "#fce2de",
+ "value": 0
+ },
+ {
+ "color": "#d44a3a",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 5,
+ "x": 19,
+ "y": 11
+ },
+ "id": 8,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "(eth_balance{name=\"${name}\"} - eth_balance{name=\"${name}\"} offset 1h) / eth_balance{name=\"${name}\"}",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "${name} ETH 1 Hour Change",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#299c46",
+ "value": null
+ },
+ {
+ "color": "#fce2de",
+ "value": 0
+ },
+ {
+ "color": "#d44a3a",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 5,
+ "x": 14,
+ "y": 15
+ },
+ "id": 10,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "(eth_balance{name=\"${name}\"} - eth_balance{name=\"${name}\"} offset 24h) / eth_balance{name=\"${name}\"}",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "${name} ETH 24 Hour Change",
+ "type": "stat"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "thresholds"
+ },
+ "mappings": [
+ {
+ "options": {
+ "match": "null",
+ "result": {
+ "text": "N/A"
+ }
+ },
+ "type": "special"
+ }
+ ],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "#299c46",
+ "value": null
+ },
+ {
+ "color": "#fce2de",
+ "value": 0
+ },
+ {
+ "color": "#d44a3a",
+ "value": 100
+ }
+ ]
+ },
+ "unit": "percent"
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 4,
+ "w": 5,
+ "x": 19,
+ "y": 15
+ },
+ "id": 9,
+ "links": [],
+ "maxDataPoints": 100,
+ "options": {
+ "colorMode": "value",
+ "graphMode": "none",
+ "justifyMode": "auto",
+ "orientation": "horizontal",
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ },
+ "textMode": "auto",
+ "wideLayout": true
+ },
+ "pluginVersion": "10.2.2",
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "expr": "(eth_balance{name=\"${name}\"} - eth_balance{name=\"${name}\"} offset 7d) / eth_balance{name=\"${name}\"}",
+ "format": "time_series",
+ "instant": false,
+ "intervalFactor": 1,
+ "refId": "A"
+ }
+ ],
+ "title": "${name} ETH Week Change",
+ "type": "stat"
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 38,
+ "tags": [],
+ "templating": {
+ "list": [
+ {
+ "current": {
+ "selected": false,
+ "text": "ethTSS",
+ "value": "ethTSS"
+ },
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "definition": "",
+ "hide": 0,
+ "includeAll": false,
+ "label": "name",
+ "multi": false,
+ "name": "name",
+ "options": [],
+ "query": "label_values(eth_balance, name)",
+ "refresh": 1,
+ "regex": "",
+ "skipUrlSync": false,
+ "sort": 1,
+ "tagValuesQuery": "",
+ "tagsQuery": "",
+ "type": "query",
+ "useTags": false
+ }
+ ]
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {
+ "refresh_intervals": [
+ "5s",
+ "10s",
+ "30s",
+ "1m",
+ "5m",
+ "15m",
+ "30m",
+ "1h",
+ "2h",
+ "1d"
+ ],
+ "time_options": [
+ "5m",
+ "15m",
+ "1h",
+ "6h",
+ "12h",
+ "24h",
+ "2d",
+ "7d",
+ "30d"
+ ]
+ },
+ "timezone": "",
+ "title": "Ethereum Wallet Monitor",
+ "uid": "pgGHUOdmv",
+ "version": 1,
+ "weekStart": ""
+}
\ No newline at end of file
diff --git a/contrib/localnet/grafana/dashboards/zetaclient_dashboard.json b/contrib/localnet/grafana/dashboards/zetaclient_dashboard.json
new file mode 100644
index 0000000000..68194ee1f9
--- /dev/null
+++ b/contrib/localnet/grafana/dashboards/zetaclient_dashboard.json
@@ -0,0 +1,551 @@
+{
+ "annotations": {
+ "list": [
+ {
+ "builtIn": 1,
+ "datasource": {
+ "type": "grafana",
+ "uid": "-- Grafana --"
+ },
+ "enable": true,
+ "hide": true,
+ "iconColor": "rgba(0, 211, 255, 1)",
+ "name": "Annotations & Alerts",
+ "type": "dashboard"
+ }
+ ]
+ },
+ "description": "Dashboard for Zetaclient Metrics",
+ "editable": true,
+ "fiscalYearStartMonth": 0,
+ "graphTooltip": 0,
+ "id": 6,
+ "links": [],
+ "liveNow": false,
+ "panels": [
+ {
+ "collapsed": false,
+ "gridPos": {
+ "h": 1,
+ "w": 24,
+ "x": 0,
+ "y": 0
+ },
+ "id": 6,
+ "panels": [],
+ "title": "Zetaclient Dashboard",
+ "type": "row"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 1
+ },
+ "id": 4,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "zetaclient_pending_txs_goerli_localnet",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Pending Transaction Goerli Localnet",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 1
+ },
+ "id": 2,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "zetaclient_hotkey_burn_rate",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Hotkey Burn Rate",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 9
+ },
+ "id": 3,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "zetaclient_pending_txs_btc_regtest",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Pending Transactions BTC",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 9
+ },
+ "id": 1,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "zetaclient_Outbound_tx_sign_count{instance=\"172.20.0.21:8886\"}",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "Outbound Transaction Sign Count",
+ "type": "timeseries"
+ },
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "fieldConfig": {
+ "defaults": {
+ "color": {
+ "mode": "palette-classic"
+ },
+ "custom": {
+ "axisBorderShow": false,
+ "axisCenteredZero": false,
+ "axisColorMode": "text",
+ "axisLabel": "",
+ "axisPlacement": "auto",
+ "barAlignment": 0,
+ "drawStyle": "line",
+ "fillOpacity": 0,
+ "gradientMode": "none",
+ "hideFrom": {
+ "legend": false,
+ "tooltip": false,
+ "viz": false
+ },
+ "insertNulls": false,
+ "lineInterpolation": "linear",
+ "lineWidth": 1,
+ "pointSize": 5,
+ "scaleDistribution": {
+ "type": "linear"
+ },
+ "showPoints": "auto",
+ "spanNulls": false,
+ "stacking": {
+ "group": "A",
+ "mode": "none"
+ },
+ "thresholdsStyle": {
+ "mode": "off"
+ }
+ },
+ "mappings": [],
+ "thresholds": {
+ "mode": "absolute",
+ "steps": [
+ {
+ "color": "green",
+ "value": null
+ },
+ {
+ "color": "red",
+ "value": 80
+ }
+ ]
+ }
+ },
+ "overrides": []
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 17
+ },
+ "id": 5,
+ "options": {
+ "legend": {
+ "calcs": [],
+ "displayMode": "list",
+ "placement": "bottom",
+ "showLegend": true
+ },
+ "tooltip": {
+ "mode": "single",
+ "sort": "none"
+ }
+ },
+ "targets": [
+ {
+ "datasource": {
+ "type": "prometheus",
+ "uid": "PBFA97CFB590B2093"
+ },
+ "disableTextWrap": false,
+ "editorMode": "builder",
+ "expr": "zetaclient_rpc_getBlockByNumber_count_goerli_localnet",
+ "fullMetaSearch": false,
+ "includeNullMetadata": true,
+ "instant": false,
+ "legendFormat": "__auto",
+ "range": true,
+ "refId": "A",
+ "useBackend": false
+ }
+ ],
+ "title": "BlockByNo Count Goerli/Localnet",
+ "type": "timeseries"
+ }
+ ],
+ "refresh": "",
+ "schemaVersion": 38,
+ "tags": [],
+ "templating": {
+ "list": []
+ },
+ "time": {
+ "from": "now-6h",
+ "to": "now"
+ },
+ "timepicker": {},
+ "timezone": "",
+ "title": "Zetaclient Dashboard",
+ "uid": "efac1ac6-2adb-4123-8833-a8634c5fa64d",
+ "version": 9,
+ "weekStart": ""
+}
\ No newline at end of file
diff --git a/contrib/localnet/grafana/datasource.yaml b/contrib/localnet/grafana/datasource.yaml
new file mode 100644
index 0000000000..59cd333a94
--- /dev/null
+++ b/contrib/localnet/grafana/datasource.yaml
@@ -0,0 +1,16 @@
+apiVersion: 1
+
+datasources:
+ - name: Prometheus
+ type: prometheus
+ access: proxy
+ # Access mode - proxy (server in the UI) or direct (browser in the UI).
+ url: http://prometheus:9090
+ jsonData:
+ httpMethod: POST
+ manageAlerts: true
+ prometheusType: Prometheus
+ prometheusVersion: 2.48.1
+ cacheLevel: 'High'
+ disableRecordingRules: false
+ incrementalQueryOverlapWindow: 10m
\ No newline at end of file
diff --git a/contrib/localnet/grafana/get-tss-address.sh b/contrib/localnet/grafana/get-tss-address.sh
new file mode 100755
index 0000000000..eae4b031af
--- /dev/null
+++ b/contrib/localnet/grafana/get-tss-address.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+/usr/sbin/sshd
+
+#retrieve value of ETH TSS Address from localnet
+ethTSS_address=$(curl 'http://localhost:1317/zeta-chain/observer/get_tss_address' | jq -r '.eth')
+
+#write value of ETH TSS Address to addresses.txt file
+printf "ethTSS:$ethTSS_address\n" > addresses.txt
\ No newline at end of file
diff --git a/contrib/localnet/prometheus/prometheus.yml b/contrib/localnet/prometheus/prometheus.yml
new file mode 100644
index 0000000000..3c73f90385
--- /dev/null
+++ b/contrib/localnet/prometheus/prometheus.yml
@@ -0,0 +1,39 @@
+# my global config
+global:
+ scrape_interval: 15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+ evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+ # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+ alertmanagers:
+ - static_configs:
+ - targets:
+ # - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+# - "first_rules.yml"
+# - "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+ # The job name is added as a label `job=` to any timeseries scraped from this config.
+ - job_name: "zetacore"
+ # metrics_path defaults to '/metrics'
+ # scheme defaults to 'http'.
+ static_configs:
+ - targets: ["172.20.0.11:26660"]
+
+ - job_name: "zetaclient"
+ # metrics_path defaults to '/metrics'
+ # scheme defaults to 'http'.
+ static_configs:
+ - targets: ["172.20.0.21:8886"]
+
+ - job_name: "ethbalance"
+ # metrics_path defaults to '/metrics'
+ # scheme defaults to 'http'.
+ static_configs:
+ - targets: ["172.20.0.32:9015"]
\ No newline at end of file
diff --git a/contrib/localnet/zetacored/common/app.toml b/contrib/localnet/zetacored/common/app.toml
index 624e89d40c..7bbf48cf96 100644
--- a/contrib/localnet/zetacored/common/app.toml
+++ b/contrib/localnet/zetacored/common/app.toml
@@ -80,7 +80,7 @@ service-name = ""
# Enabled enables the application telemetry functionality. When enabled,
# an in-memory sink is also enabled by default. Operators may also enabled
# other sinks such as Prometheus.
-enabled = false
+enabled = true
# Enable prefixing gauge values with hostname.
enable-hostname = false
diff --git a/contrib/localnet/zetacored/common/config.toml b/contrib/localnet/zetacored/common/config.toml
index 29b433e824..c0f955ea4b 100644
--- a/contrib/localnet/zetacored/common/config.toml
+++ b/contrib/localnet/zetacored/common/config.toml
@@ -451,7 +451,7 @@ psql-conn = ""
# When true, Prometheus metrics are served under /metrics on
# PrometheusListenAddr.
# Check out the documentation for the list of available metrics.
-prometheus = false
+prometheus = true
# Address to listen for Prometheus collector(s) connections
prometheus_listen_addr = ":26660"
diff --git a/go.mod b/go.mod
index a8dd44e772..b583a97b16 100644
--- a/go.mod
+++ b/go.mod
@@ -52,7 +52,7 @@ require (
require (
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
- github.com/mattn/go-sqlite3 v1.14.16 // indirect
+ github.com/mattn/go-sqlite3 v1.14.19 // indirect
gorm.io/driver/sqlite v1.4.4
gorm.io/gorm v1.24.6
)
diff --git a/go.sum b/go.sum
index 238beaf820..da78e7283c 100644
--- a/go.sum
+++ b/go.sum
@@ -2154,8 +2154,8 @@ github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOq
github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
-github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
-github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
+github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE=
github.com/mattn/go-zglob v0.0.1/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo=
github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
diff --git a/scripts/protoc-gen-typescript.sh b/scripts/protoc-gen-typescript.sh
index 526d4773da..e56c951de1 100755
--- a/scripts/protoc-gen-typescript.sh
+++ b/scripts/protoc-gen-typescript.sh
@@ -8,7 +8,7 @@ rm -rf $DIR
cat < $DIR/package.json
{
- "name": "@zetachain/blockchain-types",
+ "name": "@zetachain/node-types",
"version": "0.0.0-set-on-publish",
"description": "",
"main": "",
diff --git a/typescript/package.json b/typescript/package.json
index 369d5cbb82..31e60016e1 100644
--- a/typescript/package.json
+++ b/typescript/package.json
@@ -1,5 +1,5 @@
{
- "name": "@zetachain/blockchain-types",
+ "name": "@zetachain/node-types",
"version": "0.0.0-set-on-publish",
"description": "",
"main": "",
diff --git a/zetaclient/bitcoin_client.go b/zetaclient/bitcoin_client.go
index 4e0651b45e..2b9f242b56 100644
--- a/zetaclient/bitcoin_client.go
+++ b/zetaclient/bitcoin_client.go
@@ -57,9 +57,9 @@ type BitcoinChainClient struct {
Mu *sync.Mutex // lock for all the maps, utxos and core params
pendingNonce uint64
- includedTxHashes map[string]uint64 // key: tx hash
- includedTxResults map[string]btcjson.GetTransactionResult // key: chain-tss-nonce
- broadcastedTx map[string]string // key: chain-tss-nonce, value: outTx hash
+ includedTxHashes map[string]uint64 // key: tx hash
+ includedTxResults map[string]*btcjson.GetTransactionResult // key: chain-tss-nonce
+ broadcastedTx map[string]string // key: chain-tss-nonce, value: outTx hash
utxos []btcjson.ListUnspentResult
params observertypes.ChainParams
@@ -148,7 +148,7 @@ func NewBitcoinClient(
ob.zetaClient = bridge
ob.Tss = tss
ob.includedTxHashes = make(map[string]uint64)
- ob.includedTxResults = make(map[string]btcjson.GetTransactionResult)
+ ob.includedTxResults = make(map[string]*btcjson.GetTransactionResult)
ob.broadcastedTx = make(map[string]string)
ob.params = btcCfg.ChainParams
@@ -442,24 +442,23 @@ func (ob *BitcoinChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64
}
// Try including this outTx broadcasted by myself
- inMempool, err := ob.checkNSaveIncludedTx(txnHash, params)
- if err != nil {
- ob.logger.ObserveOutTx.Error().Err(err).Msg("IsSendOutTxProcessed: checkNSaveIncludedTx failed")
+ txResult, inMempool := ob.checkIncludedTx(txnHash, params)
+ if txResult == nil {
+ ob.logger.ObserveOutTx.Error().Err(err).Msg("IsSendOutTxProcessed: checkIncludedTx failed")
return false, false, err
- }
- if inMempool { // to avoid unnecessary Tss keysign
+ } else if inMempool { // still in mempool (should avoid unnecessary Tss keysign)
ob.logger.ObserveOutTx.Info().Msgf("IsSendOutTxProcessed: outTx %s is still in mempool", outTxID)
return true, false, nil
+ } else { // included
+ ob.setIncludedTx(nonce, txResult)
}
// Get tx result again in case it is just included
- ob.Mu.Lock()
- res, included = ob.includedTxResults[outTxID]
- ob.Mu.Unlock()
- if !included {
+ res = ob.getIncludedTx(nonce)
+ if res == nil {
return false, false, nil
}
- ob.logger.ObserveOutTx.Info().Msgf("IsSendOutTxProcessed: checkNSaveIncludedTx succeeded for outTx %s", outTxID)
+ ob.logger.ObserveOutTx.Info().Msgf("IsSendOutTxProcessed: setIncludedTx succeeded for outTx %s", outTxID)
}
// It's safe to use cctx's amount to post confirmation because it has already been verified in observeOutTx()
@@ -830,14 +829,11 @@ func (ob *BitcoinChainClient) refreshPendingNonce() {
}
func (ob *BitcoinChainClient) getOutTxidByNonce(nonce uint64, test bool) (string, error) {
- ob.Mu.Lock()
- res, included := ob.includedTxResults[ob.GetTxID(nonce)]
- ob.Mu.Unlock()
// There are 2 types of txids an observer can trust
// 1. The ones had been verified and saved by observer self.
// 2. The ones had been finalized in zetacore based on majority vote.
- if included {
+ if res := ob.getIncludedTx(nonce); res != nil {
return res.TxID, nil
}
if !test { // if not unit test, get cctx from zetacore
@@ -1020,13 +1016,28 @@ func (ob *BitcoinChainClient) observeOutTx() {
if len(tracker.HashList) > 1 {
ob.logger.ObserveOutTx.Warn().Msgf("observeOutTx: oops, outTxID %s got multiple (%d) outTx hashes", outTxID, len(tracker.HashList))
}
- // verify outTx hashes
+ // iterate over all txHashes to find the truly included one.
+ // we do it this (inefficient) way because we don't rely on the first one as it may be a false positive (for unknown reason).
+ txCount := 0
+ var txResult *btcjson.GetTransactionResult
for _, txHash := range tracker.HashList {
- _, err := ob.checkNSaveIncludedTx(txHash.TxHash, params)
- if err != nil {
- ob.logger.ObserveOutTx.Error().Err(err).Msg("observeOutTx: checkNSaveIncludedTx failed")
+ result, inMempool := ob.checkIncludedTx(txHash.TxHash, params)
+ if result != nil && !inMempool { // included
+ txCount++
+ txResult = result
+ ob.logger.ObserveOutTx.Info().Msgf("observeOutTx: included outTx %s for chain %d nonce %d", txHash.TxHash, ob.chain.ChainId, tracker.Nonce)
+ if txCount > 1 {
+ ob.logger.ObserveOutTx.Error().Msgf(
+ "observeOutTx: checkIncludedTx passed, txCount %d chain %d nonce %d result %v", txCount, ob.chain.ChainId, tracker.Nonce, result)
+ }
}
}
+ if txCount == 1 { // should be only one txHash included for each nonce
+ ob.setIncludedTx(tracker.Nonce, txResult)
+ } else if txCount > 1 {
+ ob.removeIncludedTx(tracker.Nonce) // we can't tell which txHash is true, so we remove all (if any) to be safe
+ ob.logger.ObserveOutTx.Error().Msgf("observeOutTx: included multiple (%d) outTx for chain %d nonce %d", txCount, ob.chain.ChainId, tracker.Nonce)
+ }
}
ticker.UpdateInterval(ob.GetChainParams().OutTxTicker, ob.logger.ObserveOutTx)
case <-ob.stop:
@@ -1036,50 +1047,69 @@ func (ob *BitcoinChainClient) observeOutTx() {
}
}
-// checkNSaveIncludedTx either includes a new outTx or update an existing outTx result.
-// Returns inMempool, error
-func (ob *BitcoinChainClient) checkNSaveIncludedTx(txHash string, params types.OutboundTxParams) (bool, error) {
+// checkIncludedTx checks if a txHash is included and returns (txResult, inMempool)
+// Note: if txResult is nil, then inMempool flag should be ignored.
+func (ob *BitcoinChainClient) checkIncludedTx(txHash string, params types.OutboundTxParams) (*btcjson.GetTransactionResult, bool) {
outTxID := ob.GetTxID(params.OutboundTxTssNonce)
hash, getTxResult, err := ob.GetTxResultByHash(txHash)
if err != nil {
- return false, errors.Wrapf(err, "checkNSaveIncludedTx: error GetTxResultByHash: %s", txHash)
+ ob.logger.ObserveOutTx.Error().Err(err).Msgf("checkIncludedTx: error GetTxResultByHash: %s", txHash)
+ return nil, false
+ }
+ if txHash != getTxResult.TxID { // just in case, we'll use getTxResult.TxID later
+ ob.logger.ObserveOutTx.Error().Msgf("checkIncludedTx: inconsistent txHash %s and getTxResult.TxID %s", txHash, getTxResult.TxID)
+ return nil, false
}
if getTxResult.Confirmations >= 0 { // check included tx only
err = ob.checkTssOutTxResult(hash, getTxResult, params, params.OutboundTxTssNonce)
if err != nil {
- return false, errors.Wrapf(err, "checkNSaveIncludedTx: error verify bitcoin outTx %s outTxID %s", txHash, outTxID)
+ ob.logger.ObserveOutTx.Error().Err(err).Msgf("checkIncludedTx: error verify bitcoin outTx %s outTxID %s", txHash, outTxID)
+ return nil, false
}
+ return getTxResult, false // included
+ }
+ return getTxResult, true // in mempool
+}
- ob.Mu.Lock()
- defer ob.Mu.Unlock()
- nonce, foundHash := ob.includedTxHashes[txHash]
- res, foundRes := ob.includedTxResults[outTxID]
-
- // include new outTx and enforce rigid 1-to-1 mapping: outTxID(nonce) <===> txHash
- if !foundHash && !foundRes {
- ob.includedTxHashes[txHash] = params.OutboundTxTssNonce
- ob.includedTxResults[outTxID] = *getTxResult
- if params.OutboundTxTssNonce >= ob.pendingNonce { // try increasing pending nonce on every newly included outTx
- ob.pendingNonce = params.OutboundTxTssNonce + 1
- }
- ob.logger.ObserveOutTx.Info().Msgf("checkNSaveIncludedTx: included new bitcoin outTx %s outTxID %s pending nonce %d", txHash, outTxID, ob.pendingNonce)
- }
- // update saved tx result as confirmations may increase
- if foundHash && foundRes {
- ob.includedTxResults[outTxID] = *getTxResult
- if getTxResult.Confirmations > res.Confirmations {
- ob.logger.ObserveOutTx.Info().Msgf("checkNSaveIncludedTx: bitcoin outTx %s got confirmations %d", txHash, getTxResult.Confirmations)
- }
- }
- if !foundHash && foundRes { // be alert for duplicate payment!!! As we got a new hash paying same cctx. It might happen (e.g. majority of signers get crupted)
- ob.logger.ObserveOutTx.Error().Msgf("checkNSaveIncludedTx: duplicate payment by bitcoin outTx %s outTxID %s, prior result %v, current result %v", txHash, outTxID, res, *getTxResult)
+// setIncludedTx saves included tx result in memory
+func (ob *BitcoinChainClient) setIncludedTx(nonce uint64, getTxResult *btcjson.GetTransactionResult) {
+ txHash := getTxResult.TxID
+ outTxID := ob.GetTxID(nonce)
+
+ ob.Mu.Lock()
+ defer ob.Mu.Unlock()
+ res, found := ob.includedTxResults[outTxID]
+
+ if !found { // not found.
+ ob.includedTxResults[outTxID] = getTxResult // include new outTx and enforce rigid 1-to-1 mapping: nonce <===> txHash
+ if nonce >= ob.pendingNonce { // try increasing pending nonce on every newly included outTx
+ ob.pendingNonce = nonce + 1
}
- if foundHash && !foundRes {
- ob.logger.ObserveOutTx.Error().Msgf("checkNSaveIncludedTx: unreachable code path! outTx %s outTxID %s, prior nonce %d, current nonce %d", txHash, outTxID, nonce, params.OutboundTxTssNonce)
+ ob.logger.ObserveOutTx.Info().Msgf("setIncludedTx: included new bitcoin outTx %s outTxID %s pending nonce %d", txHash, outTxID, ob.pendingNonce)
+ } else if txHash == res.TxID { // found same hash.
+ ob.includedTxResults[outTxID] = getTxResult // update tx result as confirmations may increase
+ if getTxResult.Confirmations > res.Confirmations {
+ ob.logger.ObserveOutTx.Info().Msgf("setIncludedTx: bitcoin outTx %s got confirmations %d", txHash, getTxResult.Confirmations)
}
- return false, nil
+ } else { // found other hash.
+ // be alert for duplicate payment!!! As we got a new hash paying same cctx (for whatever reason).
+ delete(ob.includedTxResults, outTxID) // we can't tell which txHash is true, so we remove all to be safe
+ ob.logger.ObserveOutTx.Error().Msgf("setIncludedTx: duplicate payment by bitcoin outTx %s outTxID %s, prior outTx %s", txHash, outTxID, res.TxID)
}
- return true, nil // in mempool
+}
+
+// getIncludedTx gets the receipt and transaction from memory
+func (ob *BitcoinChainClient) getIncludedTx(nonce uint64) *btcjson.GetTransactionResult {
+ ob.Mu.Lock()
+ defer ob.Mu.Unlock()
+ return ob.includedTxResults[ob.GetTxID(nonce)]
+}
+
+// removeIncludedTx removes included tx's result from memory
+func (ob *BitcoinChainClient) removeIncludedTx(nonce uint64) {
+ ob.Mu.Lock()
+ defer ob.Mu.Unlock()
+ delete(ob.includedTxResults, ob.GetTxID(nonce))
}
// Basic TSS outTX checks:
diff --git a/zetaclient/btc_signer_test.go b/zetaclient/btc_signer_test.go
index 8bee941a98..5c12e67876 100644
--- a/zetaclient/btc_signer_test.go
+++ b/zetaclient/btc_signer_test.go
@@ -398,7 +398,7 @@ func createTestClient(t *testing.T) *BitcoinChainClient {
client := &BitcoinChainClient{
Tss: tss,
Mu: &sync.Mutex{},
- includedTxResults: make(map[string]btcjson.GetTransactionResult),
+ includedTxResults: make(map[string]*btcjson.GetTransactionResult),
}
// Create 10 dummy UTXOs (22.44 BTC in total)
@@ -413,7 +413,7 @@ func createTestClient(t *testing.T) *BitcoinChainClient {
func mineTxNSetNonceMark(ob *BitcoinChainClient, nonce uint64, txid string, preMarkIndex int) {
// Mine transaction
outTxID := ob.GetTxID(nonce)
- ob.includedTxResults[outTxID] = btcjson.GetTransactionResult{TxID: txid}
+ ob.includedTxResults[outTxID] = &btcjson.GetTransactionResult{TxID: txid}
// Set nonce mark
tssAddress := ob.Tss.BTCAddressWitnessPubkeyHash().EncodeAddress()
diff --git a/zetaclient/evm_client.go b/zetaclient/evm_client.go
index 005fa10260..badce4eca6 100644
--- a/zetaclient/evm_client.go
+++ b/zetaclient/evm_client.go
@@ -296,11 +296,11 @@ func (ob *EVMChainClient) Stop() {
// returns: isIncluded, isConfirmed, Error
// If isConfirmed, it also post to ZetaCore
func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, cointype common.CoinType, logger zerolog.Logger) (bool, bool, error) {
- if !ob.isTxConfirmed(nonce) {
- return false, false, nil
- }
params := ob.GetChainParams()
receipt, transaction := ob.GetTxNReceipt(nonce)
+ if receipt == nil || transaction == nil { // not confirmed yet
+ return false, false, nil
+ }
sendID := fmt.Sprintf("%s-%d", ob.chain.String(), nonce)
logger = logger.With().Str("sendID", sendID).Logger()
@@ -543,13 +543,6 @@ func (ob *EVMChainClient) IsSendOutTxProcessed(sendHash string, nonce uint64, co
return false, false, nil
}
-// The lowest nonce we observe outTx for each chain
-var lowestOutTxNonceToObserve = map[int64]uint64{
- 5: 113000, // Goerli
- 97: 102600, // BSC testnet
- 80001: 154500, // Mumbai
-}
-
// FIXME: there's a chance that a txhash in OutTxChan may not deliver when Stop() is called
// observeOutTx periodically checks all the txhash in potential outbound txs
func (ob *EVMChainClient) observeOutTx() {
@@ -558,7 +551,7 @@ func (ob *EVMChainClient) observeOutTx() {
if err != nil || timeoutNonce <= 0 {
timeoutNonce = 100 * 3 // process up to 100 hashes
}
- ob.logger.ObserveOutTx.Info().Msgf("observeOutTx using timeoutNonce %d seconds", timeoutNonce)
+ ob.logger.ObserveOutTx.Info().Msgf("observeOutTx: using timeoutNonce %d seconds", timeoutNonce)
ticker, err := NewDynamicTicker(fmt.Sprintf("EVM_observeOutTx_%d", ob.chain.ChainId), ob.GetChainParams().OutTxTicker)
if err != nil {
@@ -577,28 +570,37 @@ func (ob *EVMChainClient) observeOutTx() {
//FIXME: remove this timeout here to ensure that all trackers are queried
outTimeout := time.After(time.Duration(timeoutNonce) * time.Second)
TRACKERLOOP:
- // Skip old gabbage trackers as we spent too much time on querying them
for _, tracker := range trackers {
nonceInt := tracker.Nonce
- if nonceInt < lowestOutTxNonceToObserve[ob.chain.ChainId] {
- continue
- }
if ob.isTxConfirmed(nonceInt) { // Go to next tracker if this one already has a confirmed tx
continue
}
+ txCount := 0
+ var receipt *ethtypes.Receipt
+ var transaction *ethtypes.Transaction
for _, txHash := range tracker.HashList {
select {
case <-outTimeout:
- ob.logger.ObserveOutTx.Warn().Msgf("observeOutTx timeout on chain %d nonce %d", ob.chain.ChainId, nonceInt)
+ ob.logger.ObserveOutTx.Warn().Msgf("observeOutTx: timeout on chain %d nonce %d", ob.chain.ChainId, nonceInt)
break TRACKERLOOP
default:
- if ob.confirmTxByHash(txHash.TxHash, nonceInt) {
- ob.logger.ObserveOutTx.Info().Msgf("observeOutTx confirmed outTx %s for chain %d nonce %d", txHash.TxHash, ob.chain.ChainId, nonceInt)
- break
+ if recpt, tx, ok := ob.checkConfirmedTx(txHash.TxHash, nonceInt); ok {
+ txCount++
+ receipt = recpt
+ transaction = tx
+ ob.logger.ObserveOutTx.Info().Msgf("observeOutTx: confirmed outTx %s for chain %d nonce %d", txHash.TxHash, ob.chain.ChainId, nonceInt)
+ if txCount > 1 {
+ ob.logger.ObserveOutTx.Error().Msgf(
+ "observeOutTx: checkConfirmedTx passed, txCount %d chain %d nonce %d receipt %v transaction %v", txCount, ob.chain.ChainId, nonceInt, receipt, transaction)
+ }
}
- ob.logger.ObserveOutTx.Debug().Msgf("observeOutTx outTx %s for chain %d nonce %d not confirmed yet", txHash.TxHash, ob.chain.ChainId, nonceInt)
}
}
+ if txCount == 1 { // should be only one txHash confirmed for each nonce.
+ ob.SetTxNReceipt(nonceInt, receipt, transaction)
+ } else if txCount > 1 { // should not happen. We can't tell which txHash is true. It might happen (e.g. glitchy/hacked endpoint)
+ ob.logger.ObserveOutTx.Error().Msgf("observeOutTx: confirmed multiple (%d) outTx for chain %d nonce %d", txCount, ob.chain.ChainId, nonceInt)
+ }
}
ticker.UpdateInterval(ob.GetChainParams().OutTxTicker, ob.logger.ObserveOutTx)
case <-ob.stop:
@@ -611,47 +613,45 @@ func (ob *EVMChainClient) observeOutTx() {
// SetPendingTx sets the pending transaction in memory
func (ob *EVMChainClient) SetPendingTx(nonce uint64, transaction *ethtypes.Transaction) {
ob.Mu.Lock()
+ defer ob.Mu.Unlock()
ob.outTxPendingTransactions[ob.GetTxID(nonce)] = transaction
- ob.Mu.Unlock()
}
// GetPendingTx gets the pending transaction from memory
func (ob *EVMChainClient) GetPendingTx(nonce uint64) *ethtypes.Transaction {
ob.Mu.Lock()
- transaction := ob.outTxPendingTransactions[ob.GetTxID(nonce)]
- ob.Mu.Unlock()
- return transaction
+ defer ob.Mu.Unlock()
+ return ob.outTxPendingTransactions[ob.GetTxID(nonce)]
}
// SetTxNReceipt sets the receipt and transaction in memory
func (ob *EVMChainClient) SetTxNReceipt(nonce uint64, receipt *ethtypes.Receipt, transaction *ethtypes.Transaction) {
ob.Mu.Lock()
+ defer ob.Mu.Unlock()
delete(ob.outTxPendingTransactions, ob.GetTxID(nonce)) // remove pending transaction, if any
ob.outTXConfirmedReceipts[ob.GetTxID(nonce)] = receipt
ob.outTXConfirmedTransactions[ob.GetTxID(nonce)] = transaction
- ob.Mu.Unlock()
}
-// getTxNReceipt gets the receipt and transaction from memory
+// GetTxNReceipt gets the receipt and transaction from memory
func (ob *EVMChainClient) GetTxNReceipt(nonce uint64) (*ethtypes.Receipt, *ethtypes.Transaction) {
ob.Mu.Lock()
+ defer ob.Mu.Unlock()
receipt := ob.outTXConfirmedReceipts[ob.GetTxID(nonce)]
transaction := ob.outTXConfirmedTransactions[ob.GetTxID(nonce)]
- ob.Mu.Unlock()
return receipt, transaction
}
// isTxConfirmed returns true if there is a confirmed tx for 'nonce'
func (ob *EVMChainClient) isTxConfirmed(nonce uint64) bool {
ob.Mu.Lock()
- confirmed := ob.outTXConfirmedReceipts[ob.GetTxID(nonce)] != nil && ob.outTXConfirmedTransactions[ob.GetTxID(nonce)] != nil
- ob.Mu.Unlock()
- return confirmed
+ defer ob.Mu.Unlock()
+ return ob.outTXConfirmedReceipts[ob.GetTxID(nonce)] != nil && ob.outTXConfirmedTransactions[ob.GetTxID(nonce)] != nil
}
-// confirmTxByHash checks if a txHash is confirmed and saves transaction and receipt in memory
-// returns true if confirmed or false otherwise
-func (ob *EVMChainClient) confirmTxByHash(txHash string, nonce uint64) bool {
+// checkConfirmedTx checks if a txHash is confirmed
+// returns (receipt, transaction, true) if confirmed or (nil, nil, false) otherwise
+func (ob *EVMChainClient) checkConfirmedTx(txHash string, nonce uint64) (*ethtypes.Receipt, *ethtypes.Transaction, bool) {
ctxt, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
@@ -659,15 +659,34 @@ func (ob *EVMChainClient) confirmTxByHash(txHash string, nonce uint64) bool {
transaction, isPending, err := ob.evmClient.TransactionByHash(ctxt, ethcommon.HexToHash(txHash))
if err != nil {
log.Error().Err(err).Msgf("confirmTxByHash: TransactionByHash error, txHash %s nonce %d", txHash, nonce)
- return false
+ return nil, nil, false
}
if transaction == nil { // should not happen
log.Error().Msgf("confirmTxByHash: transaction is nil for txHash %s nonce %d", txHash, nonce)
- return false
+ return nil, nil, false
+ }
+
+ // check tx sender and nonce
+ signer := ethtypes.NewLondonSigner(big.NewInt(ob.chain.ChainId))
+ from, err := signer.Sender(transaction)
+ if err != nil {
+ log.Error().Err(err).Msgf("confirmTxByHash: local recovery of sender address failed for txHash %s chain %d", transaction.Hash().Hex(), ob.chain.ChainId)
+ return nil, nil, false
+ }
+ if from != ob.Tss.EVMAddress() { // must be TSS address
+ log.Error().Msgf("confirmTxByHash: sender %s for txHash %s chain %d is not TSS address %s",
+ from.Hex(), transaction.Hash().Hex(), ob.chain.ChainId, ob.Tss.EVMAddress().Hex())
+ return nil, nil, false
}
- if isPending { // save pending transaction
+ if transaction.Nonce() != nonce { // must match cctx nonce
+ log.Error().Msgf("confirmTxByHash: txHash %s nonce mismatch: wanted %d, got tx nonce %d", txHash, nonce, transaction.Nonce())
+ return nil, nil, false
+ }
+
+ // save pending transaction
+ if isPending {
ob.SetPendingTx(nonce, transaction)
- return false
+ return nil, nil, false
}
// query receipt
@@ -676,33 +695,26 @@ func (ob *EVMChainClient) confirmTxByHash(txHash string, nonce uint64) bool {
if err != ethereum.NotFound {
log.Warn().Err(err).Msgf("confirmTxByHash: TransactionReceipt error, txHash %s nonce %d", txHash, nonce)
}
- return false
+ return nil, nil, false
}
if receipt == nil { // should not happen
log.Error().Msgf("confirmTxByHash: receipt is nil for txHash %s nonce %d", txHash, nonce)
- return false
+ return nil, nil, false
}
- // check nonce and confirmations
- if transaction.Nonce() != nonce {
- log.Error().Msgf("confirmTxByHash: txHash %s nonce mismatch: wanted %d, got tx nonce %d", txHash, nonce, transaction.Nonce())
- return false
- }
+ // check confirmations
confHeight := receipt.BlockNumber.Uint64() + ob.GetChainParams().ConfirmationCount
if confHeight >= math.MaxInt64 {
log.Error().Msgf("confirmTxByHash: confHeight is too large for txHash %s nonce %d", txHash, nonce)
- return false
+ return nil, nil, false
}
if confHeight > ob.GetLastBlockHeight() {
log.Info().Msgf("confirmTxByHash: txHash %s nonce %d included but not confirmed: receipt block %d, current block %d",
txHash, nonce, receipt.BlockNumber, ob.GetLastBlockHeight())
- return false
+ return nil, nil, false
}
- // confirmed, save receipt and transaction
- ob.SetTxNReceipt(nonce, receipt, transaction)
-
- return true
+ return receipt, transaction, true
}
// SetLastBlockHeightScanned set last block height scanned (not necessarily caught up with external block; could be slow/paused)