test: Stateful upgrade

Dockerfile-versioned

@@ -0,0 +1,70 @@
+FROM golang:1.19-alpine
+
+ENV GOPATH /go
+ENV GOOS=linux
+ENV CGO_ENABLED=1
+
+ARG old_version
+ARG new_version
+
+RUN apk --no-cache add git make build-base jq openssh libusb-dev linux-headers bash curl tmux
+RUN ssh-keygen -b 2048 -t rsa -f /root/.ssh/localtest.pem -q -N ""
+
+WORKDIR /go/delivery/zeta-node
+
+RUN mkdir -p  $GOPATH/bin/old
+RUN mkdir -p  $GOPATH/bin/new
+
+ENV NEW_VERSION=${new_version}
+
+# Checkout and build new binary
+RUN git clone https://github.com/zeta-chain/node.git
+RUN cd node && git fetch
+RUN cd node && git checkout ${new_version}
+RUN cd node && make install
+RUN cd node && make install-smoketest
+RUN cp $GOPATH/bin/zetacored $GOPATH/bin/new/
+RUN cp $GOPATH/bin/zetaclientd $GOPATH/bin/new/
+RUN cp $GOPATH/bin/smoketest $GOPATH/bin/new/
+
+# Checkout and build old binary
+RUN cd node && git checkout ${old_version}
+RUN cd node && git pull
+RUN cd node && make install
+RUN cd node && make install-smoketest
+RUN cp $GOPATH/bin/zetacored $GOPATH/bin/old/
+RUN cp $GOPATH/bin/zetaclientd $GOPATH/bin/old/
+RUN cp $GOPATH/bin/smoketest $GOPATH/bin/old/
+
+RUN git clone https://github.com/zeta-chain/cosmos-sdk.git
+RUN cd cosmos-sdk && git checkout zetavisor-v0.1.5
+RUN cd cosmos-sdk/cosmovisor && make zetavisor
+#
+#FROM golang:1.19-alpine
+
+#RUN apk --no-cache add openssh jq tmux vim curl bash
+RUN ssh-keygen -A
+WORKDIR /root
+
+RUN cp /root/.ssh/localtest.pem.pub /root/.ssh/authorized_keys
+
+RUN cp /go/bin/zetaclientd /usr/local/bin
+RUN cp /go/bin/zetacored /usr/local/bin
+RUN cp /go/bin/smoketest /usr/local/bin
+RUN cp /go/bin/zetavisor /usr/local/bin
+
+COPY contrib/localnet/scripts /root
+COPY contrib/localnet/preparams /root/preparams
+COPY contrib/localnet/ssh_config /root/.ssh/config
+COPY contrib/localnet/zetacored /root/zetacored
+COPY contrib/localnet/tss /root/tss
+
+RUN chmod 755 /root/*.sh
+RUN chmod 700 /root/.ssh
+RUN chmod 600 /root/.ssh/*
+
+WORKDIR /usr/local/bin
+ENV SHELL /bin/sh
+EXPOSE 22
+
+ENTRYPOINT ["/usr/sbin/sshd", "-D"]

Makefile

@@ -217,4 +217,13 @@ stress-test: zetanode
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-stresstest.yml up -d
 
 stop-stress-test:
-	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-stresstest.yml down --remove-orphans
+	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-stresstest.yml down --remove-orphans
+
+stateful-upgrade:
+	@echo "--> Starting stateful smoketest"
+	$(DOCKER) build --build-arg old_version=v9.0.0-rc2 --build-arg new_version=v10.0.0 -t zetanode -f ./Dockerfile-versioned .
+	$(DOCKER) build -t orchestrator -f contrib/localnet/orchestrator/Dockerfile-upgrade.fastbuild .
+	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-stateful.yml up -d
+
+stop-stateful-upgrade:
+	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-stateful.yml down --remove-orphans

contrib/localnet/docker-compose-stateful.yml

@@ -0,0 +1,109 @@
+version: "3"
+
+networks:
+  mynetwork:
+    ipam:
+      config:
+        - subnet: 172.20.0.0/24
+
+
+services:
+  zetacore0:
+    image: zetanode:latest
+    container_name: zetacore0
+    build:
+      context: ../../.
+      dockerfile: Dockerfile
+    hostname: zetacore0
+    ports:
+      - "1317:1317"
+      - "9545:8545"
+      - "9546:8546"
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.11
+    entrypoint: ["/root/genesis-stateful.sh", "2"] # Need to specify that there are 2 nodes
+
+
+  zetacore1:
+    image: zetanode:latest
+    container_name: zetacore1
+    build:
+      context: ../../.
+      dockerfile: Dockerfile
+    hostname: zetacore1
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.12
+    entrypoint: ["/root/genesis-stateful.sh", "2"]
+
+  zetaclient0:
+    image: zetanode:latest
+    container_name: zetaclient0
+    build:
+      context: ../../.
+      dockerfile: Dockerfile
+    hostname: zetaclient0
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.21
+    entrypoint: /root/start-zetaclientd-background.sh
+    environment:
+      - ETHDEV_ENDPOINT=http://eth:8545
+
+  zetaclient1:
+    image: zetanode:latest
+    container_name: zetaclient1
+    build:
+      context: ../../.
+      dockerfile: Dockerfile
+    hostname: zetaclient1
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.22
+    entrypoint: /root/start-zetaclientd-background.sh
+    environment:
+      - ETHDEV_ENDPOINT=http://eth:8545
+
+  bitcoin:
+    image: ruimarinho/bitcoin-core:22 # version 23 is not working with btcd 0.22.0 due to change in createwallet rpc
+    container_name: bitcoin
+    hostname: bitcoin
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.101
+    ports:
+      - "18443:18443"
+    command:
+      -printtoconsole
+      -regtest=1
+      -rpcallowip=172.20.0.0/16
+      -rpcbind=0.0.0.0
+      -rpcauth=smoketest:63acf9b8dccecce914d85ff8c044b78b$$5892f9bbc84f4364e79f0970039f88bdd823f168d4acc76099ab97b14a766a99
+      -txindex=1
+
+  eth:
+    image: ethereum/client-go:v1.10.26
+    container_name: eth
+    hostname: eth
+    ports:
+      - "8545:8545"
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.100
+    entrypoint: [ "geth", "--dev", "--http", "--http.addr", "172.20.0.100", "--http.vhosts", "*", "--http.api", "eth,web3,net", "--http.corsdomain", "https://remix.ethereum.org", "--dev.period", "2" ]
+
+  orchestrator:
+    image: orchestrator:latest
+    container_name: orchestrator
+    build:
+      context: ../../.
+      dockerfile: contrib/localnet/orchestrator/Dockerfile
+    depends_on:
+      - zetacore0
+      - eth
+    hostname: orchestrator
+    networks:
+      mynetwork:
+        ipv4_address: 172.20.0.2
+    entrypoint: [ "/work/start-upgrade.sh", "local"]

contrib/localnet/orchestrator/Dockerfile

@@ -2,7 +2,7 @@ FROM zetanode:latest as zeta
 FROM ethereum/client-go:v1.10.26 as geth
 FROM golang:1.20-alpine as orchestrator
 
-RUN apk --no-cache add jq curl bash make git build-base
+RUN apk --no-cache add jq curl bash make git build-base openssh
 
 COPY --from=zeta /usr/local/bin/zetacored /usr/local/bin/
 COPY --from=zeta /usr/local/bin/zetaclientd /usr/local/bin/
@@ -12,7 +12,9 @@ COPY --from=zeta /root/.ssh/localtest.pem.pub /root/.ssh/authorized_keys
 COPY --from=zeta /root/.ssh/localtest.pem /root/.ssh/localtest.pem
 
 COPY contrib/localnet/orchestrator/start.sh /work/
-RUN chmod +x /work/start.sh
+COPY contrib/localnet/orchestrator/start-upgrade.sh /work/
+COPY contrib/localnet/orchestrator/restart-zetaclientd.sh /work/
+RUN chmod +x /work/*.sh
 
 ENV GOPATH /go
 ENV GOOS=linux

contrib/localnet/orchestrator/Dockerfile-upgrade.fastbuild

@@ -0,0 +1,24 @@
+FROM zetanode:latest as zeta
+FROM ethereum/client-go:v1.10.26 as geth
+FROM golang:1.20-alpine as orchestrator
+
+RUN apk --no-cache add jq curl bash make git build-base openssh
+
+COPY --from=zeta /usr/local/bin/zetacored /usr/local/bin/
+COPY --from=zeta /usr/local/bin/zetaclientd /usr/local/bin/
+COPY --from=geth /usr/local/bin/geth /usr/local/bin/
+
+COPY --from=zeta /root/.ssh/localtest.pem.pub /root/.ssh/authorized_keys
+COPY --from=zeta /root/.ssh/localtest.pem /root/.ssh/localtest.pem
+
+COPY contrib/localnet/orchestrator/start.sh /work/
+COPY contrib/localnet/orchestrator/start-upgrade.sh /work/
+COPY contrib/localnet/orchestrator/restart-zetaclientd.sh /work/
+RUN chmod +x /work/*.sh
+
+
+COPY --from=zeta /go/bin/old/smoketest /usr/local/bin/smoketest-old
+COPY --from=zeta /go/bin/new/smoketest /usr/local/bin/smoketest-new
+RUN chmod +x /usr/local/bin/smoketest-*
+
+WORKDIR /work

contrib/localnet/orchestrator/Dockerfile.fastbuild

@@ -18,4 +18,4 @@ RUN chmod +x /work/start.sh
 COPY --from=zeta /usr/local/bin/smoketest /usr/local/bin/
 RUN chmod +x /usr/local/bin/smoketest
 
-WORKDIR /work
+WORKDIR /work

contrib/localnet/orchestrator/restart-zetaclientd.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+clibuilder()
+{
+   echo ""
+   echo "Usage: $0 -u UPGRADE_HEIGHT"
+   echo -e "\t-u Height of upgrade, should match governance proposal"
+   echo -e "\t-n Number of clients in the network"
+   exit 1 # Exit script after printing help
+}
+
+while getopts "u:n:" opt
+do
+   case "$opt" in
+      u ) UPGRADE_HEIGHT="$OPTARG" ;;
+      n ) NUM_OF_NODES="$OPTARG" ;;
+      ? ) clibuilder ;; # Print cliBuilder in case parameter is non-existent
+   esac
+done
+
+# generate client list
+START=0
+END=$((NUM_OF_NODES-1))
+CLIENT_LIST=()
+for i in $(eval echo "{$START..$END}")
+do
+  CLIENT_LIST+=("zetaclient$i")
+done
+
+echo "$UPGRADE_HEIGHT"
+
+CURRENT_HEIGHT=0
+
+while [[ $CURRENT_HEIGHT -lt $UPGRADE_HEIGHT ]]
+do
+    CURRENT_HEIGHT=$(curl zetacore0:26657/status | jq '.result.sync_info.latest_block_height' | tr -d '"')
+    sleep 5
+done
+
+echo current height is "$CURRENT_HEIGHT", restarting zetaclients
+for NODE in "${CLIENT_LIST[@]}"; do
+    ssh -o "StrictHostKeyChecking no" "$NODE" -i ~/.ssh/localtest.pem killall zetaclientd
+    ssh -o "StrictHostKeyChecking no" "$NODE" -i ~/.ssh/localtest.pem "$GOPATH/bin/new/zetaclientd start < /dev/null > $HOME/zetaclient.log 2>&1 &"
+done

contrib/localnet/orchestrator/smoketest/main.go

@@ -67,13 +67,15 @@ var LocalCmd = &cobra.Command{
 
 type localArgs struct {
 	contractsDeployed bool
+	waitForHeight     int64
 }
 
 var localTestArgs = localArgs{}
 
 func init() {
 	RootCmd.AddCommand(LocalCmd)
 	LocalCmd.Flags().BoolVar(&localTestArgs.contractsDeployed, "deployed", false, "set to to true if running smoketest again with existing state")
+	LocalCmd.Flags().Int64Var(&localTestArgs.waitForHeight, "wait-for", 0, "block height for smoketest to begin, ex. --wait-for 100")
 }
 
 func main() {
@@ -94,6 +96,10 @@ func LocalSmokeTest(_ *cobra.Command, _ []string) {
 		os.Exit(1)
 	}()
 
+	if localTestArgs.waitForHeight != 0 {
+		WaitForBlockHeight(localTestArgs.waitForHeight)
+	}
+
 	// set account prefix to zeta
 	cfg := sdk.GetConfig()
 	cfg.SetBech32PrefixForAccount(app.Bech32PrefixAccAddr, app.Bech32PrefixAccPub)

contrib/localnet/orchestrator/smoketest/test_deposit_eth.go

@@ -6,6 +6,8 @@ package main
 import (
 	"context"
 	"fmt"
+	"github.com/zeta-chain/zetacore/common/ethereum"
+	observertypes "github.com/zeta-chain/zetacore/x/observer/types"
 	"math/big"
 	"time"
 
@@ -15,9 +17,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	zrc20 "github.com/zeta-chain/protocol-contracts/pkg/contracts/zevm/zrc20.sol"
 	"github.com/zeta-chain/zetacore/common"
-	"github.com/zeta-chain/zetacore/common/ethereum"
 	"github.com/zeta-chain/zetacore/x/crosschain/types"
-	observertypes "github.com/zeta-chain/zetacore/x/observer/types"
 	"github.com/zeta-chain/zetacore/zetaclient"
 )
 
@@ -93,8 +93,8 @@ func (sm *SmokeTest) TestDepositEtherIntoZRC20() {
 				BlockHash: blockHash.Bytes(),
 			})
 			if err != nil {
-				fmt.Printf("WARN: block header not found; retrying...\n")
-				time.Sleep(2 * time.Second)
+				fmt.Printf("WARN: block header not found; retrying... error: %s \n", err.Error())
+				time.Sleep(5 * time.Second)
 			} else {
 				fmt.Printf("OK: block header found\n")
 				break

contrib/localnet/orchestrator/smoketest/utils.go

@@ -8,6 +8,8 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	rpchttp "github.com/tendermint/tendermint/rpc/client/http"
+	coretypes "github.com/tendermint/tendermint/rpc/core/types"
 	"sync"
 	"time"
 
@@ -113,3 +115,17 @@ func ScriptPKToAddress(scriptPKHex string) string {
 	}
 	return ""
 }
+
+func WaitForBlockHeight(height int64) {
+	// initialize rpc and check status
+	rpc, err := rpchttp.New("http://zetacore0:26657", "/websocket")
+	if err != nil {
+		panic(err)
+	}
+	status := &coretypes.ResultStatus{}
+	for status.SyncInfo.LatestBlockHeight < height {
+		status, _ = rpc.Status(context.Background())
+		time.Sleep(time.Second * 5)
+		fmt.Printf("waiting for block: %d, current height: %d\n", height, status.SyncInfo.LatestBlockHeight)
+	}
+}