From 7370facfeda1713ad45516bc40d663fa1e248b62 Mon Sep 17 00:00:00 2001 From: Alex Gartner Date: Tue, 14 May 2024 08:51:46 -0700 Subject: [PATCH] refactor: unify CI docker images (#2158) * wip * remove static ssh keys * exclude contrib in go build to improve iteration speed * add upgradehost and upgrade orchestrator * fix exclusions? * final changes * fix zetaclient-supervisor rebase * use make target * add full log dump on failure * use variables * fix zetacore1 startup race * more race condition fixing * review feedback --- .dockerignore | 8 +- .github/workflows/build.yml | 22 ++-- .github/workflows/execute_advanced_tests.yaml | 26 +++- Dockerfile-localnet | 67 ++++++++--- Dockerfile-upgrade | 77 ------------ Makefile | 8 +- .../localnet/docker-compose-upgrade-light.yml | 27 +---- contrib/localnet/docker-compose-upgrade.yml | 45 ++++--- contrib/localnet/docker-compose.yml | 30 +++-- contrib/localnet/orchestrator/Dockerfile | 30 ----- .../orchestrator/Dockerfile.fastbuild | 15 +-- .../localnet/orchestrator/start-zetae2e.sh | 6 + .../localnet/scripts/start-upgrade-host.sh | 4 + .../scripts/start-upgrade-orchestrator.sh | 103 ++++++++++++++++ contrib/localnet/scripts/start-zetaclientd.sh | 17 ++- contrib/localnet/scripts/start-zetacored.sh | 113 ++++++------------ 16 files changed, 311 insertions(+), 287 deletions(-) delete mode 100644 Dockerfile-upgrade delete mode 100644 contrib/localnet/orchestrator/Dockerfile create mode 100644 contrib/localnet/scripts/start-upgrade-host.sh create mode 100755 contrib/localnet/scripts/start-upgrade-orchestrator.sh diff --git a/.dockerignore b/.dockerignore index adc6255bfe..d53ebfff61 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,4 +3,10 @@ package.json yarn.lock .github/ .gitignore -dist/** \ No newline at end of file +dist/** + +# dockerfiles are not needed inside the docker build +Dockerfile +Dockerfile-localnet +Dockerfile-upgrade +docker-compose*.yml \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8732a01795..83ef3e964c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -125,23 +125,21 @@ jobs: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_READ_ONLY }} - - name: Build zetanode - run: | - make zetanode + - name: Start Test + run: make start-e2e-test - - name: Start Private Network + # use docker logs -f rather than docker attach to make sure we get the initial logs + - name: Watch Test run: | - cd contrib/localnet/ - docker compose up -d zetacore0 zetacore1 zetaclient0 zetaclient1 eth bitcoin + container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}") + docker logs -f "${container_id}" & + exit $(docker wait "${container_id}") - - name: Run E2E Test + - name: Full Log Dump On Failure + if: failure() run: | cd contrib/localnet - docker-compose up orchestrator --exit-code-from orchestrator - if [ $? -ne 0 ]; then - echo "E2E Test Failed" - exit 1 - fi + docker compose logs - name: Notify Slack on Failure if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop' diff --git a/.github/workflows/execute_advanced_tests.yaml b/.github/workflows/execute_advanced_tests.yaml index e66953ee28..201fcc829f 100644 --- a/.github/workflows/execute_advanced_tests.yaml +++ b/.github/workflows/execute_advanced_tests.yaml @@ -46,6 +46,12 @@ jobs: container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}") docker logs -f "${container_id}" & exit $(docker wait "${container_id}") + + - name: Full Log Dump On Failure + if: failure() + run: | + cd contrib/localnet + docker compose logs - name: Notify Slack on Failure if: failure() && github.event_name == 'schedule' @@ -73,6 +79,12 @@ jobs: docker logs -f "${container_id}" & exit $(docker wait "${container_id}") + - name: Full Log Dump On Failure + if: failure() + run: | + cd contrib/localnet + docker compose logs + - name: Notify Slack on Failure if: failure() && github.event_name == 'schedule' uses: 8398a7/action-slack@v3 @@ -99,6 +111,12 @@ jobs: docker logs -f "${container_id}" & exit $(docker wait "${container_id}") + - name: Full Log Dump On Failure + if: failure() + run: | + cd contrib/localnet + docker compose logs + e2e-performance-test: if: ${{ github.event.inputs.e2e-performance-test == 'true' }} runs-on: buildjet-4vcpu-ubuntu-2204 @@ -114,4 +132,10 @@ jobs: run: | container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}") docker logs -f "${container_id}" & - exit $(docker wait "${container_id}") \ No newline at end of file + exit $(docker wait "${container_id}") + + - name: Full Log Dump On Failure + if: failure() + run: | + cd contrib/localnet + docker compose logs \ No newline at end of file diff --git a/Dockerfile-localnet b/Dockerfile-localnet index ba86debb86..c6e4efdd94 100644 --- a/Dockerfile-localnet +++ b/Dockerfile-localnet @@ -1,31 +1,47 @@ -FROM golang:1.20-alpine3.18 +# syntax=docker/dockerfile:1.7-labs +FROM golang:1.20.14-bullseye AS base-build ENV GOPATH /go ENV GOOS=linux ENV CGO_ENABLED=1 ENV GOCACHE=/root/.cache/go-build -RUN apk --no-cache add git make build-base jq openssh libusb-dev linux-headers bash curl tmux python3 py3-pip -RUN pip install requests -RUN ssh-keygen -b 2048 -t rsa -f /root/.ssh/localtest.pem -q -N "" +RUN apt update && \ + apt install -yq libusb-dev WORKDIR /go/delivery/zeta-node + +FROM base-build AS latest-build + COPY go.mod . COPY go.sum . RUN go mod download -COPY . . +COPY version.sh . +COPY --exclude=*.sh --exclude=*.md --exclude=*.yml . . RUN --mount=type=cache,target="/root/.cache/go-build" make install RUN --mount=type=cache,target="/root/.cache/go-build" make install-zetae2e -RUN ssh-keygen -A -WORKDIR /root +FROM golang:1.20.14-bullseye AS cosmovisor-build +RUN go install cosmossdk.io/tools/cosmovisor/cmd/cosmovisor@v1.5.0 + +FROM debian:bullseye-slim AS base-runtime -RUN cp /root/.ssh/localtest.pem.pub /root/.ssh/authorized_keys +RUN apt update && \ + apt install -yq jq curl tmux python3 openssh-server iputils-ping iproute2 && \ + rm -rf /var/lib/apt/lists/* -RUN cp /go/bin/zetaclientd /usr/local/bin -RUN cp /go/bin/zetacored /usr/local/bin -RUN cp /go/bin/zetae2e /usr/local/bin +RUN ssh-keygen -A && \ + mkdir -p /var/run/sshd + +RUN mkdir -p /root/.zetacored/cosmovisor/genesis/bin && \ + ln -s /usr/local/bin/zetacored /root/.zetacored/cosmovisor/genesis/bin/zetacored && \ + ln -s /root/.zetacored/cosmovisor/genesis /root/.zetacored/cosmovisor/current && \ + mkdir -p /root/.zetaclientd/upgrades/genesis && \ + ln -s /usr/local/bin/zetaclientd /root/.zetaclientd/upgrades/genesis/zetacored && \ + ln -s /root/.zetaclientd/upgrades/genesis /root/.zetaclientd/upgrades/current + +ENV PATH /root/.zetacored/cosmovisor/current/bin/:/root/.zetaclientd/upgrades/current/:${PATH} COPY contrib/localnet/scripts /root COPY contrib/localnet/preparams /root/preparams @@ -34,11 +50,32 @@ COPY contrib/localnet/zetacored /root/zetacored COPY contrib/localnet/tss /root/tss RUN chmod 755 /root/*.sh -RUN chmod 700 /root/.ssh -RUN chmod 600 /root/.ssh/* WORKDIR /usr/local/bin -ENV SHELL /bin/sh EXPOSE 22 -ENTRYPOINT ["/usr/sbin/sshd", "-D"] +FROM base-runtime AS latest-runtime + +COPY --from=cosmovisor-build /go/bin/cosmovisor /usr/local/bin +COPY --from=latest-build /go/bin/zetacored /go/bin/zetaclientd /go/bin/zetaclientd-supervisor /go/bin/zetae2e /usr/local/bin + +# optional old version build. This old build is used as the genesis version in the upgrade tests. +# use --target latest-runtime to skip +# +# TODO: just download binaries from github release now that we're using glibc +# we can't do this right now since we do not have a v16 release candidate +# https://github.com/zeta-chain/node/issues/2179 +FROM base-build as old-build + +ARG OLD_VERSION +RUN git clone https://github.com/zeta-chain/node.git +RUN cd node && git fetch + +RUN cd node && git checkout ${OLD_VERSION} +RUN cd node && make install + +FROM base-runtime AS old-runtime + +COPY --from=cosmovisor-build /go/bin/cosmovisor /usr/local/bin +COPY --from=old-build /go/bin/zetacored /go/bin/zetaclientd /usr/local/bin +COPY --from=latest-build /go/bin/zetaclientd-supervisor /usr/local/bin \ No newline at end of file diff --git a/Dockerfile-upgrade b/Dockerfile-upgrade deleted file mode 100644 index 8142f91297..0000000000 --- a/Dockerfile-upgrade +++ /dev/null @@ -1,77 +0,0 @@ -FROM golang:1.20-alpine3.18 as base - -# Purpose: This Dockerfile creates an environment for performing an upgrade test on ZetaChain -# It contains the ZetaChain and ZetaClient binaries for two different versions of ZetaChain -# One is specified with the old_version argument and the other is the current source code -# The image also contains the cosmovisor binary for running the upgrade test - -ENV GOPATH /go -ENV GOOS=linux -ENV CGO_ENABLED=1 -ENV GOCACHE=/root/.cache/go-build - -RUN apk --no-cache add git make build-base jq openssh libusb-dev linux-headers bash curl tmux -RUN ssh-keygen -b 2048 -t rsa -f /root/.ssh/localtest.pem -q -N "" - -# Build cosmovisor -RUN go install cosmossdk.io/tools/cosmovisor/cmd/cosmovisor@v1.5.0 - -WORKDIR /go/delivery/zeta-node - -RUN mkdir -p $GOPATH/bin/old -RUN mkdir -p $GOPATH/bin/new - -RUN ssh-keygen -A -RUN cp /root/.ssh/localtest.pem.pub /root/.ssh/authorized_keys - -# Checkout and build old binary -FROM base as oldbuild -ARG OLD_VERSION -RUN git clone https://github.com/zeta-chain/node.git -RUN cd node && git fetch - -RUN cd node && git checkout ${OLD_VERSION} -RUN cd node && make install - -# Build new release from the current source -FROM base -COPY go.mod /go/delivery/zeta-node/ -COPY go.sum /go/delivery/zeta-node/ -RUN cd /go/delivery/zeta-node/ && go mod download -COPY . /go/delivery/zeta-node/ -RUN --mount=type=cache,target="/root/.cache/go-build" cd /go/delivery/zeta-node/ && make install -RUN --mount=type=cache,target="/root/.cache/go-build" cd /go/delivery/zeta-node/ && make install-zetae2e -RUN cp $GOPATH/bin/zetacored $GOPATH/bin/new/ && \ - cp $GOPATH/bin/zetaclientd $GOPATH/bin/new/ - -COPY --from=oldbuild ${GOPATH}/bin/zetaclientd /root/.zetaclientd/upgrades/genesis/ -RUN mkdir -p /root/.zetaclientd/upgrades/${NEW_VERSION}/ && \ - cp ${GOPATH}/bin/zetaclientd /root/.zetaclientd/upgrades/${NEW_VERSION}/ -RUN ln -s /root/.zetaclientd/upgrades/genesis /root/.zetaclientd/upgrades/current -ENV PATH="/root/.zetaclientd/upgrades/current:${PATH}" - -COPY --from=oldbuild $GOPATH/bin/zetacored $GOPATH/bin/zetaclientd $GOPATH/bin/ -COPY --from=oldbuild $GOPATH/bin/zetacored $GOPATH/bin/zetaclientd $GOPATH/bin/old - -WORKDIR /root - -RUN cp /go/bin/zetaclientd /usr/local/bin -RUN cp /go/bin/zetacored /usr/local/bin -RUN cp /go/bin/zetae2e /usr/local/bin -RUN cp /go/bin/cosmovisor /usr/local/bin - -COPY contrib/localnet/scripts /root -COPY contrib/localnet/preparams /root/preparams -COPY contrib/localnet/ssh_config /root/.ssh/config -COPY contrib/localnet/zetacored /root/zetacored -COPY contrib/localnet/tss /root/tss - -RUN chmod 755 /root/*.sh -RUN chmod 700 /root/.ssh -RUN chmod 600 /root/.ssh/* - -WORKDIR /usr/local/bin -ENV SHELL /bin/sh -EXPOSE 22 - -ENTRYPOINT ["/usr/sbin/sshd", "-D"] diff --git a/Makefile b/Makefile index 2a55c49a76..635e718857 100644 --- a/Makefile +++ b/Makefile @@ -194,7 +194,7 @@ generate: proto-gen openapi specs typescript docs-zetacored zetanode: @echo "Building zetanode" - $(DOCKER) build -t zetanode -f ./Dockerfile-localnet . + $(DOCKER) build -t zetanode --target latest-runtime -f ./Dockerfile-localnet . $(DOCKER) build -t orchestrator -f contrib/localnet/orchestrator/Dockerfile.fastbuild . .PHONY: zetanode @@ -220,9 +220,9 @@ start-stress-test: zetanode cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-stresstest.yml up -d #TODO: replace OLD_VERSION with v16 tag once its available -zetanode-upgrade: +zetanode-upgrade: zetanode @echo "Building zetanode-upgrade" - $(DOCKER) build -t zetanode -f ./Dockerfile-upgrade --build-arg OLD_VERSION='release/v16' . + $(DOCKER) build -t zetanode:old -f Dockerfile-localnet --target old-runtime --build-arg OLD_VERSION='release/v16' . $(DOCKER) build -t orchestrator -f contrib/localnet/orchestrator/Dockerfile.fastbuild . .PHONY: zetanode-upgrade @@ -232,7 +232,7 @@ start-upgrade-test: zetanode-upgrade start-upgrade-test-light: zetanode-upgrade @echo "--> Starting light upgrade test (no ZetaChain state populating before upgrade)" - cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-upgrade-light.yml up -d + cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-upgrade.yml -f docker-compose-upgrade-light.yml up -d start-localnet: zetanode @echo "--> Starting localnet" diff --git a/contrib/localnet/docker-compose-upgrade-light.yml b/contrib/localnet/docker-compose-upgrade-light.yml index 0599f1acfc..55dd312a59 100644 --- a/contrib/localnet/docker-compose-upgrade-light.yml +++ b/contrib/localnet/docker-compose-upgrade-light.yml @@ -4,29 +4,8 @@ version: "3" # By using 90, the orchestrator will automatically run setup only for the first e2e test execution. services: - zetacore0: - entrypoint: ["/root/start-zetacored.sh", "2", "upgrade", "90"] - build: - context: ../../. - dockerfile: Dockerfile-upgrade - - zetacore1: - entrypoint: ["/root/start-zetacored.sh", "2", "upgrade", "90"] - build: - context: ../../. - dockerfile: Dockerfile-upgrade - - zetaclient0: - entrypoint: ["/root/start-zetaclientd.sh", "background"] - build: - context: ../../. - dockerfile: Dockerfile-upgrade - - zetaclient1: - entrypoint: ["/root/start-zetaclientd.sh", "background"] - build: - context: ../../. - dockerfile: Dockerfile-upgrade - orchestrator: entrypoint: ["/work/start-zetae2e.sh", "local", "upgrade", "90"] + + upgradeorchestrator: + entrypoint: ["/root/start-upgradeorchestrator.sh", "90"] \ No newline at end of file diff --git a/contrib/localnet/docker-compose-upgrade.yml b/contrib/localnet/docker-compose-upgrade.yml index 3bf858ec9a..43af2c6768 100644 --- a/contrib/localnet/docker-compose-upgrade.yml +++ b/contrib/localnet/docker-compose-upgrade.yml @@ -7,28 +7,45 @@ version: "3" services: zetacore0: - entrypoint: ["/root/start-zetacored.sh", "2", "upgrade"] - build: - context: ../../. - dockerfile: Dockerfile-upgrade + image: zetanode:old zetacore1: - entrypoint: ["/root/start-zetacored.sh", "2", "upgrade"] - build: - context: ../../. - dockerfile: Dockerfile-upgrade + image: zetanode:old zetaclient0: entrypoint: ["/root/start-zetaclientd.sh", "background"] - build: - context: ../../. - dockerfile: Dockerfile-upgrade + image: zetanode:old zetaclient1: entrypoint: ["/root/start-zetaclientd.sh", "background"] - build: - context: ../../. - dockerfile: Dockerfile-upgrade + image: zetanode:old orchestrator: entrypoint: ["/work/start-zetae2e.sh", "local", "upgrade"] + image: orchestrator + + upgrade-host: + image: zetanode:latest + container_name: upgrade-host + hostname: upgrade-host + entrypoint: ["/root/start-upgrade-host.sh"] + networks: + mynetwork: + ipv4_address: 172.20.0.250 + volumes: + - ssh:/root/.ssh + + upgrade-orchestrator: + # must run from old node for api compatibility + image: zetanode:old + container_name: upgrade-orchestrator + hostname: upgrade-orchestrator + entrypoint: ["/root/start-upgrade-orchestrator.sh", "225"] + networks: + mynetwork: + ipv4_address: 172.20.0.251 + depends_on: + - zetacore0 + - upgrade-host + volumes: + - ssh:/root/.ssh \ No newline at end of file diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml index 52ce7d2124..1c930d29d3 100644 --- a/contrib/localnet/docker-compose.yml +++ b/contrib/localnet/docker-compose.yml @@ -26,13 +26,12 @@ services: mynetwork: ipv4_address: 172.20.0.200 entrypoint: ["zetacored", "rosetta", "--tendermint", "zetacore0:26657", "--grpc", "zetacore0:9090", "--network", "athens_101-1", "--blockchain", "zetacore"] + volumes: + - ssh:/root/.ssh zetacore0: image: zetanode:latest container_name: zetacore0 - build: - context: ../../. - dockerfile: Dockerfile-localnet hostname: zetacore0 ports: - "1317:1317" @@ -48,13 +47,12 @@ services: environment: - HOTKEY_BACKEND=file - HOTKEY_PASSWORD=password # test purposes only + volumes: + - ssh:/root/.ssh zetacore1: image: zetanode:latest container_name: zetacore1 - build: - context: ../../. - dockerfile: Dockerfile-localnet hostname: zetacore1 networks: mynetwork: @@ -63,13 +61,12 @@ services: environment: - HOTKEY_BACKEND=file - HOTKEY_PASSWORD=password # test purposes only + volumes: + - ssh:/root/.ssh zetaclient0: image: zetanode:latest container_name: zetaclient0 - build: - context: ../../. - dockerfile: Dockerfile-localnet hostname: zetaclient0 networks: mynetwork: @@ -79,13 +76,12 @@ services: - ETHDEV_ENDPOINT=http://eth:8545 - HOTKEY_BACKEND=file - HOTKEY_PASSWORD=password # test purposes only + volumes: + - ssh:/root/.ssh zetaclient1: image: zetanode:latest container_name: zetaclient1 - build: - context: ../../. - dockerfile: Dockerfile-localnet hostname: zetaclient1 networks: mynetwork: @@ -95,6 +91,8 @@ services: - ETHDEV_ENDPOINT=http://eth:8545 - HOTKEY_BACKEND=file - HOTKEY_PASSWORD=password # test purposes only + volumes: + - ssh:/root/.ssh eth: image: ethereum/client-go:v1.10.26 @@ -128,9 +126,6 @@ services: image: orchestrator:latest tty: true container_name: orchestrator - build: - context: ../../. - dockerfile: contrib/localnet/orchestrator/Dockerfile depends_on: - zetacore0 - eth @@ -139,4 +134,7 @@ services: mynetwork: ipv4_address: 172.20.0.2 entrypoint: ["/work/start-zetae2e.sh", "local"] - + volumes: + - ssh:/root/.ssh +volumes: + ssh: \ No newline at end of file diff --git a/contrib/localnet/orchestrator/Dockerfile b/contrib/localnet/orchestrator/Dockerfile deleted file mode 100644 index e159d8ba84..0000000000 --- a/contrib/localnet/orchestrator/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM zetanode:latest as zeta -FROM ethereum/client-go:v1.10.26 as geth -FROM golang:1.20-alpine as orchestrator - -RUN apk --no-cache add jq curl bash make git build-base openssh - -COPY --from=zeta /usr/local/bin/zetacored /usr/local/bin/ -COPY --from=zeta /usr/local/bin/zetaclientd /usr/local/bin/ -COPY --from=geth /usr/local/bin/geth /usr/local/bin/ - -COPY --from=zeta /root/.ssh/localtest.pem.pub /root/.ssh/authorized_keys -COPY --from=zeta /root/.ssh/localtest.pem /root/.ssh/localtest.pem - -COPY contrib/localnet/orchestrator/start-zetae2e.sh /work/ -COPY contrib/localnet/orchestrator/restart-zetaclientd.sh /work/ -RUN chmod +x /work/*.sh - -ENV GOPATH /go -ENV GOOS=linux -ENV CGO_ENABLED=1 -WORKDIR /go/delivery/zeta-node -COPY go.mod . -COPY go.sum . -RUN go mod download -COPY . . -RUN make install-zetae2e -RUN mv /go/bin/zetae2e /usr/local/bin/zetae2e -RUN chmod +x /usr/local/bin/zetae2e - -WORKDIR /work diff --git a/contrib/localnet/orchestrator/Dockerfile.fastbuild b/contrib/localnet/orchestrator/Dockerfile.fastbuild index 9be2c48124..332ed3330e 100644 --- a/contrib/localnet/orchestrator/Dockerfile.fastbuild +++ b/contrib/localnet/orchestrator/Dockerfile.fastbuild @@ -1,20 +1,15 @@ FROM zetanode:latest as zeta FROM ethereum/client-go:v1.10.26 as geth -FROM golang:1.20-alpine as orchestrator +FROM debian:bullseye-slim as orchestrator -RUN apk --no-cache add jq curl bash make git build-base openssh +RUN apt update && \ + apt install -yq jq curl tmux python3 openssh-server iputils-ping iproute2 && \ + rm -rf /var/lib/apt/lists/* -COPY --from=zeta /usr/local/bin/zetacored /usr/local/bin/ -COPY --from=zeta /usr/local/bin/zetaclientd /usr/local/bin/ COPY --from=geth /usr/local/bin/geth /usr/local/bin/ - -COPY --from=zeta /root/.ssh/localtest.pem.pub /root/.ssh/authorized_keys -COPY --from=zeta /root/.ssh/localtest.pem /root/.ssh/localtest.pem +COPY --from=zeta /usr/local/bin/zetacored /usr/local/bin/zetaclientd /usr/local/bin/zetae2e /usr/local/bin/ COPY contrib/localnet/orchestrator/start-zetae2e.sh /work/ RUN chmod +x /work/*.sh -COPY --from=zeta /usr/local/bin/zetae2e /usr/local/bin/ -RUN chmod +x /usr/local/bin/zetae2e - WORKDIR /work diff --git a/contrib/localnet/orchestrator/start-zetae2e.sh b/contrib/localnet/orchestrator/start-zetae2e.sh index 772d7dd496..f1f7899f43 100644 --- a/contrib/localnet/orchestrator/start-zetae2e.sh +++ b/contrib/localnet/orchestrator/start-zetae2e.sh @@ -8,6 +8,12 @@ ZETAE2E_CMD=$1 OPTION=$2 +# Wait for authorized_keys file to exist (generated by zetacore0) +while [ ! -f ~/.ssh/authorized_keys ]; do + echo "Waiting for authorized_keys file to exist..." + sleep 1 +done + echo "waiting for geth RPC to start..." sleep 2 diff --git a/contrib/localnet/scripts/start-upgrade-host.sh b/contrib/localnet/scripts/start-upgrade-host.sh new file mode 100644 index 0000000000..013e575a11 --- /dev/null +++ b/contrib/localnet/scripts/start-upgrade-host.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +echo "serving binaries for upgrade proposal" +python3 -m http.server \ No newline at end of file diff --git a/contrib/localnet/scripts/start-upgrade-orchestrator.sh b/contrib/localnet/scripts/start-upgrade-orchestrator.sh new file mode 100755 index 0000000000..c64982fa85 --- /dev/null +++ b/contrib/localnet/scripts/start-upgrade-orchestrator.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +UPGRADE_HEIGHT=$1 + +CHAINID="athens_101-1" +UPGRADE_AUTHORITY_ACCOUNT="zeta10d07y265gmmuvt4z0w9aw880jnsr700jvxasvr" + +# Wait for authorized_keys file to exist (populated by zetacore0) +while [ ! -f ~/.ssh/authorized_keys ]; do + echo "Waiting for authorized_keys file to exist..." + sleep 1 +done + +while ! curl -s -o /dev/null zetacore0:26657/status ; do + echo "Waiting for zetacore0 rpc" + sleep 1 +done + +# wait for minimum height +CURRENT_HEIGHT=0 +while [[ $CURRENT_HEIGHT -lt 1 ]] +do + CURRENT_HEIGHT=$(curl -s zetacore0:26657/status | jq '.result.sync_info.latest_block_height' | tr -d '"') + echo "current height is ${CURRENT_HEIGHT}, waiting for 1" + sleep 1 +done + +# copy zetacore0 config and keys +scp -r zetacore0:"~/.zetacored/config ~/.zetacored/os_info ~/.zetacored/config ~/.zetacored/keyring-file ~/.zetacored/keyring-test" ~/.zetacored/ +sed -i 's|tcp://localhost:26657|tcp://zetacore0:26657|g' ~/.zetacored/config/client.toml + +# get new zetacored version +curl -o /tmp/zetacored.new http://upgradehost:8000/zetacored +chmod +x /tmp/zetacored.new +UPGRADE_NAME=$(/tmp/zetacored.new version) + +# if explicit upgrade height not provided, use dumb estimator +if [[ -z $UPGRADE_HEIGHT ]]; then + UPGRADE_HEIGHT=$(( $(curl -s zetacore0:26657/status | jq '.result.sync_info.latest_block_height' | tr -d '"') + 60)) + echo "Upgrade height was not provided. Estimating ${UPGRADE_HEIGHT}." +fi + +cat > upgrade.json < upgrade_plan_info.json < tmp.json && mv tmp.json /root/.zetacored/config/zetaclient_config.json } +# Wait for authorized_keys file to exist (generated by zetacore0) +while [ ! -f ~/.ssh/authorized_keys ]; do + echo "Waiting for authorized_keys file to exist..." + sleep 1 +done + # read HOTKEY_BACKEND env var for hotkey keyring backend and set default to test BACKEND="test" if [ "$HOTKEY_BACKEND" == "file" ]; then BACKEND="file" fi -cp /root/preparams/PreParams_$HOSTNAME.json /root/preParams.json +cp /root/preparams/PreParams_$HOSTNAME.json /root/preParams.json num=$(echo $HOSTNAME | tr -dc '0-9') node="zetacore$num" -echo "Wait for zetacore to exchange genesis file" -sleep 40 +while [ ! -f $HOME/.zetacored/os.json ]; do + echo "Waiting for zetacore to exchange os.json file..." + sleep 1 +done operator=$(cat $HOME/.zetacored/os.json | jq '.ObserverAddress' ) operatorAddress=$(echo "$operator" | tr -d '"') echo "operatorAddress: $operatorAddress" @@ -54,7 +63,7 @@ else SEED=$(curl --retry 10 --retry-delay 5 --retry-connrefused -s zetaclient0:8123/p2p) done rm ~/.tss/* - zetaclientd init --peer /ip4/172.20.0.21/tcp/6668/p2p/"$SEED" --zetacore-url "$node" --chain-id athens_101-1 --operator "$operatorAddress" --log-format=text --public-ip "$MYIP" --log-level 1 --keyring-backend "$BACKEND" + zetaclientd init --peer "/ip4/172.20.0.21/tcp/6668/p2p/${SEED}" --zetacore-url "$node" --chain-id athens_101-1 --operator "$operatorAddress" --log-format=text --public-ip "$MYIP" --log-level 1 --keyring-backend "$BACKEND" # check if the option is additional-evm # in this case, the additional evm is represented with the sepolia chain, we set manually the eth2 endpoint to the sepolia chain (11155111 -> http://eth2:8545) diff --git a/contrib/localnet/scripts/start-zetacored.sh b/contrib/localnet/scripts/start-zetacored.sh index 39ad997eef..57c4097b0d 100755 --- a/contrib/localnet/scripts/start-zetacored.sh +++ b/contrib/localnet/scripts/start-zetacored.sh @@ -4,8 +4,6 @@ # It initializes the nodes and creates the genesis.json file # It also starts the nodes # The number of nodes is passed as an first argument to the script -# The second argument is optional and can have the following value: -# 1. upgrade : This is used to test the upgrade process, a proposal is created for the upgrade and the nodes are started using cosmovisor /usr/sbin/sshd @@ -15,14 +13,12 @@ then exit 1 fi NUMOFNODES=$1 -OPTION=$2 # create keys CHAINID="athens_101-1" KEYRING="test" HOSTNAME=$(hostname) INDEX=${HOSTNAME:0-1} -UPGRADE_AUTHORITY_ACCOUNT="zeta10d07y265gmmuvt4z0w9aw880jnsr700jvxasvr" # Environment variables used for upgrade testing export DAEMON_HOME=$HOME/.zetacored @@ -35,7 +31,6 @@ export DAEMON_DATA_BACKUP_DIR=$DAEMON_HOME export CLIENT_SKIP_UPGRADE=true export CLIENT_START_PROCESS=false export UNSAFE_SKIP_BACKUP=true -export UpgradeName=$(${GOPATH}/bin/new/zetacored version) # generate node list START=1 @@ -49,6 +44,25 @@ done echo "HOSTNAME: $HOSTNAME" +# init ssh keys +# we generate keys at runtime to ensure that keys are never pushed to +# a docker registry +if [ $HOSTNAME == "zetacore0" ]; then + if [[ ! -f ~/.ssh/id_rsa ]]; then + ssh-keygen -t rsa -q -N "" -f ~/.ssh/id_rsa + cp ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys + # keep localtest.pem for compatibility + cp ~/.ssh/id_rsa ~/.ssh/localtest.pem + chmod 600 ~/.ssh/* + fi +fi + +# Wait for authorized_keys file to exist (zetacore1+) +while [ ! -f ~/.ssh/authorized_keys ]; do + echo "Waiting for authorized_keys file to exist..." + sleep 1 +done + # Init a new node to generate genesis file . # Copy config files from existing folders which get copied via Docker Copy when building images mkdir -p ~/.backup/config @@ -67,9 +81,16 @@ source ~/add-keys.sh # Pause other nodes so that the primary can node can do the genesis creation if [ $HOSTNAME != "zetacore0" ] then - echo "Waiting for zetacore0 to create genesis.json" - sleep 10 - echo "genesis.json created" + while [ ! -f ~/.zetacored/config/genesis.json ]; do + echo "Waiting for genesis.json file to exist..." + sleep 1 + done + # need to wait for zetacore0 to be up otherwise you get + # + while ! curl -s -o /dev/null zetacore0:26657/status ; do + echo "Waiting for zetacore0 rpc" + sleep 1 +done fi # Genesis creation following steps @@ -94,7 +115,10 @@ then for NODE in "${NODELIST[@]}"; do INDEX=${NODE:0-1} ssh zetaclient"$INDEX" mkdir -p ~/.zetacored/ - scp "$NODE":~/.zetacored/os_info/os.json ~/.zetacored/os_info/os_z"$INDEX".json + while ! scp "$NODE":~/.zetacored/os_info/os.json ~/.zetacored/os_info/os_z"$INDEX".json; do + echo "Waiting for os_info.json from node $NODE" + sleep 1 + done scp ~/.zetacored/os_info/os_z"$INDEX".json zetaclient"$INDEX":~/.zetacored/os.json done @@ -180,73 +204,4 @@ then sed -i -e "/persistent_peers =/s/=.*/= \"$pps\"/" "$HOME"/.zetacored/config/config.toml fi -# 7 Start the nodes -# If upgrade option is passed, use cosmovisor to start the nodes and create a governance proposal for upgrade -if [ "$OPTION" != "upgrade" ]; then - - exec zetacored start --pruning=nothing --minimum-gas-prices=0.0001azeta --json-rpc.api eth,txpool,personal,net,debug,web3,miner --api.enable --home /root/.zetacored - -else - - # Setup cosmovisor - mkdir -p $DAEMON_HOME/cosmovisor/genesis/bin - mkdir -p $DAEMON_HOME/cosmovisor/upgrades/"$UpgradeName"/bin - - # Genesis - cp $GOPATH/bin/old/zetacored $DAEMON_HOME/cosmovisor/genesis/bin - cp $GOPATH/bin/zetaclientd $DAEMON_HOME/cosmovisor/genesis/bin - - #Upgrades - cp $GOPATH/bin/new/zetacored $DAEMON_HOME/cosmovisor/upgrades/$UpgradeName/bin/ - - #Permissions - chmod +x $DAEMON_HOME/cosmovisor/genesis/bin/zetacored - chmod +x $DAEMON_HOME/cosmovisor/genesis/bin/zetaclientd - chmod +x $DAEMON_HOME/cosmovisor/upgrades/$UpgradeName/bin/zetacored - - # Start the node using cosmovisor - cosmovisor run start --pruning=nothing --minimum-gas-prices=0.0001azeta --json-rpc.api eth,txpool,personal,net,debug,web3,miner --api.enable --home /root/.zetacored >> zetanode.log 2>&1 & - sleep 20 - echo - - # Fetch the height of the upgrade, default is 225, if arg3 is passed, use that value - UPGRADE_HEIGHT=${3:-225} - - # If this is the first node, create a governance proposal for upgrade - if [ $HOSTNAME = "zetacore0" ] - then - cat > upgrade.json <