refactor: unify CI docker images (#2158)

* wip

* remove static ssh keys

* exclude contrib in go build to improve iteration speed

* add upgradehost and upgrade orchestrator

* fix exclusions?

* final changes

* fix zetaclient-supervisor rebase

* use make target

* add full log dump on failure

* use variables

* fix zetacore1 startup race

* more race condition fixing

* review feedback

.dockerignore

@@ -3,4 +3,10 @@ package.json
 yarn.lock
 .github/
 .gitignore
-dist/**
+dist/**
+
+# dockerfiles are not needed inside the docker build
+Dockerfile
+Dockerfile-localnet
+Dockerfile-upgrade
+docker-compose*.yml

.github/workflows/build.yml

@@ -125,23 +125,21 @@ jobs:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_READ_ONLY }}
 
-      - name: Build zetanode
-        run: |
-          make zetanode
+      - name: Start Test
+        run: make start-e2e-test
 
-      - name: Start Private Network
+      # use docker logs -f rather than docker attach to make sure we get the initial logs
+      - name: Watch Test
         run: |
-          cd contrib/localnet/ 
-          docker compose up -d zetacore0 zetacore1 zetaclient0 zetaclient1 eth bitcoin
+          container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
+          docker logs -f "${container_id}" &
+          exit $(docker wait "${container_id}")
 
-      - name: Run E2E Test
+      - name: Full Log Dump On Failure
+        if: failure()
         run: |
           cd contrib/localnet
-          docker-compose up orchestrator --exit-code-from orchestrator
-          if [ $? -ne 0 ]; then
-            echo "E2E Test Failed"
-            exit 1
-          fi
+          docker compose logs
 
       - name: Notify Slack on Failure
         if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'

.github/workflows/execute_advanced_tests.yaml

@@ -46,6 +46,12 @@ jobs:
           container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
           docker logs -f "${container_id}" &
           exit $(docker wait "${container_id}")
+        
+      - name: Full Log Dump On Failure
+        if: failure()
+        run: |
+          cd contrib/localnet
+          docker compose logs
 
       - name: Notify Slack on Failure
         if: failure() && github.event_name == 'schedule'
@@ -73,6 +79,12 @@ jobs:
           docker logs -f "${container_id}" &
           exit $(docker wait "${container_id}")
 
+      - name: Full Log Dump On Failure
+        if: failure()
+        run: |
+          cd contrib/localnet
+          docker compose logs
+
       - name: Notify Slack on Failure
         if: failure() && github.event_name == 'schedule'
         uses: 8398a7/action-slack@v3
@@ -99,6 +111,12 @@ jobs:
           docker logs -f "${container_id}" &
           exit $(docker wait "${container_id}")
 
+      - name: Full Log Dump On Failure
+        if: failure()
+        run: |
+          cd contrib/localnet
+          docker compose logs
+
   e2e-performance-test:
     if: ${{ github.event.inputs.e2e-performance-test == 'true' }}
     runs-on: buildjet-4vcpu-ubuntu-2204
@@ -114,4 +132,10 @@ jobs:
         run: |
           container_id=$(docker ps --filter "ancestor=orchestrator:latest" --format "{{.ID}}")
           docker logs -f "${container_id}" &
-          exit $(docker wait "${container_id}")
+          exit $(docker wait "${container_id}")
+
+      - name: Full Log Dump On Failure
+        if: failure()
+        run: |
+          cd contrib/localnet
+          docker compose logs

Dockerfile-localnet

@@ -1,31 +1,47 @@
-FROM golang:1.20-alpine3.18
+# syntax=docker/dockerfile:1.7-labs
+FROM golang:1.20.14-bullseye AS base-build
 
 ENV GOPATH /go
 ENV GOOS=linux
 ENV CGO_ENABLED=1
 ENV GOCACHE=/root/.cache/go-build
 
-RUN apk --no-cache add git make build-base jq openssh libusb-dev linux-headers bash curl tmux python3 py3-pip
-RUN pip install requests
-RUN ssh-keygen -b 2048 -t rsa -f /root/.ssh/localtest.pem -q -N ""
+RUN apt update && \
+    apt install -yq libusb-dev
 
 WORKDIR /go/delivery/zeta-node
+
+FROM base-build AS latest-build
+
 COPY go.mod .
 COPY go.sum .
 RUN go mod download
-COPY . .
+COPY version.sh .
+COPY --exclude=*.sh --exclude=*.md --exclude=*.yml . .
 
 RUN --mount=type=cache,target="/root/.cache/go-build" make install
 RUN --mount=type=cache,target="/root/.cache/go-build" make install-zetae2e
 
-RUN ssh-keygen -A
-WORKDIR /root
+FROM golang:1.20.14-bullseye AS cosmovisor-build
+RUN go install cosmossdk.io/tools/cosmovisor/cmd/[email protected]
+
+FROM debian:bullseye-slim AS base-runtime
 
-RUN cp /root/.ssh/localtest.pem.pub /root/.ssh/authorized_keys
+RUN apt update && \
+    apt install -yq jq curl tmux python3 openssh-server iputils-ping iproute2 && \
+    rm -rf /var/lib/apt/lists/*
 
-RUN cp /go/bin/zetaclientd /usr/local/bin
-RUN cp /go/bin/zetacored /usr/local/bin
-RUN cp /go/bin/zetae2e /usr/local/bin
+RUN ssh-keygen -A && \ 
+    mkdir -p /var/run/sshd
+
+RUN mkdir -p /root/.zetacored/cosmovisor/genesis/bin && \
+    ln -s /usr/local/bin/zetacored /root/.zetacored/cosmovisor/genesis/bin/zetacored && \
+    ln -s /root/.zetacored/cosmovisor/genesis /root/.zetacored/cosmovisor/current && \
+    mkdir -p /root/.zetaclientd/upgrades/genesis && \
+    ln -s /usr/local/bin/zetaclientd /root/.zetaclientd/upgrades/genesis/zetacored && \
+    ln -s /root/.zetaclientd/upgrades/genesis /root/.zetaclientd/upgrades/current
+
+ENV PATH /root/.zetacored/cosmovisor/current/bin/:/root/.zetaclientd/upgrades/current/:${PATH}
 
 COPY contrib/localnet/scripts /root
 COPY contrib/localnet/preparams /root/preparams
@@ -34,11 +50,32 @@ COPY contrib/localnet/zetacored /root/zetacored
 COPY contrib/localnet/tss /root/tss
 
 RUN chmod 755 /root/*.sh
-RUN chmod 700 /root/.ssh
-RUN chmod 600 /root/.ssh/*
 
 WORKDIR /usr/local/bin
-ENV SHELL /bin/sh
 EXPOSE 22
 
-ENTRYPOINT ["/usr/sbin/sshd", "-D"]
+FROM base-runtime AS latest-runtime
+
+COPY --from=cosmovisor-build /go/bin/cosmovisor /usr/local/bin
+COPY --from=latest-build /go/bin/zetacored /go/bin/zetaclientd /go/bin/zetaclientd-supervisor /go/bin/zetae2e /usr/local/bin
+
+# optional old version build. This old build is used as the genesis version in the upgrade tests.
+# use --target latest-runtime to skip
+# 
+# TODO: just download binaries from github release now that we're using glibc
+# we can't do this right now since we do not have a v16 release candidate
+# https://github.com/zeta-chain/node/issues/2179
+FROM base-build as old-build
+
+ARG OLD_VERSION
+RUN git clone https://github.com/zeta-chain/node.git
+RUN cd node && git fetch
+
+RUN cd node && git checkout ${OLD_VERSION}
+RUN cd node && make install
+
+FROM base-runtime AS old-runtime
+
+COPY --from=cosmovisor-build /go/bin/cosmovisor /usr/local/bin
+COPY --from=old-build /go/bin/zetacored /go/bin/zetaclientd /usr/local/bin
+COPY --from=latest-build /go/bin/zetaclientd-supervisor /usr/local/bin

Makefile

@@ -194,7 +194,7 @@ generate: proto-gen openapi specs typescript docs-zetacored
 
 zetanode:
 	@echo "Building zetanode"
-	$(DOCKER) build -t zetanode -f ./Dockerfile-localnet .
+	$(DOCKER) build -t zetanode --target latest-runtime -f ./Dockerfile-localnet .
 	$(DOCKER) build -t orchestrator -f contrib/localnet/orchestrator/Dockerfile.fastbuild .
 .PHONY: zetanode
 
@@ -220,9 +220,9 @@ start-stress-test: zetanode
 	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-stresstest.yml up -d
 
 #TODO: replace OLD_VERSION with v16 tag once its available
-zetanode-upgrade:
+zetanode-upgrade: zetanode
 	@echo "Building zetanode-upgrade"
-	$(DOCKER) build -t zetanode -f ./Dockerfile-upgrade --build-arg OLD_VERSION='release/v16' .
+	$(DOCKER) build -t zetanode:old -f Dockerfile-localnet --target old-runtime --build-arg OLD_VERSION='release/v16' .
 	$(DOCKER) build -t orchestrator -f contrib/localnet/orchestrator/Dockerfile.fastbuild .
 .PHONY: zetanode-upgrade
 
@@ -232,7 +232,7 @@ start-upgrade-test: zetanode-upgrade
 
 start-upgrade-test-light: zetanode-upgrade
 	@echo "--> Starting light upgrade test (no ZetaChain state populating before upgrade)"
-	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-upgrade-light.yml up -d
+	cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml -f docker-compose-upgrade.yml -f docker-compose-upgrade-light.yml up -d
 
 start-localnet: zetanode
 	@echo "--> Starting localnet"

contrib/localnet/docker-compose-upgrade-light.yml

@@ -4,29 +4,8 @@ version: "3"
 # By using 90, the orchestrator will automatically run setup only for the first e2e test execution.
 
 services:
-  zetacore0:
-    entrypoint: ["/root/start-zetacored.sh", "2", "upgrade", "90"]
-    build:
-      context: ../../.
-      dockerfile: Dockerfile-upgrade
-
-  zetacore1:
-    entrypoint: ["/root/start-zetacored.sh", "2", "upgrade", "90"]
-    build:
-      context: ../../.
-      dockerfile: Dockerfile-upgrade
-
-  zetaclient0:
-    entrypoint: ["/root/start-zetaclientd.sh", "background"]
-    build:
-      context: ../../.
-      dockerfile: Dockerfile-upgrade
-
-  zetaclient1:
-    entrypoint: ["/root/start-zetaclientd.sh", "background"]
-    build:
-      context: ../../.
-      dockerfile: Dockerfile-upgrade
-
   orchestrator:
     entrypoint: ["/work/start-zetae2e.sh", "local", "upgrade", "90"]
+
+  upgradeorchestrator:
+    entrypoint: ["/root/start-upgradeorchestrator.sh", "90"]