ci: Dockerfile & Docker Pipeline & Docker Compose Improvements

.github/workflows/docker-build-and-push.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Set Version from the PR title.
         if: github.event_name == 'pull_request'
         run: |
-          echo "GITHUB_TAG_MAJOR_VERSION=${{ github.event.pull_request.title }}" >> ${GITHUB_ENV}
+          echo "GITHUB_TAG_MAJOR_VERSION=$(grep 'const releaseVersion =' app/setup_handlers.go | cut -d '"' -f 2 | tr -d '\n')" >> ${GITHUB_ENV}
 
       - name: Set Version for Hotfix Release from Input.
         if: github.event_name != 'pull_request'
@@ -66,7 +66,7 @@ jobs:
       - name: Set Version from the PR title.
         if: github.event_name == 'pull_request'
         run: |
-          echo "GITHUB_TAG_MAJOR_VERSION=${{ github.event.pull_request.title }}" >> ${GITHUB_ENV}
+          echo "GITHUB_TAG_MAJOR_VERSION=$(grep 'const releaseVersion =' app/setup_handlers.go | cut -d '"' -f 2 | tr -d '\n')" >> ${GITHUB_ENV}
 
       - name: Set Version for Hotfix Release from Input.
         if: github.event_name != 'pull_request'
@@ -109,7 +109,7 @@ jobs:
       - name: Set Version from the PR title.
         if: github.event_name == 'pull_request'
         run: |
-          echo "GITHUB_TAG_MAJOR_VERSION=${{ github.event.pull_request.title }}" >> ${GITHUB_ENV}
+          echo "GITHUB_TAG_MAJOR_VERSION=$(grep 'const releaseVersion =' app/setup_handlers.go | cut -d '"' -f 2 | tr -d '\n')" >> ${GITHUB_ENV}
 
       - name: Set Version for Hotfix Release from Input.
         if: github.event_name != 'pull_request'

Dockerfile

@@ -1,51 +1,53 @@
-# Purpose: This Dockerfile creates an environment for running ZetaChain
-# It contains:
-# - zetacored: the ZetaChain node binary
-# - zetaclientd: the ZetaChain client binary for observers
-# - zetae2e: the ZetaChain end-to-end tests CLI
-
-FROM golang:1.20-alpine3.18
+# Build Stage
+FROM golang:1.20-alpine3.18 AS builder
 
 ENV GOPATH /go
 ENV GOOS=linux
 ENV CGO_ENABLED=1
 
-RUN apk --no-cache add git make build-base jq openssh libusb-dev linux-headers bash curl tmux
-RUN ssh-keygen -b 2048 -t rsa -f /root/.ssh/localtest.pem -q -N ""
+# Install build dependencies
+RUN apk --no-cache add git make build-base jq openssh libusb-dev linux-headers bash curl tmux python3 py3-pip
 
+# Set the working directory
 WORKDIR /go/delivery/zeta-node
+
+# Copy module files and download dependencies
 COPY go.mod .
 COPY go.sum .
-
 RUN go mod download
+
+# Copy the rest of the source code and build the application
 COPY . .
-RUN make install
-RUN make install-zetae2e
-#
-#FROM golang:1.20-alpine
 
-#RUN apk --no-cache add openssh jq tmux vim curl bash
-RUN ssh-keygen -A
-WORKDIR /root
+RUN make install
 
-RUN cp /root/.ssh/localtest.pem.pub /root/.ssh/authorized_keys
+# Run Stage
+FROM alpine:3.18
 
-RUN cp /go/bin/zetaclientd /usr/local/bin
-RUN cp /go/bin/zetacored /usr/local/bin
-RUN cp /go/bin/zetae2e /usr/local/bin
+# Copy Start Script Helpers
+COPY contrib/docker-scripts/* /scripts/
 
-COPY contrib/localnet/scripts /root
-COPY contrib/localnet/preparams /root/preparams
-COPY contrib/localnet/ssh_config /root/.ssh/config
-COPY contrib/localnet/zetacored /root/zetacored
-COPY contrib/localnet/tss /root/tss
+# Install runtime dependencies
+RUN apk --no-cache add git jq bash curl tmux python3 libusb-dev linux-headers make build-base wget py3-pip qemu-img qemu-system-x86_64 && \
+    pip install requests && \
+    chmod a+x -R /scripts && \
+    wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.31-r0/glibc-2.31-r0.apk && \
+    apk add --force-overwrite --allow-untrusted glibc-2.31-r0.apk
 
-RUN chmod 755 /root/*.sh
-RUN chmod 700 /root/.ssh
-RUN chmod 600 /root/.ssh/*
+# Copy the binaries from the build stage
+COPY --from=builder /go/bin/zetaclientd /usr/local/bin/zetaclientd
+COPY --from=builder /go/bin/zetacored /usr/local/bin/zetacored
 
+# Set the working directory
 WORKDIR /usr/local/bin
-ENV SHELL /bin/sh
-EXPOSE 22
 
-ENTRYPOINT ["/usr/sbin/sshd", "-D"]
+# Set the default shell
+ENV SHELL /bin/bash
+
+EXPOSE 26656
+EXPOSE 1317
+EXPOSE 8545
+EXPOSE 8546
+EXPOSE 9090
+EXPOSE 26657
+EXPOSE 9091

Dockerfile-localnet

@@ -0,0 +1,57 @@
+FROM golang:1.20-alpine3.18
+
+ENV GOPATH /go
+ENV GOOS=linux
+ENV CGO_ENABLED=1
+
+RUN apk --no-cache add git make build-base jq openssh libusb-dev linux-headers bash curl tmux python3 py3-pip
+RUN pip install requests
+RUN ssh-keygen -b 2048 -t rsa -f /root/.ssh/localtest.pem -q -N ""
+
+  apt-get update
+  apt-get install nano jq python3 -y
+  pip3 install requests
+
+
+WORKDIR /go/delivery/zeta-node
+COPY go.mod .
+COPY go.sum .
+#RUN --mount=type=cache,target=/root/.cache/go-build \
+#    go mod download
+RUN go mod download
+COPY . .
+
+#RUN --mount=type=cache,target=/root/.cache/go-build \
+#    make install
+#RUN --mount=type=cache,target=/root/.cache/go-build \
+#    make install-zetae2e
+RUN make install
+RUN make install-zetae2e
+#
+#FROM golang:1.20-alpine
+
+#RUN apk --no-cache add openssh jq tmux vim curl bash
+RUN ssh-keygen -A
+WORKDIR /root
+
+RUN cp /root/.ssh/localtest.pem.pub /root/.ssh/authorized_keys
+
+RUN cp /go/bin/zetaclientd /usr/local/bin
+RUN cp /go/bin/zetacored /usr/local/bin
+RUN cp /go/bin/zetae2e /usr/local/bin
+
+COPY contrib/localnet/scripts /root
+COPY contrib/localnet/preparams /root/preparams
+COPY contrib/localnet/ssh_config /root/.ssh/config
+COPY contrib/localnet/zetacored /root/zetacored
+COPY contrib/localnet/tss /root/tss
+
+RUN chmod 755 /root/*.sh
+RUN chmod 700 /root/.ssh
+RUN chmod 600 /root/.ssh/*
+
+WORKDIR /usr/local/bin
+ENV SHELL /bin/sh
+EXPOSE 22
+
+ENTRYPOINT ["/usr/sbin/sshd", "-D"]

Makefile

@@ -266,7 +266,10 @@ release:
 ###############################################################################
 
 mainnet-zetarpc-node:
-	cd contrib/local-mainnet/zetacored && docker-compose up
+	cd contrib/mainnet/zetacored && DOCKER_TAG=$(DOCKER_TAG) docker-compose up
 
 mainnet-bitcoind-node:
-	cd contrib/local-mainnet/bitcoind && docker-compose up
+	cd contrib/mainnet/bitcoind && DOCKER_TAG=$(DOCKER_TAG) docker-compose up
+
+athens3-zetarpc-node:
+	cd contrib/athens3/zetacored && DOCKER_TAG=$(DOCKER_TAG) docker-compose up

changelog.md

@@ -75,12 +75,17 @@
 * Added docker-compose and make commands for launching full nodes. `make mainnet-zetarpc-node`  `make mainnet-bitcoind-node`
 * Made adjustments to the docker-compose for launching mainnet full nodes to include examples of using the docker images build from the docker image build pipeline.
 * [1736](https://github.com/zeta-chain/node/pull/1736) - chore: add Ethermint endpoints to OpenAPI
+* Re-wrote Dockerfile for building Zetacored docker images. 
+* Adjusted the docker-compose files for Zetacored nodes to utilize the new docker image.
+* Added scripts for the new docker image that facilitate the start up automation.
+* Adjusted the docker pipeline slightly to pull the version on PR from the app.go file.
 * [1781](https://github.com/zeta-chain/node/pull/1781) - add codecov coverage report in CI
 
 ### Features
 
 * [1425](https://github.com/zeta-chain/node/pull/1425) add `whitelist-erc20` command
 
+
 ### Chores
 
 * [1729](https://github.com/zeta-chain/node/pull/1729) - add issue templates

contrib/athens3/zetacored/docker-compose.yml

@@ -0,0 +1,44 @@
+version: '3.8'
+services:
+  zetachain_mainnet_rpc:
+    platform: linux/amd64
+    #This will build the binary from the GIT_REF you are locally on.
+#    build:
+#      context: ../../..
+#      dockerfile: Dockerfile
+    image: zetachain/zeatcored:${DOCKER_TAG:-ubuntu-v14}
+    environment:
+      DAEMON_HOME: "/root/.zetacored"
+      NETWORK: athens3
+      #RESTORE_TYPE=snapshot/snapshot-archive/statesync
+      RESTORE_TYPE: "statesync"
+      SNAPSHOT_API: https://snapshots.zetachain.com
+      TRUST_HEIGHT_DIFFERENCE_STATE_SYNC: 40000
+      COSMOVISOR_VERSION: "v1.5.0"
+      CHAIN_ID: "athens_7001-1"
+      COSMOVISOR_CHECKSUM: "626dfc58c266b85f84a7ed8e2fe0e2346c15be98cfb9f9b88576ba899ed78cdc"
+      VISOR_NAME: "cosmovisor"
+      DAEMON_NAME: "zetacored"
+      DAEMON_ALLOW_DOWNLOAD_BINARIES: "false"
+      DAEMON_RESTART_AFTER_UPGRADE: "true"
+      UNSAFE_SKIP_BACKUP: "true"
+      CLIENT_DAEMON_NAME: "zetaclientd"
+      CLIENT_DAEMON_ARGS: ""
+      CLIENT_SKIP_UPGRADE: "true"
+      CLIENT_START_PROCESS: "false"
+      MONIKER: local-test
+      RE_DO_START_SEQUENCE: "false"
+    ports:
+      - "26656:26656"
+      - "1317:1317"
+      - "8545:8545"
+      - "8546:8546"
+      - "26657:26657"
+      - "9090:9090"
+      - "9091:9091"
+    volumes:
+      - zetacored_data:/root/.zetacored/
+    entrypoint: bash /scripts/start.sh
+
+volumes:
+  zetacored_data:

contrib/docker-scripts/download_binaries.py

@@ -0,0 +1,125 @@
+import re
+import requests
+import os
+import json
+import logging
+import sys
+import shutil
+
+
+# Logger class for easier logging setup
+class Logger:
+    def __init__(self):
+        self.log = logging.getLogger()
+        self.log.setLevel(logging.INFO)
+        self.handler = logging.StreamHandler(sys.stdout)
+        self.handler.setLevel(logging.DEBUG)
+        self.formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+        self.handler.setFormatter(self.formatter)
+        self.log.addHandler(self.handler)
+
+
+# Initialize logger instance
+logger = Logger()
+
+# Define the path where upgrades will be stored, using an environment variable for the base path
+upgrade_path = f'{os.environ["DAEMON_HOME"]}/cosmovisor/upgrades/'
+
+
+# Function to find the latest patch version of a binary based on major and optional minor version
+def find_latest_patch_version(major_version, minor_version=None):
+    # Define a regex pattern to match version directories
+    pattern = re.compile(
+        f"v{major_version}\.{minor_version}\.(\d+)" if minor_version else f"v{major_version}\.0\.(\d+)")
+    # List directories that match the version pattern
+    versions = [folder for folder in os.listdir(upgrade_path) if pattern.match(folder)]
+    if versions:
+        try:
+            # Find the maximum version, assuming it's the latest patch
+            latest_patch_version = max(versions)
+            # Return the path to the binary of the latest patch version
+            return os.path.join(upgrade_path, latest_patch_version, "bin", "zetacored")
+        except ValueError as e:
+            logger.log.error(f"Error finding latest patch version: {e}")
+            return None
+    return None
+
+
+# Function to replace an old binary with a new one
+def replace_binary(source, target):
+    try:
+        # Log deletion of old binary
+        if os.path.exists(target):
+            logger.log.info(f"Deleted old binary: {target}")
+            os.remove(target)
+        # Copy the new binary to the target location
+        shutil.copy(source, target)
+        logger.log.info(f"Binary replaced: {target} -> {source}")
+    except Exception as e:
+        logger.log.error(f"Error replacing binary: {e}")
+
+
+# Parse JSON from an environment variable to get binary download information
+info = json.loads(os.environ["DOWNLOAD_BINARIES"])
+
+try:
+    # Iterate over binaries to download
+    for binary in info["binaries"]:
+        download_link = binary
+        # Log download link
+        logger.log.info(f"DOWNLOAD LINK: {download_link}")
+        split_download_link = download_link.split("/")
+        # Log split download link parts
+        logger.log.info(f"SPLIT DOWNLOAD LINK: {split_download_link}")
+        # Extract binary name and version from the download link
+        binary_name = download_link.split("/")[8]
+        version = download_link.split("/")[7]
+        formatted_version = re.search(r'v\d{1,2}\.\d{1,2}\.\d{1,2}', version).group()
+        end_binary_name = os.environ["DAEMON_NAME"]
+        # Define the directory path where the binary will be stored
+        directory_path = f"{os.environ['DAEMON_HOME']}/{os.environ['VISOR_NAME']}/upgrades/{formatted_version}/bin"
+        # Check if binary already exists
+        logger.log.info(f"CHECKING / DOWNLOADING {directory_path}/{end_binary_name}")
+
+        if os.path.exists(f"{directory_path}/{end_binary_name}"):
+            # If binary exists, log and do nothing
+            logger.log.info(f"BINARY EXISTS ALREADY: {directory_path}/{end_binary_name}")
+        else:
+            # If binary doesn't exist, download and save it
+            logger.log.info("BINARY DOES NOT EXIST.")
+            os.makedirs(directory_path, exist_ok=True)
+            response = requests.get(download_link)
+            if response.status_code == 200:
+                with open(f"{directory_path}/{end_binary_name}", "wb") as f:
+                    f.write(response.content)
+                os.chmod(f"{directory_path}/{end_binary_name}", 0o755)
+                logger.log.info("BINARY DOWNLOADED SUCCESSFULLY.")
+            else:
+                logger.log.info("FAILED TO DOWNLOAD BINARY. Status code:", response.status_code)
+
+    logger.log.info("BINARIES DOWNLOAD FINISHED...")
+
+    # Start the process of upgrading binaries to the latest patch version
+    # versions = set()
+    # logger.log.info("UPGRADING BINARIES WITH LATEST PATCH VERSION UPGRADE")
+    # # Collect versions of all binaries
+    # for folder in os.listdir(upgrade_path):
+    #     match = re.match(r'v(\d+)\.(\d+)\.(\d+)', folder)
+    #     if match:
+    #         versions.add(match.groups())
+    #
+    # # For each version, find and replace with the latest patch version if applicable
+    # for major_version, minor_version, patch_version in versions:
+    #     logger.log.info(f"BINARY VERSION: v{major_version}.{minor_version}.{patch_version}")
+    #     latest_patch_version_path = find_latest_patch_version(major_version, minor_version)
+    #     if latest_patch_version_path:
+    #         logger.log.info(f"LATEST PATCH VERSION: {latest_patch_version_path}")
+    #         symlink_path = os.path.join(upgrade_path, f"v{major_version}.{minor_version}.{patch_version}", "bin",
+    #                                     "zetacored")
+    #         logger.log.info(f"UPDATING BINARY: {symlink_path} TO: {latest_patch_version_path}")
+    #         replace_binary(latest_patch_version_path, symlink_path)
+    #     else:
+    #         logger.log.info(f"NO PATCH UPDATE FOR v{major_version}.{minor_version}")
+
+except Exception as e:
+    logger.log.error(str(e))