rebase develop

zeta-chain · Jul 15, 2024 · 825ed46 · 825ed46
2 parents 43073d4 + 7363b33
commit 825ed46
Show file tree

Hide file tree

Showing 167 changed files with 5,406 additions and 3,242 deletions.
diff --git a/.github/actions/install-dependencies/action.yml b/.github/actions/install-dependencies/action.yml
@@ -33,7 +33,7 @@ runs:
     - uses: actions/setup-go@v5
       if: ${{ inputs.skip_go == 'false' }}
       with:
-        go-version: '1.20'
+        go-version: '1.22'
         cache: false
 
     - uses: actions/setup-python@v4

diff --git a/.github/actions/performance-tests/art.yaml b/.github/actions/performance-tests/art.yaml
@@ -1,8 +1,8 @@
 config:
-  target: "http://localhost:9545"
+  target: "-=http_request_protocol=-://-=endpoint_to_test=-"
   phases:
-    - duration: 60
-      arrivalRate: 10
+    - duration: 300
+      arrivalRate: 400
   defaults:
     headers:
       content-type: "application/json"
@@ -19,7 +19,6 @@ config:
     - type: "html"
       filename: "artillery_report.html"
   logLevel: debug
-
 scenarios:
   - name: web3_clientVersion
     flow:

diff --git a/.github/workflows/ci-nightly-performance-testing.yaml b/.github/workflows/ci-nightly-performance-testing.yaml
@@ -2,13 +2,20 @@ name: "NIGHTLY:EVM:PERFORMANCE:TESTING"
 
 on:
   workflow_dispatch:
+    inputs:
+      endpoint:
+        description: 'The EVM endpoint you want to test.'
+        required: false
+        default: 'http://localhost:9545'
   schedule:
     - cron: '0 0 * * *' # Runs every day at midnight UTC
 
 jobs:
   nightly_evm_api_performance_test:
     name: "NIGHTLY:EVM:API:PERFORMANCE:TEST"
     runs-on: "buildjet-16vcpu-ubuntu-2204"
+    env:
+      endpoint_to_test: "http://localhost:9545"
     steps:
       - uses: actions/checkout@v4
 
@@ -25,16 +32,62 @@ jobs:
         run: | 
           npm install -g artillery@latest
 
+      - name: "CONFIGURE:PERFORMANCE:TEMPLATE"
+        run: |
+          # Check if the inputs.endpoint starts with https:// or http://
+          if [[ "${{inputs.endpoint}}" =~ ^https:// ]]; then
+            # Set the http_request_protocol to https if the URL starts with https://
+            export http_request_protocol=https
+          elif [[ "${{inputs.endpoint}}" =~ ^http:// ]]; then
+            # Set the http_request_protocol to http if the URL starts with http://
+            export http_request_protocol=http
+          else
+            # If the URL doesn't start with either http:// or https://, print an error and exit
+            echo "Please input a url with either https:// or http://" && exit 1
+          fi
+          
+          # Check if inputs.endpoint is different from the default "http://localhost:9545"
+          if [ "${{inputs.endpoint}}" != "http://localhost:9545" ]; then
+            # Check if inputs.endpoint is not empty
+            if [ -n "${{inputs.endpoint}}" ]; then
+              # If inputs.endpoint isn't different from the default and is not empty, use inputs.endpoint
+              export endpoint_to_test=${{inputs.endpoint}}
+              # Remove the protocol (http:// or https://) from endpoint_to_test
+              endpoint_to_test=${endpoint_to_test#http://}
+              endpoint_to_test=${endpoint_to_test#https://}
+            else
+              # If inputs.endpoint is empty, strip the protocol from endpoint_to_test
+              endpoint_to_test=${endpoint_to_test#http://}
+              endpoint_to_test=${endpoint_to_test#https://}
+            fi
+          else
+            # If inputs.endpoint is "http://localhost:9545", strip the protocol from endpoint_to_test
+            endpoint_to_test=${endpoint_to_test#http://}
+            endpoint_to_test=${endpoint_to_test#https://}
+          fi
+          
+          # Replace -=endpoint_to_test=- placeholder in the art.yaml file with the endpoint_to_test value
+          sed -i "s/-=endpoint_to_test=-/${endpoint_to_test}/g" .github/actions/performance-tests/art.yaml
+          # Replace -=http_request_protocol=- placeholder in the art.yaml file with the http_request_protocol value
+          sed -i "s/-=http_request_protocol=-/${http_request_protocol}/g" .github/actions/performance-tests/art.yaml
+
       - name: "EXECUTE:PERFORMANCE:TESTS"
         run: | 
-          artillery run .github/actions/performance-tests/art.yaml --record --key ${{ secrets.ARTILLERY_KEY }} --output ./report.json
+          # Execute Artillery to run performance tests using the specified configuration file
+          artillery run .github/actions/performance-tests/art.yaml \
+          --record \
+          --key ${{ secrets.ARTILLERY_KEY }} \
+          --output ./report.json
+
+      - name: "CHECK:TEST:RESULTS"
+        run: |
           # Check Artillery exit status
           if [ $? -ne 0 ]; then
             echo "Artillery command failed to execute."
             exit 1
           fi
           
-          # Parse the results.json file to check for failed vusers and http response codes
+          # Parse the results.json file to check for failed vusers and HTTP response codes
           failed_vusers=$(jq '.aggregate.counters["vusers.failed"] // 0' ./report.json)
           http_codes_200=$(jq '.aggregate.counters["http.codes.200"] // 0' ./report.json)
           http_responses=$(jq '.aggregate.counters["http.responses"] // 0' ./report.json)
@@ -45,6 +98,26 @@ jobs:
           else
             echo "EVM Performance Testing Successful"
           fi
+      
+          # Read the JSON file and extract the required metrics
+          p99_values=$(jq -r '.aggregate.summaries["http.response_time"].p99, .aggregate.summaries["plugins.metrics-by-endpoint.response_time./"].p99, .aggregate.summaries["vusers.session_length"].p99' ./report.json)
+          p50_values=$(jq -r '.aggregate.summaries["http.response_time"].p50, .aggregate.summaries["plugins.metrics-by-endpoint.response_time./"].p50, .aggregate.summaries["vusers.session_length"].p50' ./report.json)
+
+          # Check P99 values
+          for p99 in $p99_values; do
+            if (( $(echo "$p99 > 2000" | bc -l) )); then
+              echo "P99 value $p99 exceeds 2000ms"
+              exit 1
+            fi
+          done
+
+          # Check P50 values
+          for p50 in $p50_values; do
+            if (( $(echo "$p50 > 40" | bc -l) )); then
+              echo "P50 value $p50 exceeds 40ms"
+              exit 1
+            fi
+          done
 
       - name: "GENERATE:REPORT"
         if: always()

diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml
@@ -48,7 +48,7 @@ jobs:
         if: ${{ github.event.inputs.skip_checks != 'true' }}
         uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: '1.22'
 
       - name: Run Gosec Security Scanner
         if: ${{ github.event.inputs.skip_checks != 'true' }}
@@ -79,7 +79,7 @@ jobs:
         if: ${{ github.event.inputs.skip_checks != 'true' }}
         uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: '1.22'
 
       - name: Run Cosmos Gosec Security Scanner
         if: ${{ github.event.inputs.skip_checks != 'true' }}
@@ -109,7 +109,7 @@ jobs:
         if: ${{ github.event.inputs.skip_checks != 'true' }}
         uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: '1.22'
 
       - name: Run golangci-lint
         if: ${{ github.event.inputs.skip_checks != 'true' }}

diff --git a/.github/workflows/reusable-e2e.yml b/.github/workflows/reusable-e2e.yml
@@ -101,7 +101,7 @@ jobs:
           path: /tmp/logs.txt
 
       - name: Notify Slack on Failure
-        if: failure() && (github.event_name == 'push' && github.ref == 'refs/heads/develop') || github.event_name == 'schedule'
+        if: failure() && ((github.event_name == 'push' && github.ref == 'refs/heads/develop') || github.event_name == 'schedule')
         uses: 8398a7/action-slack@v3
         with:
           status: ${{ job.status }}

diff --git a/.github/workflows/sast-linters.yml b/.github/workflows/sast-linters.yml
@@ -25,33 +25,10 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: '1.20'
-
       - name: Run Gosec Security Scanner
-        uses: securego/[email protected]
-        with:
-          args: ./...
-
-  gosec-cosmos:
-    runs-on: ubuntu-22.04
-    env:
-      GO111MODULE: on
-    steps:
-      - name: Checkout Source
-        uses: actions/checkout@v4
+        uses: zeta-chain/[email protected]
         with:
-          fetch-depth: 0
-
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: '1.20'
-
-      - name: Run Cosmos Gosec Security Scanner
-        run: make lint-cosmos-gosec
+          args: -exclude-generated -exclude-dir testutil ./...
 
   lint:
     runs-on: ubuntu-22.04
@@ -67,7 +44,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: '1.20'
+          go-version: '1.22'
 
       - name: Run golangci-lint
         uses: golangci/golangci-lint-action@v6

diff --git a/Dockerfile-localnet b/Dockerfile-localnet
@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.7-labs
-FROM golang:1.20.14-bookworm AS base-build
+FROM golang:1.22.5-bookworm AS base-build
 
 ENV GOPATH /go
 ENV GOOS=linux
@@ -22,10 +22,10 @@ COPY --exclude=*.sh --exclude=*.md --exclude=*.yml . .
 RUN --mount=type=cache,target="/root/.cache/go-build" make install
 RUN --mount=type=cache,target="/root/.cache/go-build" make install-zetae2e
 
-FROM golang:1.20.14-bookworm AS cosmovisor-build
+FROM golang:1.22.5-bookworm AS cosmovisor-build
 RUN go install cosmossdk.io/tools/cosmovisor/cmd/[email protected]
 
-FROM golang:1.20.14-bookworm AS base-runtime
+FROM golang:1.22.5-bookworm AS base-runtime
 
 RUN apt update && \
     apt install -yq jq yq curl tmux python3 openssh-server iputils-ping iproute2 bind9-host && \
@@ -59,13 +59,9 @@ FROM base-runtime AS latest-runtime
 COPY --from=cosmovisor-build /go/bin/cosmovisor /usr/local/bin
 COPY --from=latest-build /go/bin/zetacored /go/bin/zetaclientd /go/bin/zetaclientd-supervisor /go/bin/zetae2e /usr/local/bin
 
-# optional old version build. This old build is used as the genesis version in the upgrade tests.
-# use --target latest-runtime to skip
-# 
-# TODO: just download binaries from github release now that we're using glibc
-# we can't do this right now since we do not have a v16 release candidate
-# https://github.com/zeta-chain/node/issues/2179
-FROM base-build as old-build
+# Optional old version build (from source). This old build is used as the genesis version in the upgrade tests. 
+# Use --target latest-runtime to skip.
+FROM base-build as old-build-source
 
 ARG OLD_VERSION
 RUN git clone https://github.com/zeta-chain/node.git
@@ -74,8 +70,22 @@ RUN cd node && git fetch
 RUN cd node && git checkout ${OLD_VERSION}
 RUN cd node && make install
 
+FROM base-runtime AS old-runtime-source
+
+COPY --from=cosmovisor-build /go/bin/cosmovisor /usr/local/bin
+COPY --from=old-build-source /go/bin/zetacored /go/bin/zetaclientd /usr/local/bin
+COPY --from=latest-build /go/bin/zetaclientd-supervisor /usr/local/bin
+
+# Optional old version build (from binary).
+# Use --target latest-runtime to skip.
 FROM base-runtime AS old-runtime
 
+ARG OLD_VERSION
+ARG BUILDARCH
+
 COPY --from=cosmovisor-build /go/bin/cosmovisor /usr/local/bin
-COPY --from=old-build /go/bin/zetacored /go/bin/zetaclientd /usr/local/bin
-COPY --from=latest-build /go/bin/zetaclientd-supervisor /usr/local/bin
+COPY --from=latest-build /go/bin/zetaclientd-supervisor /usr/local/bin
+RUN curl -Lo /usr/local/bin/zetacored ${OLD_VERSION}/zetacored-linux-${BUILDARCH} && \
+    chmod 755 /usr/local/bin/zetacored && \
+    curl -Lo /usr/local/bin/zetaclientd ${OLD_VERSION}/zetaclientd-linux-${BUILDARCH} && \
+    chmod 755 /usr/local/bin/zetaclientd
diff --git a/Makefile b/Makefile
@@ -139,8 +139,8 @@ lint-pre:
 lint: lint-pre
 	@golangci-lint run
 
-lint-cosmos-gosec:
-	@bash ./scripts/cosmos-gosec.sh
+lint-gosec:
+	@bash ./scripts/gosec.sh
 
 gosec:
 	gosec  -exclude-dir=localnet ./...
@@ -263,12 +263,19 @@ start-tss-migration-test: zetanode
 ###                         Upgrade Tests              						###
 ###############################################################################
 
-
+# build from source only if requested
+ifdef UPGRADE_TEST_FROM_SOURCE
 zetanode-upgrade: zetanode
-	@echo "Building zetanode-upgrade"
-	$(DOCKER) build -t zetanode:old -f Dockerfile-localnet --target old-runtime --build-arg OLD_VERSION='release/v17' .
-	$(DOCKER) build -t orchestrator -f contrib/localnet/orchestrator/Dockerfile.fastbuild .
+	@echo "Building zetanode-upgrade from source"
+	$(DOCKER) build -t zetanode:old -f Dockerfile-localnet --target old-runtime-source --build-arg OLD_VERSION='release/v17' .
+.PHONY: zetanode-upgrade
+else
+zetanode-upgrade: zetanode
+	@echo "Building zetanode-upgrade from binaries"
+	$(DOCKER) build -t zetanode:old -f Dockerfile-localnet --target old-runtime --build-arg OLD_VERSION='https://github.com/zeta-chain/ci-testing-node/releases/download/v17.0.1-internal' .
 .PHONY: zetanode-upgrade
+endif
+
 
 start-upgrade-test: zetanode-upgrade
 	@echo "--> Starting upgrade test"
@@ -282,6 +289,14 @@ start-upgrade-test-light: zetanode-upgrade
 	export UPGRADE_HEIGHT=90 && \
 	cd contrib/localnet/ && $(DOCKER) compose --profile upgrade -f docker-compose.yml -f docker-compose-upgrade.yml up -d
 
+
+start-upgrade-test-admin: zetanode-upgrade
+	@echo "--> Starting admin upgrade test"
+	export LOCALNET_MODE=upgrade && \
+	export UPGRADE_HEIGHT=90 && \
+	export E2E_ARGS="--skip-regular --test-admin" && \
+	cd contrib/localnet/ && $(DOCKER) compose --profile upgrade -f docker-compose.yml -f docker-compose-upgrade.yml up -d
+
 start-upgrade-import-mainnet-test: zetanode-upgrade
 	@echo "--> Starting import-data upgrade test"
 	export LOCALNET_MODE=upgrade && \