[WIP] k8s workflow tests

.github/workflows/.gitignore

@@ -0,0 +1 @@
+terraform

.github/workflows/agglayer-regression-tests.yml

@@ -0,0 +1,95 @@
+name: Regression Tests - Agglayer
+'on':
+  workflow_dispatch:
+    inputs:
+      zkevm_agglayer_commit_id:
+        description: 0xPolygon/agglayer (commit id)
+        required: true
+      bake_time:
+        description: bake time (minutes)
+        required: false
+        default: 30
+  push:
+    branches:
+      - main
+    inputs:
+      zkevm_agglayer_commit_id:
+        description: 0xPolygon/agglayer commit id
+        required: false
+        default: '${{ github.event.before }}' # Set default value to commit ID being merged to main
+      bake_time:
+        description: bake time (minutes)
+        required: false
+        default: 20
+jobs:
+  deploy_devnet:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          ref: '${{ github.event.before }}'
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v1
+      - name: Clone and build agglayer image locally
+        run: |
+          git clone https://github.com/0xPolygon/agglayer.git
+          cd agglayer
+          git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
+          docker compose -f docker/docker-compose.yaml build --no-cache agglayer
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+      - name: Clone internal kurtosis-cdk repo
+        run: |
+          git clone https://github.com/0xPolygon/kurtosis-cdk.git
+          cd kurtosis-cdk
+          git checkout dan/jit_containers
+      - name: Install kurtosis
+        run: |
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt update
+          sudo apt install kurtosis-cli
+      - name: Run kurtosis agent in background
+        run: |
+          kurtosis gateway &  # Run cmd in background
+      - name: Deploy CDK devnet on local github runner
+        run: |
+          cd kurtosis-cdk
+          kurtosis engine restart
+          kurtosis run --enclave cdk-v1 --args-file params.yml .
+      - name: Monitor and report any potential regressions to CI logs
+        run: |
+          bake_time="${{ github.event.inputs.bake_time }}"
+          end_minute=$(( $(date +'%M') + bake_time))
+
+          export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
+          INITIAL_STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null)
+          incremented=false
+
+          while [ $(date +'%M') -lt $end_minute ]; do
+            # Attempt to connect to the service
+            if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
+              echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
+
+              # Check if STATUS has incremented
+              if [ "$STATUS" != "$INITIAL_STATUS" ]; then
+                incremented=true
+                echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
+                exit 0
+              fi
+            else
+              echo "Failed to connect, waiting and retrying..."
+              sleep 60
+              continue
+            fi
+            sleep 60
+          done
+
+          if ! $incremented; then
+            echo "ZKEVM_VERIFIED_BATCH_NUMBER did not increment. This may indicate chain experienced a regression. Please investigate."
+            exit 1
+          fi
+      - name: Finally, remove all devnet resources locally
+        run: |
+          cd kurtosis-cdk
+          kurtosis clean -a

.github/workflows/cdk-kurtosis-gke-devnet.yml

@@ -0,0 +1,79 @@
+name: Regression Detector - Ephemeral CDK Devnet
+
+jobs:
+  deploy_devnet:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.before }}
+
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v1
+
+    - name: Setup Google Cloud SDK
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        version: '>= 363.0.0'
+
+    - uses: 'google-github-actions/auth@v2'
+      with:
+        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+        project_id: 'prj-polygonlabs-devtools-dev'
+
+    - name: Install kubectl and gcloud kubectl auth plugin
+      run: |
+        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
+        && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && gcloud components install gke-gcloud-auth-plugin
+
+    - name: Install tomlq for config file preparation
+      run: |
+        pip3 install tomlq
+
+    - name: Clone private kurtosis-cdk repo
+      run: |
+        eval `ssh-agent -s`
+        ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
+        git clone -b dan/contracts_running_state [email protected]:0xPolygon/kurtosis-cdk.git
+
+## add necessary manipulation steps here...
+    - name: Install kurtosis
+      run: |
+        echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+        sudo apt update
+        sudo apt install kurtosis-cli
+
+    - name: Setup necessary gke custom kurtosis netorking configs
+      run: |        
+        export KURTOSIS_CONFIG_PATH="$(kurtosis config path)"
+        cp /home/runner/work/bor/bor/kurtosis-config.yml $KURTOSIS_CONFIG_PATH
+
+    - name: Set kurtosis cloud env
+      run: |       
+        mkdir -p ~/.kube
+        echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/config     
+        export KUBECONFIG=~/.kube/config && kurtosis cluster set cloud
+
+    - name: Run kurtosis agent in background
+      run: |  
+        kurtosis gateway &  # Run cmd in background
+        sleep 10
+
+    - name: REMOVE, temp for CI testing remove all devnet resources
+      run: |
+        cd kurtosis-cdk
+        kurtosis clean -a
+        sleep 180
+
+    - name: Deploy CDK devnet to isolated GKE namespace
+      run: |
+        cd kurtosis-cdk
+        kurtosis engine restart
+        kurtosis run --enclave cdk-v1 --args-file params.yml .
+        sleep 600
+
+    - name: Finally, remove all devnet resources
+      run: |
+        cd kurtosis-cdk
+        kurtosis clean -a

.github/workflows/cdk-superuser-regression-tests.yml

@@ -0,0 +1,110 @@
+name: Regression Tests - Superuser
+'on':
+  workflow_dispatch:
+    inputs:
+      zkevm_agglayer_commit_id:
+        description: 0xPolygon/agglayer (commit id)
+        required: true
+      zkevm_bridge_service_commit_id:
+        description: 0xPolygonHermez/zkevm-bridge-service (commit id)
+        required: true
+      zkevm_bridge_ui_commit_id:
+        description: 0xPolygonHermez/zkevm-bridge-ui (commit id)
+        required: true
+      zkevm_dac_commit_id:
+        description: 0xPolygon/cdk-data-availability (commit id)
+        required: true
+      zkevm_node_commit_id:
+        description: 0xPolygon/cdk-validium-node (commit id)
+        required: true
+      bake_time:
+        description: bake time (minutes)
+        required: false
+        default: 30
+jobs:
+  deploy_devnet:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v1
+      - name: Clone and build agglayer
+        run: |
+          git clone https://github.com/0xPolygon/agglayer.git
+          cd agglayer
+          git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
+          docker compose -f docker/docker-compose.yaml build --no-cache agglayer
+      - name: Clone and build zkevm-bridge-service
+        run: |
+          git clone https://github.com/0xPolygonHermez/zkevm-bridge-service.git
+          cd zkevm-bridge-service
+          git checkout "${{ github.event.inputs.zkevm_bridge_service_commit_id }}"
+          docker build -t zkevm-bridge-service:local -f ./Dockerfile .
+      - name: Clone and build zkevm-bridge-ui
+        run: |
+          git clone https://github.com/0xPolygonHermez/zkevm-bridge-ui.git
+          cd zkevm-bridge-ui
+          git checkout "${{ github.event.inputs.zkevm_bridge_ui_commit_id }}"
+          docker build -t zkevm-bridge-ui:local -f ./Dockerfile .
+      - name: Clone and build cdk-data-availability
+        run: |
+          git clone https://github.com/0xPolygon/cdk-data-availability.git
+          cd cdk-data-availability
+          git checkout "${{ github.event.inputs.zkevm_dac_commit_id }}"
+          docker build -t cdk-data-availability:local -f ./Dockerfile .    
+      - name: Clone and build cdk-validium-node
+        run: |
+          git clone https://github.com/0xPolygon/cdk-validium-node.git
+          cd cdk-validium-node
+          git checkout "${{ github.event.inputs.zkevm_node_commit_id }}"
+          docker build -t cdk-validium-node:local -f ./Dockerfile .
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+      - name: Clone internal kurtosis-cdk repo
+        run: |
+          git clone https://github.com/0xPolygon/kurtosis-cdk.git
+          cd kurtosis-cdk
+          git checkout dan/jit_containers_superusers
+      - name: Install kurtosis
+        run: |
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt update
+          sudo apt install kurtosis-cli
+          kurtosis analytics disable
+      - name: Deploy CDK devnet on local github runner
+        run: |
+          cd kurtosis-cdk
+          kurtosis run --enclave cdk-v1 --args-file params.yml .
+      - name: Monitor and report any potential regressions to CI logs
+        run: |
+          bake_time="${{ github.event.inputs.bake_time }}"
+          end_minute=$(( $(date +'%M') + bake_time))
+
+          export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
+          INITIAL_STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null)
+          incremented=false
+
+          while [ $(date +'%M') -lt $end_minute ]; do
+            # Attempt to connect to the service
+            if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
+              echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
+
+              # Check if STATUS has incremented
+              if [ "$STATUS" != "$INITIAL_STATUS" ]; then
+                incremented=true
+                echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
+                exit 0
+              fi
+            else
+              echo "Failed to connect, waiting and retrying..."
+              sleep 60
+              continue
+            fi
+            sleep 60
+          done
+
+          if ! $incremented; then
+            echo "ZKEVM_VERIFIED_BATCH_NUMBER did not increment. This may indicate chain experienced a regression. Please investigate."
+            exit 1
+          fi

.github/workflows/kind-config.yaml.sample

@@ -0,0 +1,18 @@
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+networking:
+  apiServerAddress: "0.0.0.0"
+
+kubeadmConfigPatchesJSON6902:
+  - group: kubeadm.k8s.io
+    version: v1beta2
+    kind: ClusterConfiguration
+    patch: |
+      - op: add
+        path: /apiServer/certSANs/-
+        value: docker
+nodes:
+- role: control-plane
+  extraPortMappings:
+  - containerPort: 11111
+    hostPort: 11111

.github/workflows/make-k8s-images.yml

@@ -0,0 +1,39 @@
+name: Image Sync
+
+on:
+  schedule:
+    - cron: '*/30 * * * *'
+
+jobs:
+  sync-images:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v2
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v1
+    - uses: 'google-github-actions/auth@v2'
+      with:
+        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+    - name: 'Set up Cloud SDK'
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        version: '>= 363.0.0'
+    - name: Fetch repos, rebuild for k8s, push to internal artifact registry
+      run: |
+        gcloud auth configure-docker europe-west2-docker.pkg.dev -q
+        while IFS= read -r image_path; do
+          latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/" | jq -r '.results[].name | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))' | sort -V | tail -n 1)
+          if [ -n "$latest_tag" ]; then
+            # Check if the image version already exists in GCP Container Registry
+            if ! gcloud container images list-tags europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
+              docker pull "${image_path}:${latest_tag}"
+              docker tag "${image_path}:${latest_tag}" "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path}:${latest_tag}"
+              sudo -u $USER docker push "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path}:${latest_tag}"
+              # also tag image as 'latest' for internal tracking
+              sudo -u $USER docker push "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path}:latest"
+            else
+              echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
+            fi
+          fi
+        done < .github/workflows/repo-list.txt.sample

.github/workflows/repo-list.txt.sample

@@ -0,0 +1,3 @@
+0xpolygon/bor
+0xpolygon/heimdall
+thorax/erigon

kurtosis-config.yml

@@ -0,0 +1,17 @@
+config-version: 2
+should-send-metrics: true
+kurtosis-clusters:
+  docker:
+    type: "docker"
+  minikube:
+    type: "kubernetes"
+    config:
+      kubernetes-cluster-name: "minikube"
+      storage-class: "standard"
+      enclave-size-in-megabytes: 400
+  cloud:
+    type: "kubernetes"
+    config:
+      kubernetes-cluster-name: "gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1"
+      storage-class: "standard"
+      enclave-size-in-megabytes: 400