From e40d102fc6428b839268f5ee327c4642c8c0a6e1 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 15:51:21 -0600
Subject: [PATCH 001/144] add files

---
 .github/workflows/k8s-regression-tester.yml | 66 +++++++++++++++++++++
 .github/workflows/kind-config.yaml          | 18 ++++++
 2 files changed, 84 insertions(+)
 create mode 100644 .github/workflows/k8s-regression-tester.yml
 create mode 100644 .github/workflows/kind-config.yaml

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
new file mode 100644
index 0000000000..4686b4eaaa
--- /dev/null
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -0,0 +1,66 @@
+name: Deploy Devnet
+
+on:
+  pull_request:
+  push:
+    branches:
+      - '*'
+
+jobs:
+  deploy_devnet:
+    runs-on: ubuntu-latest
+    
+    services:
+      kind:
+        image: kindest/node:v1.22.2
+        ports:
+          - 11111:11111
+        options: --config .github/workflows/kind-config.yaml
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.before }}
+
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: 1.17
+        
+    - name: Install Python 3.9.6
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9.6
+
+    - name: Install kubectl
+      run: |
+        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
+        && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \
+        && kubectl version --client
+
+    - name: Install kind
+      run: go install sigs.k8s.io/kind@v0.20.0
+
+    - name: Create local k8s cluster
+      run: |
+        kind create cluster --name regression-tester --config kind-config.yaml
+        kubectl cluster-info --context kind-regression-tester
+
+    - name: Clone and build devnet images
+      run: |
+        git clone https://github.com/maticnetwork/polygon-devnets
+        cd polygon-devnets/docker/pos
+        make all DEV=true K8S_ENV=true K8S_NS=test
+        
+      env:
+        PRIVATE_REPO_TOKEN: ${{ secrets.CI_PAT }}        
+
+    - name: Load images into cluster
+      run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
+
+    - name: Deploy devnet
+      run: kubectl apply -k . --context kind-regression-tester
+
+    - name: Check deployed resources
+      run: kubectl get pods -n test
diff --git a/.github/workflows/kind-config.yaml b/.github/workflows/kind-config.yaml
new file mode 100644
index 0000000000..6b05f312e4
--- /dev/null
+++ b/.github/workflows/kind-config.yaml
@@ -0,0 +1,18 @@
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+networking:
+  apiServerAddress: "0.0.0.0"
+
+kubeadmConfigPatchesJSON6902:
+  - group: kubeadm.k8s.io
+    version: v1beta2
+    kind: ClusterConfiguration
+    patch: |
+      - op: add
+        path: /apiServer/certSANs/-
+        value: docker
+nodes:
+- role: control-plane
+  extraPortMappings:
+  - containerPort: 11111
+    hostPort: 11111
\ No newline at end of file

From 1239a46ca024a0947d8c15f3f1adfc0b19ecadd1 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 16:08:38 -0600
Subject: [PATCH 002/144] test workflow

---
 .github/workflows/k8s-regression-tester.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 4686b4eaaa..805739c6f3 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -1,7 +1,6 @@
 name: Deploy Devnet
 
 on:
-  pull_request:
   push:
     branches:
       - '*'

From c6bdc0c1af4896e938e769faab516f8605554df0 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 16:11:50 -0600
Subject: [PATCH 003/144] test

---
 .github/workflows/k8s-regression-tester.yml                     | 2 +-
 .github/workflows/{kind-config.yaml => kind-config.yaml.sample} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename .github/workflows/{kind-config.yaml => kind-config.yaml.sample} (100%)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 805739c6f3..309505cf5c 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -14,7 +14,7 @@ jobs:
         image: kindest/node:v1.22.2
         ports:
           - 11111:11111
-        options: --config .github/workflows/kind-config.yaml
+        options: --config .github/workflows/kind-config.yaml.sample
 
     steps:
     - name: Checkout Repository
diff --git a/.github/workflows/kind-config.yaml b/.github/workflows/kind-config.yaml.sample
similarity index 100%
rename from .github/workflows/kind-config.yaml
rename to .github/workflows/kind-config.yaml.sample

From 52f7a450ee326bf2de6990ebbd69c1c61f7d7345 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 16:13:40 -0600
Subject: [PATCH 004/144] testing

---
 .github/workflows/k8s-regression-tester.yml | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 309505cf5c..5ab35eefc2 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -1,65 +1,49 @@
 name: Deploy Devnet
-
-on:
-  push:
-    branches:
-      - '*'
+on: [ push, pull_request ]
 
 jobs:
   deploy_devnet:
     runs-on: ubuntu-latest
-    
     services:
       kind:
         image: kindest/node:v1.22.2
         ports:
           - 11111:11111
         options: --config .github/workflows/kind-config.yaml.sample
-
     steps:
     - name: Checkout Repository
       uses: actions/checkout@v2
       with:
         ref: ${{ github.event.before }}
-
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
         go-version: 1.17
-        
     - name: Install Python 3.9.6
       uses: actions/setup-python@v2
       with:
         python-version: 3.9.6
-
     - name: Install kubectl
       run: |
         curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
         && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \
         && kubectl version --client
-
     - name: Install kind
       run: go install sigs.k8s.io/kind@v0.20.0
-
     - name: Create local k8s cluster
       run: |
         kind create cluster --name regression-tester --config kind-config.yaml
         kubectl cluster-info --context kind-regression-tester
-
     - name: Clone and build devnet images
       run: |
         git clone https://github.com/maticnetwork/polygon-devnets
         cd polygon-devnets/docker/pos
         make all DEV=true K8S_ENV=true K8S_NS=test
-        
       env:
         PRIVATE_REPO_TOKEN: ${{ secrets.CI_PAT }}        
-
     - name: Load images into cluster
       run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
-
     - name: Deploy devnet
       run: kubectl apply -k . --context kind-regression-tester
-
     - name: Check deployed resources
       run: kubectl get pods -n test

From bcbc450d09d9fc8ff81a5eac9d84155a2364e3fe Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 16:17:41 -0600
Subject: [PATCH 005/144] still testing

---
 .github/workflows/k8s-regression-tester.yml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 5ab35eefc2..64e87d9b71 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -4,12 +4,6 @@ on: [ push, pull_request ]
 jobs:
   deploy_devnet:
     runs-on: ubuntu-latest
-    services:
-      kind:
-        image: kindest/node:v1.22.2
-        ports:
-          - 11111:11111
-        options: --config .github/workflows/kind-config.yaml.sample
     steps:
     - name: Checkout Repository
       uses: actions/checkout@v2
@@ -32,7 +26,8 @@ jobs:
       run: go install sigs.k8s.io/kind@v0.20.0
     - name: Create local k8s cluster
       run: |
-        kind create cluster --name regression-tester --config kind-config.yaml
+        pwd
+        kind create cluster --name regression-tester --config .github/workflows/kind-config.yaml.sample
         kubectl cluster-info --context kind-regression-tester
     - name: Clone and build devnet images
       run: |

From d95fc90f9abf5e47f39da73c22d6a23978f248f1 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 16:20:48 -0600
Subject: [PATCH 006/144] keep testing

---
 .github/workflows/k8s-regression-tester.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 64e87d9b71..ea88008283 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -13,10 +13,10 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: 1.17
-    - name: Install Python 3.9.6
+    - name: Install Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.9.6
+        python-version: 3.9.6-117960
     - name: Install kubectl
       run: |
         curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \

From 458fa9e4da9769e4ca49f7e30200d4afb486a9e3 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 16:22:26 -0600
Subject: [PATCH 007/144] try again

---
 .github/workflows/k8s-regression-tester.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index ea88008283..51f03037eb 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -13,12 +13,10 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: 1.17
-    - name: Install Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.9.6-117960
     - name: Install kubectl
       run: |
+        python --version 
+        python3 --version
         curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
         && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \
         && kubectl version --client

From 90137d6d7c4ae6a6fd715ccc356298b6fe98eb25 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 16:27:00 -0600
Subject: [PATCH 008/144] test again

---
 .github/workflows/k8s-regression-tester.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 51f03037eb..3c3383da41 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -32,8 +32,10 @@ jobs:
         git clone https://github.com/maticnetwork/polygon-devnets
         cd polygon-devnets/docker/pos
         make all DEV=true K8S_ENV=true K8S_NS=test
+
       env:
-        PRIVATE_REPO_TOKEN: ${{ secrets.CI_PAT }}        
+        PRIVATE_REPO_TOKEN: ${{ secrets.CI_PAT }}
+        GIT_ASKPASS: echo 
     - name: Load images into cluster
       run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
     - name: Deploy devnet

From 4d0d0150e9f9926774bd7e53b004f7dfe13455a2 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 16:44:20 -0600
Subject: [PATCH 009/144] test again

---
 .github/workflows/k8s-regression-tester.yml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 3c3383da41..a567957df3 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -15,13 +15,15 @@ jobs:
         go-version: 1.17
     - name: Install kubectl
       run: |
-        python --version 
-        python3 --version
         curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
         && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \
         && kubectl version --client
     - name: Install kind
       run: go install sigs.k8s.io/kind@v0.20.0
+    - name: Configure SSH Agent
+      run: |
+        eval `ssh-agent -s`
+        ssh-add - <<< '${{ secrets.K8S_CI_SECRET }}'
     - name: Create local k8s cluster
       run: |
         pwd
@@ -32,10 +34,6 @@ jobs:
         git clone https://github.com/maticnetwork/polygon-devnets
         cd polygon-devnets/docker/pos
         make all DEV=true K8S_ENV=true K8S_NS=test
-
-      env:
-        PRIVATE_REPO_TOKEN: ${{ secrets.CI_PAT }}
-        GIT_ASKPASS: echo 
     - name: Load images into cluster
       run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
     - name: Deploy devnet

From 0721024ceeff8226373328507b6e91720cfa3753 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 16:46:28 -0600
Subject: [PATCH 010/144] moar tests

---
 .github/workflows/k8s-regression-tester.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index a567957df3..053f2a0f82 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -20,10 +20,6 @@ jobs:
         && kubectl version --client
     - name: Install kind
       run: go install sigs.k8s.io/kind@v0.20.0
-    - name: Configure SSH Agent
-      run: |
-        eval `ssh-agent -s`
-        ssh-add - <<< '${{ secrets.K8S_CI_SECRET }}'
     - name: Create local k8s cluster
       run: |
         pwd
@@ -31,6 +27,8 @@ jobs:
         kubectl cluster-info --context kind-regression-tester
     - name: Clone and build devnet images
       run: |
+        eval `ssh-agent -s`
+        ssh-add - <<< '${{ secrets.K8S_CI_SECRET }}'
         git clone https://github.com/maticnetwork/polygon-devnets
         cd polygon-devnets/docker/pos
         make all DEV=true K8S_ENV=true K8S_NS=test

From d112e87284d3e701123b71bcaf5c409663c2cf0f Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 17:04:26 -0600
Subject: [PATCH 011/144] keep testing

---
 .github/workflows/k8s-regression-tester.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 053f2a0f82..52fc4a60d2 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -29,6 +29,7 @@ jobs:
       run: |
         eval `ssh-agent -s`
         ssh-add - <<< '${{ secrets.K8S_CI_SECRET }}'
+        pip install -r requirements.txt
         git clone https://github.com/maticnetwork/polygon-devnets
         cd polygon-devnets/docker/pos
         make all DEV=true K8S_ENV=true K8S_NS=test

From f4d1f0dcd5f9babeae9f9d83b3cb7381a93cd25f Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 17:08:44 -0600
Subject: [PATCH 012/144] keep iterating

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 52fc4a60d2..5622c49133 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -28,7 +28,7 @@ jobs:
     - name: Clone and build devnet images
       run: |
         eval `ssh-agent -s`
-        ssh-add - <<< '${{ secrets.K8S_CI_SECRET }}'
+        ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
         pip install -r requirements.txt
         git clone https://github.com/maticnetwork/polygon-devnets
         cd polygon-devnets/docker/pos

From 5380723fdce4ac1639387776262f8e07853147c7 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 17:10:23 -0600
Subject: [PATCH 013/144] clone private repo

---
 .github/workflows/k8s-regression-tester.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 5622c49133..301bababed 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -29,7 +29,6 @@ jobs:
       run: |
         eval `ssh-agent -s`
         ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
-        pip install -r requirements.txt
         git clone https://github.com/maticnetwork/polygon-devnets
         cd polygon-devnets/docker/pos
         make all DEV=true K8S_ENV=true K8S_NS=test

From 15d007a544047e655d253a422e4c77f4f22d8502 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 17:17:15 -0600
Subject: [PATCH 014/144] clone private devnets

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 301bababed..186ccc90b9 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -25,7 +25,7 @@ jobs:
         pwd
         kind create cluster --name regression-tester --config .github/workflows/kind-config.yaml.sample
         kubectl cluster-info --context kind-regression-tester
-    - name: Clone and build devnet images
+    - name: Clone and then build devnet images
       run: |
         eval `ssh-agent -s`
         ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'

From 4ecd4ab399dcc269699f61844c9cb3dcb2f3b7ea Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 17:19:00 -0600
Subject: [PATCH 015/144] git clone ssh

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 186ccc90b9..cefb0377be 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -29,7 +29,7 @@ jobs:
       run: |
         eval `ssh-agent -s`
         ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
-        git clone https://github.com/maticnetwork/polygon-devnets
+        git clone git@github.com:maticnetwork/polygon-devnets.git
         cd polygon-devnets/docker/pos
         make all DEV=true K8S_ENV=true K8S_NS=test
     - name: Load images into cluster

From 527085483e4c55b32cfd9bb5838407bfa2471b6a Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 30 Jan 2024 17:54:03 -0600
Subject: [PATCH 016/144] install tomlq

---
 .github/workflows/k8s-regression-tester.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index cefb0377be..f95cfe564a 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -20,6 +20,11 @@ jobs:
         && kubectl version --client
     - name: Install kind
       run: go install sigs.k8s.io/kind@v0.20.0
+    - name: Install tomlq
+      run: |
+        pip3 list
+        pip3 install tomlq
+        pip3 list
     - name: Create local k8s cluster
       run: |
         pwd

From b44e0793d1b2b48b34fd0f00e5dbfa7feadfa5d6 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 09:34:24 -0600
Subject: [PATCH 017/144] free disk

---
 .github/workflows/k8s-regression-tester.yml | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index f95cfe564a..6e529cb9f7 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -22,12 +22,27 @@ jobs:
       run: go install sigs.k8s.io/kind@v0.20.0
     - name: Install tomlq
       run: |
-        pip3 list
         pip3 install tomlq
-        pip3 list
+    - name: Free github runner disk space for k8s image building
+      run: |
+        echo "Freeing up disk space on CI system"
+        echo "Listing 100 largest packages"
+        dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
+        df -h
+        echo "Removing large packages"
+        sudo apt-get remove -y '^ghc-8.*'
+        sudo apt-get remove -y '^dotnet-.*'
+        sudo apt-get remove -y '^llvm-.*'
+        sudo apt-get remove -y 'php.*'
+        sudo apt-get remove -y azure-cli google-cloud-sdk hhvm google-chrome-stable firefox powershell mono-devel
+        sudo apt-get autoremove -y
+        sudo apt-get clean
+        df -h
+        echo "Removing large directories"
+        rm -rf /usr/share/dotnet/
+        df -h
     - name: Create local k8s cluster
       run: |
-        pwd
         kind create cluster --name regression-tester --config .github/workflows/kind-config.yaml.sample
         kubectl cluster-info --context kind-regression-tester
     - name: Clone and then build devnet images

From 17bec591b66699af050be63572b9fd05b6cbd419 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 09:41:21 -0600
Subject: [PATCH 018/144] rm large pkgs

---
 .github/workflows/k8s-regression-tester.yml | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 6e529cb9f7..15032403b5 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -25,21 +25,17 @@ jobs:
         pip3 install tomlq
     - name: Free github runner disk space for k8s image building
       run: |
-        echo "Freeing up disk space on CI system"
-        echo "Listing 100 largest packages"
+        echo "Freeing up disk space on CI system..."
+        echo "List 100 largest packages"
         dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
         df -h
-        echo "Removing large packages"
-        sudo apt-get remove -y '^ghc-8.*'
-        sudo apt-get remove -y '^dotnet-.*'
-        sudo apt-get remove -y '^llvm-.*'
-        sudo apt-get remove -y 'php.*'
-        sudo apt-get remove -y azure-cli google-cloud-sdk hhvm google-chrome-stable firefox powershell mono-devel
+        sudo apt-get remove -y azure-cli google-cloud-cli google-cloud-sdk microsoft-edge-stable hhvm google-chrome-stable firefox powershell mono-devel
         sudo apt-get autoremove -y
         sudo apt-get clean
         df -h
         echo "Removing large directories"
         rm -rf /usr/share/dotnet/
+        rm -rf /opt/hostedtoolcache
         df -h
     - name: Create local k8s cluster
       run: |

From 62f826ec851b8093b45faffb47fdd1c917ee1d82 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 09:42:48 -0600
Subject: [PATCH 019/144] rm pkgs

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 15032403b5..29df7a2c90 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -29,7 +29,7 @@ jobs:
         echo "List 100 largest packages"
         dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
         df -h
-        sudo apt-get remove -y azure-cli google-cloud-cli google-cloud-sdk microsoft-edge-stable hhvm google-chrome-stable firefox powershell mono-devel
+        sudo apt-get remove -y azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox
         sudo apt-get autoremove -y
         sudo apt-get clean
         df -h

From 756fe37958c3c503f90b0265c5f15c70fc5d243f Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 09:51:21 -0600
Subject: [PATCH 020/144] free disk'

---
 .github/workflows/k8s-regression-tester.yml | 26 ++++++++++++---------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 29df7a2c90..9402f00fb2 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -5,6 +5,10 @@ jobs:
   deploy_devnet:
     runs-on: ubuntu-latest
     steps:
+    - name: Maximize build space
+      uses: AdityaGarg8/remove-unwanted-software@v2
+      with:
+        remove-android: 'true'
     - name: Checkout Repository
       uses: actions/checkout@v2
       with:
@@ -25,17 +29,17 @@ jobs:
         pip3 install tomlq
     - name: Free github runner disk space for k8s image building
       run: |
-        echo "Freeing up disk space on CI system..."
-        echo "List 100 largest packages"
-        dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
-        df -h
-        sudo apt-get remove -y azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox
-        sudo apt-get autoremove -y
-        sudo apt-get clean
-        df -h
-        echo "Removing large directories"
-        rm -rf /usr/share/dotnet/
-        rm -rf /opt/hostedtoolcache
+        # echo "Freeing up disk space on CI system..."
+        # echo "List 100 largest packages"
+        # dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
+        # df -h
+        # sudo apt-get remove -y azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox
+        # sudo apt-get autoremove -y
+        # sudo apt-get clean
+        # df -h
+        # echo "Removing large directories"
+        # rm -rf /usr/share/dotnet/
+        # rm -rf /opt/hostedtoolcache
         df -h
     - name: Create local k8s cluster
       run: |

From b6b1522169174b088ffc1c50fb214be55ef86eec Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 09:54:21 -0600
Subject: [PATCH 021/144] free more disk

---
 .github/workflows/k8s-regression-tester.yml | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 9402f00fb2..8b174c8f51 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -9,6 +9,10 @@ jobs:
       uses: AdityaGarg8/remove-unwanted-software@v2
       with:
         remove-android: 'true'
+        remove-dotnet: 'true'
+        remove-haskell: 'true'
+        remove-codeql: 'true'
+        remove-docker-images: 'true'
     - name: Checkout Repository
       uses: actions/checkout@v2
       with:
@@ -29,17 +33,9 @@ jobs:
         pip3 install tomlq
     - name: Free github runner disk space for k8s image building
       run: |
-        # echo "Freeing up disk space on CI system..."
-        # echo "List 100 largest packages"
-        # dpkg-query -Wf '${Installed-Size}\t${Package}\n' | sort -n | tail -n 100
-        # df -h
-        # sudo apt-get remove -y azure-cli google-cloud-cli microsoft-edge-stable google-chrome-stable firefox
-        # sudo apt-get autoremove -y
-        # sudo apt-get clean
-        # df -h
-        # echo "Removing large directories"
-        # rm -rf /usr/share/dotnet/
-        # rm -rf /opt/hostedtoolcache
+        df -h
+        rm -rf /usr/share/dotnet/
+        rm -rf /opt/hostedtoolcache
         df -h
     - name: Create local k8s cluster
       run: |

From bfc1f4ca691f7eb4ad990d392442f67167ffd989 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 11:50:34 -0600
Subject: [PATCH 022/144] cursor to correct dir

---
 .github/workflows/k8s-regression-tester.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 8b174c8f51..d63f416671 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -48,9 +48,13 @@ jobs:
         git clone git@github.com:maticnetwork/polygon-devnets.git
         cd polygon-devnets/docker/pos
         make all DEV=true K8S_ENV=true K8S_NS=test
+        cd polygon-devnets/docker/pos/k8s
+        ls
     - name: Load images into cluster
       run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
     - name: Deploy devnet
-      run: kubectl apply -k . --context kind-regression-tester
+      run: |
+        cd polygon-devnets/docker/pos/k8s
+        kubectl apply -k . --context kind-regression-tester
     - name: Check deployed resources
       run: kubectl get pods -n test

From ca6366bfb6d0c8f29019190c756d2892de423b1c Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 14:21:13 -0600
Subject: [PATCH 023/144] test

---
 .github/workflows/k8s-regression-tester.yml | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index d63f416671..28e4f1c006 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -47,14 +47,16 @@ jobs:
         ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
         git clone git@github.com:maticnetwork/polygon-devnets.git
         cd polygon-devnets/docker/pos
-        make all DEV=true K8S_ENV=true K8S_NS=test
         cd polygon-devnets/docker/pos/k8s
         ls
-    - name: Load images into cluster
-      run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
-    - name: Deploy devnet
-      run: |
-        cd polygon-devnets/docker/pos/k8s
-        kubectl apply -k . --context kind-regression-tester
-    - name: Check deployed resources
-      run: kubectl get pods -n test
+    #     make all DEV=true K8S_ENV=true K8S_NS=test
+    #     cd polygon-devnets/docker/pos/k8s
+    #     ls
+    # - name: Load images into cluster
+    #   run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
+    # - name: Deploy devnet
+    #   run: |
+    #     cd polygon-devnets/docker/pos/k8s
+    #     kubectl apply -k . --context kind-regression-tester
+    # - name: Check deployed resources
+    #   run: kubectl get pods -n test

From c6ac0bc8f4bf41c8233095994387ab9e05c2f4df Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 14:25:00 -0600
Subject: [PATCH 024/144] testing

---
 .github/workflows/k8s-regression-tester.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 28e4f1c006..f83d89a182 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -47,7 +47,8 @@ jobs:
         ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
         git clone git@github.com:maticnetwork/polygon-devnets.git
         cd polygon-devnets/docker/pos
-        cd polygon-devnets/docker/pos/k8s
+        ls
+        cd k8s
         ls
     #     make all DEV=true K8S_ENV=true K8S_NS=test
     #     cd polygon-devnets/docker/pos/k8s

From eac3a679e1bb9f43d1d219a51c78aa8ccae15f2f Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 14:28:16 -0600
Subject: [PATCH 025/144] tests

---
 .github/workflows/k8s-regression-tester.yml | 21 +++++++++------------
 1 file changed, 9 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index f83d89a182..04d3fe214a 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -47,17 +47,14 @@ jobs:
         ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
         git clone git@github.com:maticnetwork/polygon-devnets.git
         cd polygon-devnets/docker/pos
-        ls
+        # make all DEV=true K8S_ENV=true K8S_NS=test
         cd k8s
+        pwd
+    - name: Load images into cluster
+      run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
+    - name: Deploy devnet
+      run: |
         ls
-    #     make all DEV=true K8S_ENV=true K8S_NS=test
-    #     cd polygon-devnets/docker/pos/k8s
-    #     ls
-    # - name: Load images into cluster
-    #   run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
-    # - name: Deploy devnet
-    #   run: |
-    #     cd polygon-devnets/docker/pos/k8s
-    #     kubectl apply -k . --context kind-regression-tester
-    # - name: Check deployed resources
-    #   run: kubectl get pods -n test
+        kubectl apply -k . --context kind-regression-tester
+    - name: Check deployed resources
+      run: kubectl get pods -n test

From ecf5ee980b2070b39512510d34306fe98b7c6cdc Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 14:33:13 -0600
Subject: [PATCH 026/144] testingg

---
 .github/workflows/k8s-regression-tester.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 04d3fe214a..95bcd5551b 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -50,11 +50,11 @@ jobs:
         # make all DEV=true K8S_ENV=true K8S_NS=test
         cd k8s
         pwd
-    - name: Load images into cluster
-      run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
+    # - name: Load images into cluster
+    #   run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
     - name: Deploy devnet
       run: |
-        ls
+        pwd
         kubectl apply -k . --context kind-regression-tester
     - name: Check deployed resources
       run: kubectl get pods -n test

From 1fb1d2cd82478da47465fb459c2bae220a6bb73f Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 14:37:48 -0600
Subject: [PATCH 027/144] testinggg

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 95bcd5551b..e75299284d 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -48,12 +48,12 @@ jobs:
         git clone git@github.com:maticnetwork/polygon-devnets.git
         cd polygon-devnets/docker/pos
         # make all DEV=true K8S_ENV=true K8S_NS=test
-        cd k8s
         pwd
     # - name: Load images into cluster
     #   run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
     - name: Deploy devnet
       run: |
+        cd polygon-devnets/docker/pos/k8s
         pwd
         kubectl apply -k . --context kind-regression-tester
     - name: Check deployed resources

From a6e6a1af84e74d01307c07b8a16b940e516f1b25 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 14:40:49 -0600
Subject: [PATCH 028/144] keep testing

---
 .github/workflows/k8s-regression-tester.yml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index e75299284d..5f19066d1b 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -47,14 +47,12 @@ jobs:
         ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
         git clone git@github.com:maticnetwork/polygon-devnets.git
         cd polygon-devnets/docker/pos
-        # make all DEV=true K8S_ENV=true K8S_NS=test
-        pwd
-    # - name: Load images into cluster
-    #   run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
+        make all DEV=true K8S_ENV=true K8S_NS=test
+    - name: Load images into cluster
+      run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
     - name: Deploy devnet
       run: |
         cd polygon-devnets/docker/pos/k8s
-        pwd
         kubectl apply -k . --context kind-regression-tester
     - name: Check deployed resources
       run: kubectl get pods -n test

From 822766e98d3be04f113d2a432fcb667d858630f5 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 31 Jan 2024 15:26:51 -0600
Subject: [PATCH 029/144] loop checking pod status

---
 .github/workflows/k8s-regression-tester.yml | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 5f19066d1b..02d2faf78b 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -5,7 +5,7 @@ jobs:
   deploy_devnet:
     runs-on: ubuntu-latest
     steps:
-    - name: Maximize build space
+    - name: Maximize runner build space for k8s images
       uses: AdityaGarg8/remove-unwanted-software@v2
       with:
         remove-android: 'true'
@@ -13,7 +13,7 @@ jobs:
         remove-haskell: 'true'
         remove-codeql: 'true'
         remove-docker-images: 'true'
-    - name: Checkout Repository
+    - name: Checkout curent repository
       uses: actions/checkout@v2
       with:
         ref: ${{ github.event.before }}
@@ -33,10 +33,8 @@ jobs:
         pip3 install tomlq
     - name: Free github runner disk space for k8s image building
       run: |
-        df -h
         rm -rf /usr/share/dotnet/
         rm -rf /opt/hostedtoolcache
-        df -h
     - name: Create local k8s cluster
       run: |
         kind create cluster --name regression-tester --config .github/workflows/kind-config.yaml.sample
@@ -55,4 +53,12 @@ jobs:
         cd polygon-devnets/docker/pos/k8s
         kubectl apply -k . --context kind-regression-tester
     - name: Check deployed resources
-      run: kubectl get pods -n test
+      run: |
+        current_minute=$(date +'%M')
+        end_minute=$((current_minute + 60)) # Run for the next hour as test
+
+        while [ $current_minute -lt $end_minute ]; do
+          kubectl get pods -n test
+          sleep 600 # Sleep for 10 minutes
+          current_minute=$(date +'%M')
+        done

From 218393b074e7ca1a33ca74d7633ac527a926c683 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 2 Feb 2024 15:23:23 -0600
Subject: [PATCH 030/144] test gcloud auth

---
 .github/workflows/.gitignore                |  1 +
 .github/workflows/k8s-regression-tester.yml | 57 +++++++++++----------
 2 files changed, 31 insertions(+), 27 deletions(-)
 create mode 100644 .github/workflows/.gitignore

diff --git a/.github/workflows/.gitignore b/.github/workflows/.gitignore
new file mode 100644
index 0000000000..5ac7c5753d
--- /dev/null
+++ b/.github/workflows/.gitignore
@@ -0,0 +1 @@
+terraform
diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 02d2faf78b..d42d2447d1 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -35,30 +35,33 @@ jobs:
       run: |
         rm -rf /usr/share/dotnet/
         rm -rf /opt/hostedtoolcache
-    - name: Create local k8s cluster
-      run: |
-        kind create cluster --name regression-tester --config .github/workflows/kind-config.yaml.sample
-        kubectl cluster-info --context kind-regression-tester
-    - name: Clone and then build devnet images
-      run: |
-        eval `ssh-agent -s`
-        ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
-        git clone git@github.com:maticnetwork/polygon-devnets.git
-        cd polygon-devnets/docker/pos
-        make all DEV=true K8S_ENV=true K8S_NS=test
-    - name: Load images into cluster
-      run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
-    - name: Deploy devnet
-      run: |
-        cd polygon-devnets/docker/pos/k8s
-        kubectl apply -k . --context kind-regression-tester
-    - name: Check deployed resources
-      run: |
-        current_minute=$(date +'%M')
-        end_minute=$((current_minute + 60)) # Run for the next hour as test
-
-        while [ $current_minute -lt $end_minute ]; do
-          kubectl get pods -n test
-          sleep 600 # Sleep for 10 minutes
-          current_minute=$(date +'%M')
-        done
+    - uses: 'google-github-actions/auth@v2'
+      with:
+        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+    # - name: Create local k8s cluster
+    #   run: |
+    #     kind create cluster --name regression-tester --config .github/workflows/kind-config.yaml.sample
+    #     kubectl cluster-info --context kind-regression-tester
+    # - name: Clone and then build devnet images
+    #   run: |
+    #     eval `ssh-agent -s`
+    #     ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
+    #     git clone git@github.com:maticnetwork/polygon-devnets.git
+    #     cd polygon-devnets/docker/pos
+    #     make all DEV=true K8S_ENV=true K8S_NS=test
+    # - name: Load images into cluster
+    #   run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
+    # - name: Deploy devnet
+    #   run: |
+    #     cd polygon-devnets/docker/pos/k8s
+    #     kubectl apply -k . --context kind-regression-tester
+    # - name: Check deployed resources
+    #   run: |
+    #     current_minute=$(date +'%M')
+    #     end_minute=$((current_minute + 60)) # Run for the next hour as test
+    # 
+    #     while [ $current_minute -lt $end_minute ]; do
+    #       kubectl get pods -n test
+    #       sleep 600 # Sleep for 10 minutes
+    #       current_minute=$(date +'%M')
+    #     done

From 4aac181c251a1c7f8cfd107428641b56955bad13 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 5 Feb 2024 15:09:39 -0600
Subject: [PATCH 031/144] testing

---
 .github/workflows/k8s-regression-tester.yml |  3 ++
 .github/workflows/make-k8s-images.yml       | 35 +++++++++++++++++++++
 .github/workflows/repo-list.txt             |  2 ++
 3 files changed, 40 insertions(+)
 create mode 100644 .github/workflows/make-k8s-images.yml
 create mode 100644 .github/workflows/repo-list.txt

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index d42d2447d1..ab429c50f0 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -38,6 +38,9 @@ jobs:
     - uses: 'google-github-actions/auth@v2'
       with:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+    - name: Install tomlq
+      run: |
+        gcloud container clusters list
     # - name: Create local k8s cluster
     #   run: |
     #     kind create cluster --name regression-tester --config .github/workflows/kind-config.yaml.sample
diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
new file mode 100644
index 0000000000..d7f7c879db
--- /dev/null
+++ b/.github/workflows/make-k8s-images.yml
@@ -0,0 +1,35 @@
+name: Periodic Image Sync
+
+on:
+  schedule:
+    - cron: '*/15 * * * *'  # Run every 15 minutes
+
+jobs:
+  sync-images:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout Repository
+      uses: actions/checkout@v2
+
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v1
+      
+    - uses: 'google-github-actions/auth@v2'
+      with:
+        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+
+    - name: Fetch Repos and Sync Images
+      run: |
+        while IFS= read -r repo; do
+          latest_tag=$(curl -s "https://api.github.com/repos/${repo}/releases/latest" | jq -r .tag_name)
+          if [ -n "$latest_tag" ]; then
+            # Check if the image version already exists in GCP Container Registry
+            if ! docker pull "gcr.io/prj-polygonlabs-devtools-dev/${repo}:${latest_tag}" 2>/dev/null; then
+              docker tag "${repo}:${latest_tag}" "gcr.io/prj-polygonlabs-devtools-dev/${repo}:${latest_tag}"
+              docker push "gcr.io/prj-polygonlabs-devtools-dev/${repo}:${latest_tag}"
+            else
+              echo "Image version already exists for ${repo}:${latest_tag}. Skipping upload."
+            fi
+          fi
+        done < repo-list.txt
diff --git a/.github/workflows/repo-list.txt b/.github/workflows/repo-list.txt
new file mode 100644
index 0000000000..9d3c1f2bbc
--- /dev/null
+++ b/.github/workflows/repo-list.txt
@@ -0,0 +1,2 @@
+maticnetwork/bor
+maticnetwork/heimdall

From 51c0d11fb00610d0fe232d557d357d1141aa2083 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 5 Feb 2024 15:28:03 -0600
Subject: [PATCH 032/144] testing

---
 .github/workflows/k8s-regression-tester.yml |  1 +
 .github/workflows/make-k8s-images.yml       | 16 +++++++++-------
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index ab429c50f0..c8fd8ee238 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -40,6 +40,7 @@ jobs:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
     - name: Install tomlq
       run: |
+        gcloud container clusters get-credentials CLUSTER_NAME --zone=COMPUTE_ZONE
         gcloud container clusters list
     # - name: Create local k8s cluster
     #   run: |
diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index d7f7c879db..e433af4029 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -1,24 +1,26 @@
-name: Periodic Image Sync
-
+name: Image Sync
 on:
   schedule:
-    - cron: '*/15 * * * *'  # Run every 15 minutes
+    - cron: '*/2 * * * *'  # Run every 2 mins for testing...
 
 jobs:
   sync-images:
     runs-on: ubuntu-latest
-
     steps:
     - name: Checkout Repository
       uses: actions/checkout@v2
-
     - name: Set up Docker
       uses: docker/setup-buildx-action@v1
-      
     - uses: 'google-github-actions/auth@v2'
       with:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-
+    - name: Clone and then build devnet images
+      run: |
+        eval `ssh-agent -s`
+        ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
+        git clone git@github.com:maticnetwork/polygon-devnets.git
+        # cd polygon-devnets/docker/pos
+        # make all DEV=true K8S_ENV=true K8S_NS=test
     - name: Fetch Repos and Sync Images
       run: |
         while IFS= read -r repo; do

From 5e27fe4199501650e0e4f9c19ff009846bcbaa81 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 5 Feb 2024 15:31:21 -0600
Subject: [PATCH 033/144] test

---
 .github/workflows/make-k8s-images.yml                     | 2 +-
 .github/workflows/{repo-list.txt => repo-list.txt.sample} | 0
 2 files changed, 1 insertion(+), 1 deletion(-)
 rename .github/workflows/{repo-list.txt => repo-list.txt.sample} (100%)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index e433af4029..c3958a57c9 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -34,4 +34,4 @@ jobs:
               echo "Image version already exists for ${repo}:${latest_tag}. Skipping upload."
             fi
           fi
-        done < repo-list.txt
+        done < repo-list.txt.sample
diff --git a/.github/workflows/repo-list.txt b/.github/workflows/repo-list.txt.sample
similarity index 100%
rename from .github/workflows/repo-list.txt
rename to .github/workflows/repo-list.txt.sample

From a2c0afcf953fcf83d2a58549dbf65298b9cb9c37 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 5 Feb 2024 15:36:38 -0600
Subject: [PATCH 034/144] test

---
 .github/workflows/make-k8s-images.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index c3958a57c9..eb3ec5e78f 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -1,7 +1,5 @@
 name: Image Sync
-on:
-  schedule:
-    - cron: '*/2 * * * *'  # Run every 2 mins for testing...
+on: [ push, pull_request ]
 
 jobs:
   sync-images:

From 74318d14f794d3d111fa84b32a966777b849336c Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 5 Feb 2024 15:39:23 -0600
Subject: [PATCH 035/144] test

---
 .github/workflows/make-k8s-images.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index eb3ec5e78f..d993b5429e 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -32,4 +32,4 @@ jobs:
               echo "Image version already exists for ${repo}:${latest_tag}. Skipping upload."
             fi
           fi
-        done < repo-list.txt.sample
+        done < .github/workflows/repo-list.txt.sample

From a2f76e83adc400b9f8a29f3a8b0966d28f063d10 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 5 Feb 2024 15:48:34 -0600
Subject: [PATCH 036/144] test

---
 .github/workflows/make-k8s-images.yml  | 8 ++++----
 .github/workflows/repo-list.txt.sample | 5 +++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index d993b5429e..34f9d33402 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -22,12 +22,12 @@ jobs:
     - name: Fetch Repos and Sync Images
       run: |
         while IFS= read -r repo; do
-          latest_tag=$(curl -s "https://api.github.com/repos/${repo}/releases/latest" | jq -r .tag_name)
+          latest_tag=$(curl -s "https://api.github.com/repos/0xpolygon/${repo}/releases/latest" | jq -r .tag_name)
           if [ -n "$latest_tag" ]; then
             # Check if the image version already exists in GCP Container Registry
-            if ! docker pull "gcr.io/prj-polygonlabs-devtools-dev/${repo}:${latest_tag}" 2>/dev/null; then
-              docker tag "${repo}:${latest_tag}" "gcr.io/prj-polygonlabs-devtools-dev/${repo}:${latest_tag}"
-              docker push "gcr.io/prj-polygonlabs-devtools-dev/${repo}:${latest_tag}"
+            if ! docker pull "0xpolygon/${repo}:${latest_tag}" 2>/dev/null; then
+              docker tag "0xpolygon/${repo}:${latest_tag}" "gcr.io/prj-polygonlabs-devtools-dev/0xpolygon/${repo}:${latest_tag}"
+              docker push "gcr.io/prj-polygonlabs-devtools-dev/0xpolygon/${repo}:${latest_tag}"
             else
               echo "Image version already exists for ${repo}:${latest_tag}. Skipping upload."
             fi
diff --git a/.github/workflows/repo-list.txt.sample b/.github/workflows/repo-list.txt.sample
index 9d3c1f2bbc..dd6fa6e901 100644
--- a/.github/workflows/repo-list.txt.sample
+++ b/.github/workflows/repo-list.txt.sample
@@ -1,2 +1,3 @@
-maticnetwork/bor
-maticnetwork/heimdall
+bor
+heimdall
+erigon

From 5e3c6b1147e2e7efc05d1e8c24faa6cf47b5c604 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 14:47:52 -0600
Subject: [PATCH 037/144] testing

---
 .github/workflows/make-k8s-images.yml  | 20 +++++++-------------
 .github/workflows/repo-list.txt.sample |  6 +++---
 2 files changed, 10 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 34f9d33402..e0d974bc0c 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -12,24 +12,18 @@ jobs:
     - uses: 'google-github-actions/auth@v2'
       with:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-    - name: Clone and then build devnet images
-      run: |
-        eval `ssh-agent -s`
-        ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
-        git clone git@github.com:maticnetwork/polygon-devnets.git
-        # cd polygon-devnets/docker/pos
-        # make all DEV=true K8S_ENV=true K8S_NS=test
     - name: Fetch Repos and Sync Images
       run: |
-        while IFS= read -r repo; do
-          latest_tag=$(curl -s "https://api.github.com/repos/0xpolygon/${repo}/releases/latest" | jq -r .tag_name)
+        while IFS= read -r image_path; do
+          latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
           if [ -n "$latest_tag" ]; then
             # Check if the image version already exists in GCP Container Registry
-            if ! docker pull "0xpolygon/${repo}:${latest_tag}" 2>/dev/null; then
-              docker tag "0xpolygon/${repo}:${latest_tag}" "gcr.io/prj-polygonlabs-devtools-dev/0xpolygon/${repo}:${latest_tag}"
-              docker push "gcr.io/prj-polygonlabs-devtools-dev/0xpolygon/${repo}:${latest_tag}"
+            if ! gcloud container images list-tags gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
+              docker pull "${image_path}:${latest_tag}"
+              docker tag "${image_path}:${latest_tag}" "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+              docker push "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
             else
-              echo "Image version already exists for ${repo}:${latest_tag}. Skipping upload."
+              echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
             fi
           fi
         done < .github/workflows/repo-list.txt.sample
diff --git a/.github/workflows/repo-list.txt.sample b/.github/workflows/repo-list.txt.sample
index dd6fa6e901..d27de67a10 100644
--- a/.github/workflows/repo-list.txt.sample
+++ b/.github/workflows/repo-list.txt.sample
@@ -1,3 +1,3 @@
-bor
-heimdall
-erigon
+0xpolygon/bor
+0xpolygon/heimdall
+thorax/erigon

From 4a65dcd8df5de6b220c74a22c81d06125e41c719 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 14:52:44 -0600
Subject: [PATCH 038/144] keep testing

---
 .github/workflows/make-k8s-images.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index e0d974bc0c..1301afa027 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -12,8 +12,15 @@ jobs:
     - uses: 'google-github-actions/auth@v2'
       with:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+    - name: 'Set up Cloud SDK'
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        version: '>= 363.0.0'
     - name: Fetch Repos and Sync Images
       run: |
+        sudo -s
+        gcloud auth login
+        gcloud auth configure-docker
         while IFS= read -r image_path; do
           latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
           if [ -n "$latest_tag" ]; then

From 7f8056c197feb6d42e36abe84d661593e546c03e Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 14:56:00 -0600
Subject: [PATCH 039/144] keep testin

---
 .github/workflows/make-k8s-images.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 1301afa027..29b51ff13c 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -19,7 +19,7 @@ jobs:
     - name: Fetch Repos and Sync Images
       run: |
         sudo -s
-        gcloud auth login
+        gcloud auth login --no-browser
         gcloud auth configure-docker
         while IFS= read -r image_path; do
           latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')

From 8baaf4d0ebf0de8826079c9c609e7f85aaea284a Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 15:16:04 -0600
Subject: [PATCH 040/144] tests

---
 .github/workflows/make-k8s-images.yml | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 29b51ff13c..659f8bba93 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -18,9 +18,7 @@ jobs:
         version: '>= 363.0.0'
     - name: Fetch Repos and Sync Images
       run: |
-        sudo -s
-        gcloud auth login --no-browser
-        gcloud auth configure-docker
+        gcloud auth configure-docker -q
         while IFS= read -r image_path; do
           latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
           if [ -n "$latest_tag" ]; then
@@ -34,3 +32,12 @@ jobs:
             fi
           fi
         done < .github/workflows/repo-list.txt.sample
+    - uses: RafikFarhad/push-to-gcr-github-action@v5-rc1
+      with:
+        gcloud_service_key: '${{ secrets.GOOGLE_CREDENTIALS }}'
+        registry: gcr.io
+        project_id: prj-polygonlabs-devtools-dev
+        image_name: backend
+        image_tag: latest,v1
+        dockerfile: ./docker/Dockerfile.prod
+        context: ./docker

From 0f757c5c96e4abb238894e8269c942b22999859a Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 15:19:45 -0600
Subject: [PATCH 041/144] tester

---
 .github/workflows/make-k8s-images.yml | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 659f8bba93..73a94f087c 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -23,21 +23,12 @@ jobs:
           latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
           if [ -n "$latest_tag" ]; then
             # Check if the image version already exists in GCP Container Registry
-            if ! gcloud container images list-tags gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
+            if ! gcloud container images list-tags eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
               docker pull "${image_path}:${latest_tag}"
-              docker tag "${image_path}:${latest_tag}" "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
-              docker push "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+              docker tag "${image_path}:${latest_tag}" "eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+              docker push "eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
             else
               echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
             fi
           fi
         done < .github/workflows/repo-list.txt.sample
-    - uses: RafikFarhad/push-to-gcr-github-action@v5-rc1
-      with:
-        gcloud_service_key: '${{ secrets.GOOGLE_CREDENTIALS }}'
-        registry: gcr.io
-        project_id: prj-polygonlabs-devtools-dev
-        image_name: backend
-        image_tag: latest,v1
-        dockerfile: ./docker/Dockerfile.prod
-        context: ./docker

From 05a2544afdaac74e8f628328fec5ce493b47fff5 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 15:26:30 -0600
Subject: [PATCH 042/144] pls

---
 .github/workflows/make-k8s-images.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 73a94f087c..f52db787d5 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -26,7 +26,7 @@ jobs:
             if ! gcloud container images list-tags eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
               docker pull "${image_path}:${latest_tag}"
               docker tag "${image_path}:${latest_tag}" "eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
-              docker push "eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+              sudo -u $USER docker push "eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
             else
               echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
             fi

From 98e3e7cd1d21eee6c655dec3d021746d96814d53 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 15:35:09 -0600
Subject: [PATCH 043/144] more tests

---
 .github/workflows/make-k8s-images.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index f52db787d5..6eedd7e623 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -18,15 +18,15 @@ jobs:
         version: '>= 363.0.0'
     - name: Fetch Repos and Sync Images
       run: |
-        gcloud auth configure-docker -q
+        echo '${{ secrets.GOOGLE_CREDENTIALS }}' | gcloud auth configure-docker activate-service-account --key-file=- -q
         while IFS= read -r image_path; do
           latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
           if [ -n "$latest_tag" ]; then
             # Check if the image version already exists in GCP Container Registry
-            if ! gcloud container images list-tags eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
+            if ! gcloud container images list-tags gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
               docker pull "${image_path}:${latest_tag}"
-              docker tag "${image_path}:${latest_tag}" "eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
-              sudo -u $USER docker push "eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+              docker tag "${image_path}:${latest_tag}" "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+              sudo -u $USER docker push "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
             else
               echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
             fi

From 313c34762f01c5977bf94986aacb8821c4bd7469 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 15:41:28 -0600
Subject: [PATCH 044/144] cmon

---
 .github/workflows/make-k8s-images.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 6eedd7e623..3ef838de66 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -12,13 +12,14 @@ jobs:
     - uses: 'google-github-actions/auth@v2'
       with:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+        credentials_file_path: '~/svc-acct.json'
     - name: 'Set up Cloud SDK'
       uses: 'google-github-actions/setup-gcloud@v2'
       with:
         version: '>= 363.0.0'
     - name: Fetch Repos and Sync Images
       run: |
-        echo '${{ secrets.GOOGLE_CREDENTIALS }}' | gcloud auth configure-docker activate-service-account --key-file=- -q
+        gcloud auth configure-docker activate-service-account --key-file='~/svc-acct.json' -q
         while IFS= read -r image_path; do
           latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
           if [ -n "$latest_tag" ]; then

From 5264d345754875fe3feb1b7baf5e3bd63bd40a95 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 15:49:42 -0600
Subject: [PATCH 045/144] testing

---
 .github/workflows/make-k8s-images.yml | 46 +++++++++++++++------------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 3ef838de66..eccde73567 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -10,26 +10,32 @@ jobs:
     - name: Set up Docker
       uses: docker/setup-buildx-action@v1
     - uses: 'google-github-actions/auth@v2'
+      id: auth
       with:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-        credentials_file_path: '~/svc-acct.json'
-    - name: 'Set up Cloud SDK'
-      uses: 'google-github-actions/setup-gcloud@v2'
-      with:
-        version: '>= 363.0.0'
-    - name: Fetch Repos and Sync Images
+    - name: Extract credentials file path
+      id: extract
       run: |
-        gcloud auth configure-docker activate-service-account --key-file='~/svc-acct.json' -q
-        while IFS= read -r image_path; do
-          latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
-          if [ -n "$latest_tag" ]; then
-            # Check if the image version already exists in GCP Container Registry
-            if ! gcloud container images list-tags gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
-              docker pull "${image_path}:${latest_tag}"
-              docker tag "${image_path}:${latest_tag}" "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
-              sudo -u $USER docker push "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
-            else
-              echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
-            fi
-          fi
-        done < .github/workflows/repo-list.txt.sample
+        CREDENTIALS_PATH=$(echo "${{ steps.auth.outputs.credentials_file_path }}")
+        echo "Credentials file path: $CREDENTIALS_PATH"
+        echo "::set-output name=credentials_path::$CREDENTIALS_PATH"
+    # - name: 'Set up Cloud SDK'
+    #   uses: 'google-github-actions/setup-gcloud@v2'
+    #   with:
+    #     version: '>= 363.0.0'
+    # - name: Fetch Repos and Sync Images
+    #   run: |
+    #     gcloud auth configure-docker activate-service-account --key-file='~/svc-acct.json' -q
+    #     while IFS= read -r image_path; do
+    #       latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
+    #       if [ -n "$latest_tag" ]; then
+    #         # Check if the image version already exists in GCP Container Registry
+    #         if ! gcloud container images list-tags gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
+    #           docker pull "${image_path}:${latest_tag}"
+    #           docker tag "${image_path}:${latest_tag}" "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+    #           sudo -u $USER docker push "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+    #         else
+    #           echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
+    #         fi
+    #       fi
+    #     done < .github/workflows/repo-list.txt.sample

From e634a24a56aa1933eb20fa88254b9acff8536646 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 15:54:03 -0600
Subject: [PATCH 046/144] moar tests

---
 .github/workflows/make-k8s-images.yml | 40 +++++++++++++--------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index eccde73567..8afd6065af 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -19,23 +19,23 @@ jobs:
         CREDENTIALS_PATH=$(echo "${{ steps.auth.outputs.credentials_file_path }}")
         echo "Credentials file path: $CREDENTIALS_PATH"
         echo "::set-output name=credentials_path::$CREDENTIALS_PATH"
-    # - name: 'Set up Cloud SDK'
-    #   uses: 'google-github-actions/setup-gcloud@v2'
-    #   with:
-    #     version: '>= 363.0.0'
-    # - name: Fetch Repos and Sync Images
-    #   run: |
-    #     gcloud auth configure-docker activate-service-account --key-file='~/svc-acct.json' -q
-    #     while IFS= read -r image_path; do
-    #       latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
-    #       if [ -n "$latest_tag" ]; then
-    #         # Check if the image version already exists in GCP Container Registry
-    #         if ! gcloud container images list-tags gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
-    #           docker pull "${image_path}:${latest_tag}"
-    #           docker tag "${image_path}:${latest_tag}" "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
-    #           sudo -u $USER docker push "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
-    #         else
-    #           echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
-    #         fi
-    #       fi
-    #     done < .github/workflows/repo-list.txt.sample
+    - name: 'Set up Cloud SDK'
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        version: '>= 363.0.0'
+    - name: Fetch Repos and Sync Images
+      run: |
+        gcloud auth configure-docker gcr.io
+        while IFS= read -r image_path; do
+          latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
+          if [ -n "$latest_tag" ]; then
+            # Check if the image version already exists in GCP Container Registry
+            if ! gcloud container images list-tags gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
+              docker pull "${image_path}:${latest_tag}"
+              docker tag "${image_path}:${latest_tag}" "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+              sudo -u $USER docker push "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+            else
+              echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
+            fi
+          fi
+        done < .github/workflows/repo-list.txt.sample

From 4e8c9e120940139b083963dab466dcad51b83790 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 15:55:43 -0600
Subject: [PATCH 047/144] cmonn

---
 .github/workflows/make-k8s-images.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 8afd6065af..9e757abc5a 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -25,7 +25,7 @@ jobs:
         version: '>= 363.0.0'
     - name: Fetch Repos and Sync Images
       run: |
-        gcloud auth configure-docker gcr.io
+        gcloud auth configure-docker gcr.io -q
         while IFS= read -r image_path; do
           latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
           if [ -n "$latest_tag" ]; then

From 13ce926d61bf60def86099616abb3947ff1dc754 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 15:57:26 -0600
Subject: [PATCH 048/144] pls

---
 .github/workflows/make-k8s-images.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 9e757abc5a..469f5646ca 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -25,15 +25,15 @@ jobs:
         version: '>= 363.0.0'
     - name: Fetch Repos and Sync Images
       run: |
-        gcloud auth configure-docker gcr.io -q
+        gcloud auth configure-docker eu.gcr.io -q
         while IFS= read -r image_path; do
           latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
           if [ -n "$latest_tag" ]; then
             # Check if the image version already exists in GCP Container Registry
-            if ! gcloud container images list-tags gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
+            if ! gcloud container images list-tags eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
               docker pull "${image_path}:${latest_tag}"
-              docker tag "${image_path}:${latest_tag}" "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
-              sudo -u $USER docker push "gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+              docker tag "${image_path}:${latest_tag}" "eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+              sudo -u $USER docker push "eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
             else
               echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
             fi

From 34b251881dc3ad1753ebd503949b6e2359b33aa7 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 16:04:55 -0600
Subject: [PATCH 049/144] plss

---
 .github/workflows/make-k8s-images.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 469f5646ca..49fcc0a4a9 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -25,17 +25,19 @@ jobs:
         version: '>= 363.0.0'
     - name: Fetch Repos and Sync Images
       run: |
-        gcloud auth configure-docker eu.gcr.io -q
+        gcloud auth configure-docker -q
         while IFS= read -r image_path; do
           latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
           if [ -n "$latest_tag" ]; then
             # Check if the image version already exists in GCP Container Registry
-            if ! gcloud container images list-tags eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
+            if ! gcloud container images list-tags europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
               docker pull "${image_path}:${latest_tag}"
-              docker tag "${image_path}:${latest_tag}" "eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
-              sudo -u $USER docker push "eu.gcr.io/prj-polygonlabs-devtools-dev/${image_path}:${latest_tag}"
+              docker tag "${image_path}:${latest_tag}" "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path}:${latest_tag}"
+              sudo -u $USER docker push "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path}:${latest_tag}"
             else
               echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
             fi
           fi
         done < .github/workflows/repo-list.txt.sample
+        
+        

From c8d94c7d85d74dbdc008d11842e5a6eb7a0fc4fa Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 16:07:06 -0600
Subject: [PATCH 050/144] wow

---
 .github/workflows/make-k8s-images.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 49fcc0a4a9..d2a693f55c 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -25,7 +25,7 @@ jobs:
         version: '>= 363.0.0'
     - name: Fetch Repos and Sync Images
       run: |
-        gcloud auth configure-docker -q
+        gcloud auth configure-docker europe-west2-docker.pkg.dev -q
         while IFS= read -r image_path; do
           latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
           if [ -n "$latest_tag" ]; then

From f682f3926d5d91ff8d2f7f8031031a4b3d1489ad Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 16:21:32 -0600
Subject: [PATCH 051/144] testing

---
 .github/workflows/make-k8s-images.yml | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index d2a693f55c..a16ccaa339 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -13,12 +13,6 @@ jobs:
       id: auth
       with:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-    - name: Extract credentials file path
-      id: extract
-      run: |
-        CREDENTIALS_PATH=$(echo "${{ steps.auth.outputs.credentials_file_path }}")
-        echo "Credentials file path: $CREDENTIALS_PATH"
-        echo "::set-output name=credentials_path::$CREDENTIALS_PATH"
     - name: 'Set up Cloud SDK'
       uses: 'google-github-actions/setup-gcloud@v2'
       with:
@@ -27,7 +21,7 @@ jobs:
       run: |
         gcloud auth configure-docker europe-west2-docker.pkg.dev -q
         while IFS= read -r image_path; do
-          latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/?page_size=1" | jq -r '.results[0].name')
+          latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/" | jq -r '.results[].name | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))' | sort -V | tail -n 1)
           if [ -n "$latest_tag" ]; then
             # Check if the image version already exists in GCP Container Registry
             if ! gcloud container images list-tags europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then

From 5b0e277cb03d5880a7f4ef414fea50625e81cf9c Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 16:30:58 -0600
Subject: [PATCH 052/144] pls

---
 .github/workflows/make-k8s-images.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index a16ccaa339..85c502d183 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -1,5 +1,8 @@
 name: Image Sync
-on: [ push, pull_request ]
+
+on:
+  schedule:
+    - cron:  '*/5 * * * *'
 
 jobs:
   sync-images:
@@ -10,7 +13,6 @@ jobs:
     - name: Set up Docker
       uses: docker/setup-buildx-action@v1
     - uses: 'google-github-actions/auth@v2'
-      id: auth
       with:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
     - name: 'Set up Cloud SDK'
@@ -28,6 +30,8 @@ jobs:
               docker pull "${image_path}:${latest_tag}"
               docker tag "${image_path}:${latest_tag}" "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path}:${latest_tag}"
               sudo -u $USER docker push "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path}:${latest_tag}"
+              # also tag image as 'latest' for internal tracking
+              sudo -u $USER docker push "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path}:latest"
             else
               echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
             fi

From 0bfe38b2904f81069936d4cce7042cfed0f0bff9 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 16:45:48 -0600
Subject: [PATCH 053/144] cmon

---
 .github/workflows/k8s-regression-tester.yml | 1 -
 .github/workflows/make-k8s-images.yml       | 5 +++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index c8fd8ee238..ab429c50f0 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -40,7 +40,6 @@ jobs:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
     - name: Install tomlq
       run: |
-        gcloud container clusters get-credentials CLUSTER_NAME --zone=COMPUTE_ZONE
         gcloud container clusters list
     # - name: Create local k8s cluster
     #   run: |
diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 85c502d183..8568b6108d 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -1,8 +1,9 @@
 name: Image Sync
 
 on:
-  schedule:
-    - cron:  '*/5 * * * *'
+workflow_dispatch:
+schedule:
+  - cron: ‘*/5 * * * *’
 
 jobs:
   sync-images:

From e0b272c8ae229f1a1e5532a4f0279e73711fef25 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 16:48:14 -0600
Subject: [PATCH 054/144] pls work

---
 .github/workflows/make-k8s-images.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 8568b6108d..16ee29806a 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -1,9 +1,9 @@
 name: Image Sync
 
 on:
-workflow_dispatch:
-schedule:
-  - cron: ‘*/5 * * * *’
+  workflow_dispatch:
+  schedule:
+    - cron: '*/5 * * * *'
 
 jobs:
   sync-images:

From fe0841b4c7997e85dc57361585d7ab2b20e8ec6a Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 6 Feb 2024 17:02:39 -0600
Subject: [PATCH 055/144] merge to main

---
 .github/workflows/make-k8s-images.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 16ee29806a..4d4ee14ac9 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -20,7 +20,7 @@ jobs:
       uses: 'google-github-actions/setup-gcloud@v2'
       with:
         version: '>= 363.0.0'
-    - name: Fetch Repos and Sync Images
+    - name: Fetch repos, rebuild for k8s, push to internal artifact registry
       run: |
         gcloud auth configure-docker europe-west2-docker.pkg.dev -q
         while IFS= read -r image_path; do
@@ -38,5 +38,3 @@ jobs:
             fi
           fi
         done < .github/workflows/repo-list.txt.sample
-        
-        

From 2245406fddc12ec780431ef010369500874358b4 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 7 Feb 2024 14:25:02 -0600
Subject: [PATCH 056/144] prebuild all images

---
 .github/workflows/k8s-regression-tester.yml | 21 +++++++++++++++++++--
 .github/workflows/make-k8s-images.yml       |  3 +--
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index ab429c50f0..5848211964 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -21,6 +21,12 @@ jobs:
       uses: actions/setup-go@v2
       with:
         go-version: 1.17
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v1
+    - name: 'Set up Cloud SDK'
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        version: '>= 363.0.0'
     - name: Install kubectl
       run: |
         curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
@@ -38,9 +44,20 @@ jobs:
     - uses: 'google-github-actions/auth@v2'
       with:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-    - name: Install tomlq
+    - name: Clone, build, and load k8s devnet images
       run: |
-        gcloud container clusters list
+        eval `ssh-agent -s`
+        ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
+        git clone git@github.com:maticnetwork/polygon-devnets.git
+        cd polygon-devnets/docker/pos
+        make all DEV=true K8S_ENV=true K8S_NS=test
+        IMAGES=$(docker images -q)
+        echo $IMAGES
+        gcloud auth configure-docker europe-west2-docker.pkg.dev -q
+        for IMAGE_ID in $IMAGES; do
+          IMAGE_NAME=$(docker inspect --format='{{index .RepoTags 0}}' $IMAGE_ID)
+          sudo -u $USER docker push "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${IMAGE_NAME}:latest"
+        done
     # - name: Create local k8s cluster
     #   run: |
     #     kind create cluster --name regression-tester --config .github/workflows/kind-config.yaml.sample
diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 4d4ee14ac9..948334eb88 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -1,9 +1,8 @@
 name: Image Sync
 
 on:
-  workflow_dispatch:
   schedule:
-    - cron: '*/5 * * * *'
+    - cron: '*/15 * * * *'
 
 jobs:
   sync-images:

From a8062561d3e35ac68fd79af6a2e89d787e311aa2 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 04:29:09 -0600
Subject: [PATCH 057/144] e2e k8s pos ci devnet tests

---
 .github/workflows/k8s-regression-tester.yml | 103 ++++++++------------
 1 file changed, 38 insertions(+), 65 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 5848211964..98aec1652b 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -1,87 +1,60 @@
-name: Deploy Devnet
-on: [ push, pull_request ]
+name: K8S DEVNET - REGRESSION DETECTOR
+on:
+  push:
+  release:
+    types: [prereleased]
 
 jobs:
   deploy_devnet:
     runs-on: ubuntu-latest
     steps:
-    - name: Maximize runner build space for k8s images
-      uses: AdityaGarg8/remove-unwanted-software@v2
-      with:
-        remove-android: 'true'
-        remove-dotnet: 'true'
-        remove-haskell: 'true'
-        remove-codeql: 'true'
-        remove-docker-images: 'true'
-    - name: Checkout curent repository
+    - name: Checkout repository
       uses: actions/checkout@v2
       with:
         ref: ${{ github.event.before }}
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.17
+
     - name: Set up Docker
       uses: docker/setup-buildx-action@v1
-    - name: 'Set up Cloud SDK'
+
+    - name: Setup Google Cloud SDK
       uses: 'google-github-actions/setup-gcloud@v2'
       with:
         version: '>= 363.0.0'
+
+    - uses: 'google-github-actions/auth@v2'
+      with:
+        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+
     - name: Install kubectl
       run: |
         curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
-        && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl \
-        && kubectl version --client
-    - name: Install kind
-      run: go install sigs.k8s.io/kind@v0.20.0
-    - name: Install tomlq
+        && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+
+    - name: Install tomlq for config file preparation
       run: |
         pip3 install tomlq
-    - name: Free github runner disk space for k8s image building
-      run: |
-        rm -rf /usr/share/dotnet/
-        rm -rf /opt/hostedtoolcache
-    - uses: 'google-github-actions/auth@v2'
-      with:
-        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-    - name: Clone, build, and load k8s devnet images
+
+    - name: Clone private polygon-devnets repo
       run: |
         eval `ssh-agent -s`
         ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
-        git clone git@github.com:maticnetwork/polygon-devnets.git
-        cd polygon-devnets/docker/pos
-        make all DEV=true K8S_ENV=true K8S_NS=test
-        IMAGES=$(docker images -q)
-        echo $IMAGES
-        gcloud auth configure-docker europe-west2-docker.pkg.dev -q
-        for IMAGE_ID in $IMAGES; do
-          IMAGE_NAME=$(docker inspect --format='{{index .RepoTags 0}}' $IMAGE_ID)
-          sudo -u $USER docker push "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${IMAGE_NAME}:latest"
+        git clone -b fix/pos_k8s_devnets git@github.com:maticnetwork/polygon-devnets.git
+
+    - name: Deploy PoS devnet to isolated GKE namespace
+      run: |
+        cd polygon-devnets/kubernetes/pos
+        # metrics must be tagged by unique namespace (devnet name / pre-release id)
+        # deployment must also create custom monitors by namespace
+        kubectl apply -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+
+    - name: Auto-apply load and report any regressions here
+      run: |
+        current_minute=$(date +'%M')
+        end_minute=$((current_minute + 60)) # Corrected the missing closing parenthesis
+        
+        while [ $current_minute -lt $end_minute ]; do
+          # TODO: call datadog API and get composite monitor status
+          echo "REGRESSION TESTING RESULTS... PRINTED HERE... "
+          sleep 600
+          current_minute=$(date +'%M')
         done
-    # - name: Create local k8s cluster
-    #   run: |
-    #     kind create cluster --name regression-tester --config .github/workflows/kind-config.yaml.sample
-    #     kubectl cluster-info --context kind-regression-tester
-    # - name: Clone and then build devnet images
-    #   run: |
-    #     eval `ssh-agent -s`
-    #     ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
-    #     git clone git@github.com:maticnetwork/polygon-devnets.git
-    #     cd polygon-devnets/docker/pos
-    #     make all DEV=true K8S_ENV=true K8S_NS=test
-    # - name: Load images into cluster
-    #   run: kind load docker-image ganache:latest heimdall:latest bor:latest workload:latest status:latest --name regression-tester
-    # - name: Deploy devnet
-    #   run: |
-    #     cd polygon-devnets/docker/pos/k8s
-    #     kubectl apply -k . --context kind-regression-tester
-    # - name: Check deployed resources
-    #   run: |
-    #     current_minute=$(date +'%M')
-    #     end_minute=$((current_minute + 60)) # Run for the next hour as test
-    # 
-    #     while [ $current_minute -lt $end_minute ]; do
-    #       kubectl get pods -n test
-    #       sleep 600 # Sleep for 10 minutes
-    #       current_minute=$(date +'%M')
-    #     done

From 3197635cf7d5a1e50b18f19b7a12d1a4144bdf3d Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 04:31:52 -0600
Subject: [PATCH 058/144] fix: get kubeconfig before devnet deploy

---
 .github/workflows/k8s-regression-tester.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 98aec1652b..aa65f101ac 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -43,6 +43,7 @@ jobs:
     - name: Deploy PoS devnet to isolated GKE namespace
       run: |
         cd polygon-devnets/kubernetes/pos
+        gcloud container clusters get-credentials ci-cluster-1 --region europe-west2
         # metrics must be tagged by unique namespace (devnet name / pre-release id)
         # deployment must also create custom monitors by namespace
         kubectl apply -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1

From 4fc743cb07456b6da6f85638bd5e28595612b011 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 04:38:36 -0600
Subject: [PATCH 059/144] fix: gcloud auth

---
 .github/workflows/k8s-regression-tester.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index aa65f101ac..05f82def97 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -43,6 +43,8 @@ jobs:
     - name: Deploy PoS devnet to isolated GKE namespace
       run: |
         cd polygon-devnets/kubernetes/pos
+        echo "${{ secrets.GOOGLE_CREDENTIALS }}" > /tmp/credentials.json
+        gcloud auth activate-service-account ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com --key-file=/tmp/credentials.json --project=prj-polygonlabs-devtools-dev
         gcloud container clusters get-credentials ci-cluster-1 --region europe-west2
         # metrics must be tagged by unique namespace (devnet name / pre-release id)
         # deployment must also create custom monitors by namespace

From 5a232ad9e2ce7bf55ecf04b0979f9928f36b44b2 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 04:48:36 -0600
Subject: [PATCH 060/144] fix: gcloud auth cont

---
 .github/workflows/k8s-regression-tester.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 05f82def97..9ae36a8d9c 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -24,6 +24,8 @@ jobs:
     - uses: 'google-github-actions/auth@v2'
       with:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+        project_id: 'prj-polygonlabs-devtools-dev'
+        service_account: 'ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com'
 
     - name: Install kubectl
       run: |

From 354aabc2945725a40d7f5242dc3bafe8c5103017 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 04:50:51 -0600
Subject: [PATCH 061/144] fix: gcloud auth still

---
 .github/workflows/k8s-regression-tester.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 9ae36a8d9c..3a131b8df5 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -45,8 +45,6 @@ jobs:
     - name: Deploy PoS devnet to isolated GKE namespace
       run: |
         cd polygon-devnets/kubernetes/pos
-        echo "${{ secrets.GOOGLE_CREDENTIALS }}" > /tmp/credentials.json
-        gcloud auth activate-service-account ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com --key-file=/tmp/credentials.json --project=prj-polygonlabs-devtools-dev
         gcloud container clusters get-credentials ci-cluster-1 --region europe-west2
         # metrics must be tagged by unique namespace (devnet name / pre-release id)
         # deployment must also create custom monitors by namespace

From 711ab7fc8adef2d8e731c83320288e92c4c96413 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 04:53:55 -0600
Subject: [PATCH 062/144] fix: gcloud auth ongoing

---
 .github/workflows/k8s-regression-tester.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 3a131b8df5..b05d9b78d1 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -45,6 +45,7 @@ jobs:
     - name: Deploy PoS devnet to isolated GKE namespace
       run: |
         cd polygon-devnets/kubernetes/pos
+        gcloud auth login --no-launch-browser
         gcloud container clusters get-credentials ci-cluster-1 --region europe-west2
         # metrics must be tagged by unique namespace (devnet name / pre-release id)
         # deployment must also create custom monitors by namespace

From e3c879959d1ea719e1e9a9f64bf87676a337e2c6 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 05:17:50 -0600
Subject: [PATCH 063/144] fix: google auth

---
 .github/workflows/k8s-regression-tester.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index b05d9b78d1..95bf1f392a 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -44,9 +44,12 @@ jobs:
 
     - name: Deploy PoS devnet to isolated GKE namespace
       run: |
-        cd polygon-devnets/kubernetes/pos
-        gcloud auth login --no-launch-browser
+        echo '${{ secrets.GOOGLE_CREDENTIALS }}' | base64 --decode > /tmp/credentials.json
+        echo "$(< /tmp/credentials.json | head -c 10)"
+        gcloud auth activate-service-account --key-file=/tmp/credentials.json ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com --project=prj-polygonlabs-devtools-dev
         gcloud container clusters get-credentials ci-cluster-1 --region europe-west2
+        
+        cd polygon-devnets/kubernetes/pos
         # metrics must be tagged by unique namespace (devnet name / pre-release id)
         # deployment must also create custom monitors by namespace
         kubectl apply -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1

From a1e9cf1a90550bed152376c4a2bf978856a09fcd Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 05:23:05 -0600
Subject: [PATCH 064/144] fix: gcloud auth

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 95bf1f392a..0e86c8cb6c 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -44,7 +44,7 @@ jobs:
 
     - name: Deploy PoS devnet to isolated GKE namespace
       run: |
-        echo '${{ secrets.GOOGLE_CREDENTIALS }}' | base64 --decode > /tmp/credentials.json
+        echo ${{ secrets.GOOGLE_CREDENTIALS }} > /tmp/credentials.json
         echo "$(< /tmp/credentials.json | head -c 10)"
         gcloud auth activate-service-account --key-file=/tmp/credentials.json ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com --project=prj-polygonlabs-devtools-dev
         gcloud container clusters get-credentials ci-cluster-1 --region europe-west2

From fe60818595c6be4760ec8cad92560b6e28b2496e Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 05:34:01 -0600
Subject: [PATCH 065/144] fix: gauth

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 0e86c8cb6c..6e01efb00c 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -44,7 +44,7 @@ jobs:
 
     - name: Deploy PoS devnet to isolated GKE namespace
       run: |
-        echo ${{ secrets.GOOGLE_CREDENTIALS }} > /tmp/credentials.json
+        echo '${{ secrets.GOOGLE_CREDENTIALS }}' > /tmp/credentials.json
         echo "$(< /tmp/credentials.json | head -c 10)"
         gcloud auth activate-service-account --key-file=/tmp/credentials.json ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com --project=prj-polygonlabs-devtools-dev
         gcloud container clusters get-credentials ci-cluster-1 --region europe-west2

From 91518a971e34c916dc943e7f1c56ca8574f817e7 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 05:40:11 -0600
Subject: [PATCH 066/144] fix: gauth

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 6e01efb00c..f21e5cc918 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -46,7 +46,7 @@ jobs:
       run: |
         echo '${{ secrets.GOOGLE_CREDENTIALS }}' > /tmp/credentials.json
         echo "$(< /tmp/credentials.json | head -c 10)"
-        gcloud auth activate-service-account --key-file=/tmp/credentials.json ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com --project=prj-polygonlabs-devtools-dev
+        # gcloud auth activate-service-account --key-file=/tmp/credentials.json ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com --project=prj-polygonlabs-devtools-dev
         gcloud container clusters get-credentials ci-cluster-1 --region europe-west2
         
         cd polygon-devnets/kubernetes/pos

From c1abfe73aa5fc7086168e05f819013885aa934c9 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 05:56:30 -0600
Subject: [PATCH 067/144] cmon

---
 .github/workflows/k8s-regression-tester.yml | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index f21e5cc918..e1503d617c 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -21,11 +21,11 @@ jobs:
       with:
         version: '>= 363.0.0'
 
-    - uses: 'google-github-actions/auth@v2'
-      with:
-        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-        project_id: 'prj-polygonlabs-devtools-dev'
-        service_account: 'ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com'
+    # - uses: 'google-github-actions/auth@v2'
+    #   with:
+    #     credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+    #     project_id: 'prj-polygonlabs-devtools-dev'
+    #     service_account: 'ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com'
 
     - name: Install kubectl
       run: |
@@ -44,11 +44,7 @@ jobs:
 
     - name: Deploy PoS devnet to isolated GKE namespace
       run: |
-        echo '${{ secrets.GOOGLE_CREDENTIALS }}' > /tmp/credentials.json
-        echo "$(< /tmp/credentials.json | head -c 10)"
-        # gcloud auth activate-service-account --key-file=/tmp/credentials.json ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com --project=prj-polygonlabs-devtools-dev
-        gcloud container clusters get-credentials ci-cluster-1 --region europe-west2
-        
+        echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > /tmp/cluster_credentials.yaml
         cd polygon-devnets/kubernetes/pos
         # metrics must be tagged by unique namespace (devnet name / pre-release id)
         # deployment must also create custom monitors by namespace

From 76c883f9c8100855c84808e81c11dc500cb5dcf1 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 06:01:42 -0600
Subject: [PATCH 068/144] fix gauth

---
 .github/workflows/k8s-regression-tester.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index e1503d617c..9730d53d84 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -45,6 +45,7 @@ jobs:
     - name: Deploy PoS devnet to isolated GKE namespace
       run: |
         echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > /tmp/cluster_credentials.yaml
+        export KUBECONFIG=/tmp/cluster_credentials.yaml
         cd polygon-devnets/kubernetes/pos
         # metrics must be tagged by unique namespace (devnet name / pre-release id)
         # deployment must also create custom monitors by namespace

From 6dd2fb1d195264ca638e4cc1a1acf99b1d115005 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 06:04:05 -0600
Subject: [PATCH 069/144] pls

---
 .github/workflows/k8s-regression-tester.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 9730d53d84..5829949f94 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -21,11 +21,11 @@ jobs:
       with:
         version: '>= 363.0.0'
 
-    # - uses: 'google-github-actions/auth@v2'
-    #   with:
-    #     credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-    #     project_id: 'prj-polygonlabs-devtools-dev'
-    #     service_account: 'ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com'
+    - uses: 'google-github-actions/auth@v2'
+      with:
+        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+        project_id: 'prj-polygonlabs-devtools-dev'
+        service_account: 'ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com'
 
     - name: Install kubectl
       run: |

From 41842e19059385e59ba0dd178f7b06066f5937af Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 06:09:26 -0600
Subject: [PATCH 070/144] fix gauthhh

---
 .github/workflows/k8s-regression-tester.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 5829949f94..b15e377648 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -27,10 +27,10 @@ jobs:
         project_id: 'prj-polygonlabs-devtools-dev'
         service_account: 'ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com'
 
-    - name: Install kubectl
+    - name: Install kubectl and gcloud kubectl auth plugin
       run: |
         curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
-        && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+        && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && gcloud components install gke-gcloud-auth-plugin
 
     - name: Install tomlq for config file preparation
       run: |

From b25bcb65fe7089dcea100eada79baeb7302a8dcf Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 27 Feb 2024 06:20:43 -0600
Subject: [PATCH 071/144] cmon

---
 .github/workflows/k8s-regression-tester.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index b15e377648..0035d4a6a0 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -25,7 +25,6 @@ jobs:
       with:
         credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
         project_id: 'prj-polygonlabs-devtools-dev'
-        service_account: 'ci-k8s-cluster@prj-polygonlabs-devtools-dev.iam.gserviceaccount.com'
 
     - name: Install kubectl and gcloud kubectl auth plugin
       run: |

From 3bec327223deb7c6e64c819d777c2dbc71e730e1 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Thu, 7 Mar 2024 16:12:04 -0600
Subject: [PATCH 072/144] e2e flowww

---
 .github/workflows/k8s-regression-tester.yml | 41 +++++++++++++++++----
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 0035d4a6a0..cb471469c5 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -40,24 +40,49 @@ jobs:
         eval `ssh-agent -s`
         ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
         git clone -b fix/pos_k8s_devnets git@github.com:maticnetwork/polygon-devnets.git
+        
+    - name: Clone private panoptichain repo
+      run: |
+        eval `ssh-agent -s`
+        ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY_PANOPTICHAIN }}'
+        git clone -b dan/k8s_panoptichain git@github.com:0xPolygon/panoptichain.git
 
     - name: Deploy PoS devnet to isolated GKE namespace
       run: |
-        echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > /tmp/cluster_credentials.yaml
-        export KUBECONFIG=/tmp/cluster_credentials.yaml
+        mkdir -p ~/.kube
+        echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/regression-cluster.yaml
+        export KUBECONFIG=~/.kube/regression-cluster.yaml
         cd polygon-devnets/kubernetes/pos
-        # metrics must be tagged by unique namespace (devnet name / pre-release id)
-        # deployment must also create custom monitors by namespace
         kubectl apply -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+        
+    - name: Deploy panoptichain, prometheus, grafana stack for metric tracking
+      run: |
+        cd panoptichain
+        export KUBECONFIG=~/.kube/regression-cluster.yaml
+        kubectl apply -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+        
+    - name: Finally, deploy datadog-agent via helm chart
+      run: |
+        cd panoptichain/terraform
+        terraform init
+        terraform apply -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
 
     - name: Auto-apply load and report any regressions here
       run: |
+        MONITOR_ID="140974014"  # K8S DEVNET: COMPOSITE MONITOR ID
+        DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
+        DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
+
         current_minute=$(date +'%M')
-        end_minute=$((current_minute + 60)) # Corrected the missing closing parenthesis
+        end_minute=$((current_minute + 60))
         
         while [ $current_minute -lt $end_minute ]; do
-          # TODO: call datadog API and get composite monitor status
-          echo "REGRESSION TESTING RESULTS... PRINTED HERE... "
-          sleep 600
+          STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
+                    -H "Content-Type: application/json" \
+                    -H "DD-API-KEY: ${DATADOG_API_KEY}" \
+                    -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
+                    -s \
+                    | jq -r '.overall_state')
+          echo "K8S DEVNET HEALTH STATUS: $STATUS"
           current_minute=$(date +'%M')
         done

From 64e6f5893485ecc49995159e05c25929840f7357 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Thu, 7 Mar 2024 16:15:25 -0600
Subject: [PATCH 073/144] fix monitor status check loop freq

---
 .github/workflows/k8s-regression-tester.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index cb471469c5..59e0dba318 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -85,4 +85,5 @@ jobs:
                     | jq -r '.overall_state')
           echo "K8S DEVNET HEALTH STATUS: $STATUS"
           current_minute=$(date +'%M')
+          sleep 60
         done

From ee29435d0abd9dbfcd32003972f631f3f9700510 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 09:44:27 -0600
Subject: [PATCH 074/144] ship it

---
 .github/workflows/k8s-regression-tester.yml | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 59e0dba318..de8351b869 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -74,7 +74,7 @@ jobs:
         DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
 
         current_minute=$(date +'%M')
-        end_minute=$((current_minute + 60))
+        end_minute=$((current_minute + 10))
         
         while [ $current_minute -lt $end_minute ]; do
           STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
@@ -84,6 +84,25 @@ jobs:
                     -s \
                     | jq -r '.overall_state')
           echo "K8S DEVNET HEALTH STATUS: $STATUS"
+          echo "Review your devnet health dashboard here: https://app.datadoghq.com/dashboard/fqu-nh2-bzd?fromUser=false&refresh_mode=sliding&view=spans&from_ts=1709886652761&to_ts=1709890252761&live=true"
           current_minute=$(date +'%M')
           sleep 60
         done
+        
+    - name: After stress tests, cleanup all ephemeral K8S devnet resources
+      run: |
+        # remove panoptichain, prometheus, and grafana resources
+        cd panoptichain
+        export KUBECONFIG=~/.kube/regression-cluster.yaml
+        kubectl delete -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+        
+        # remove all datadog agent resources
+        cd panoptichain/terraform
+        terraform init
+        terraform delete -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
+        
+        # finally, remove all pos devnet resources to return to clean slate GKE environment
+        export KUBECONFIG=~/.kube/regression-cluster.yaml
+        cd polygon-devnets/kubernetes/pos
+        kubectl delete -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+

From 4be953e5abe722e14c8160739b674e106adf02f1 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 10:08:17 -0600
Subject: [PATCH 075/144] delete stack before demo

---
 .github/workflows/k8s-regression-tester.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index de8351b869..42ca81af4b 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -4,6 +4,7 @@ on:
   release:
     types: [prereleased]
 
+# forward-looking, devs could manually trigger this action (via github UI) and easily spin up / stress test devnets for ad hoc testing
 jobs:
   deploy_devnet:
     runs-on: ubuntu-latest
@@ -73,10 +74,9 @@ jobs:
         DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
         DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
 
-        current_minute=$(date +'%M')
-        end_minute=$((current_minute + 10))
-        
-        while [ $current_minute -lt $end_minute ]; do
+        end_minute=$(( $(date +'%M') + 1))  # Calculate the end minute
+
+        while [ $(date +'%M') -lt $end_minute ]; do
           STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
                     -H "Content-Type: application/json" \
                     -H "DD-API-KEY: ${DATADOG_API_KEY}" \
@@ -85,7 +85,6 @@ jobs:
                     | jq -r '.overall_state')
           echo "K8S DEVNET HEALTH STATUS: $STATUS"
           echo "Review your devnet health dashboard here: https://app.datadoghq.com/dashboard/fqu-nh2-bzd?fromUser=false&refresh_mode=sliding&view=spans&from_ts=1709886652761&to_ts=1709890252761&live=true"
-          current_minute=$(date +'%M')
           sleep 60
         done
         

From 34a6892e30263de98242187ff50e0fe42273e01c Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 10:13:35 -0600
Subject: [PATCH 076/144] delete resources

---
 .github/workflows/k8s-regression-tester.yml | 102 ++++++++++----------
 1 file changed, 52 insertions(+), 50 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 42ca81af4b..82e92346be 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -48,60 +48,62 @@ jobs:
         ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY_PANOPTICHAIN }}'
         git clone -b dan/k8s_panoptichain git@github.com:0xPolygon/panoptichain.git
 
-    - name: Deploy PoS devnet to isolated GKE namespace
-      run: |
-        mkdir -p ~/.kube
-        echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/regression-cluster.yaml
-        export KUBECONFIG=~/.kube/regression-cluster.yaml
-        cd polygon-devnets/kubernetes/pos
-        kubectl apply -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
-        
-    - name: Deploy panoptichain, prometheus, grafana stack for metric tracking
-      run: |
-        cd panoptichain
-        export KUBECONFIG=~/.kube/regression-cluster.yaml
-        kubectl apply -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
-        
-    - name: Finally, deploy datadog-agent via helm chart
-      run: |
-        cd panoptichain/terraform
-        terraform init
-        terraform apply -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
+    # - name: Deploy PoS devnet to isolated GKE namespace
+    #   run: |
+    #     mkdir -p ~/.kube
+    #     echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/regression-cluster.yaml
+    #     export KUBECONFIG=~/.kube/regression-cluster.yaml
+    #     cd polygon-devnets/kubernetes/pos
+    #     kubectl apply -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+    # 
+    # - name: Deploy panoptichain, prometheus, grafana stack for metric tracking
+    #   run: |
+    #     cd panoptichain
+    #     export KUBECONFIG=~/.kube/regression-cluster.yaml
+    #     kubectl apply -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+    # 
+    # - name: Finally, deploy datadog-agent via helm chart
+    #   run: |
+    #     cd panoptichain/terraform
+    #     terraform init
+    #     terraform apply -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
+    # 
+    # - name: Auto-apply load and report any regressions here
+    #   run: |
+    #     MONITOR_ID="140974014"  # K8S DEVNET: COMPOSITE MONITOR ID
+    #     DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
+    #     DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
+    # 
+    #     end_minute=$(( $(date +'%M') + 1))  # Calculate the end minute
+    # 
+    #     while [ $(date +'%M') -lt $end_minute ]; do
+    #       STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
+    #                 -H "Content-Type: application/json" \
+    #                 -H "DD-API-KEY: ${DATADOG_API_KEY}" \
+    #                 -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
+    #                 -s \
+    #                 | jq -r '.overall_state')
+    #       echo "K8S DEVNET HEALTH STATUS: $STATUS"
+    #       echo "Review your devnet health dashboard here: https://app.datadoghq.com/dashboard/fqu-nh2-bzd?fromUser=false&refresh_mode=sliding&view=spans&from_ts=1709886652761&to_ts=1709890252761&live=true"
+    #       sleep 60
+    #     done
+    # 
+    #   # After stress tests, cleanup all ephemeral K8S devnet resources    
+    # - name: Remove panoptichain, prometheus, and grafana resources
+    #   run: |
+    #     cd panoptichain
+    #     export KUBECONFIG=~/.kube/regression-cluster.yaml
+    #     kubectl delete -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
 
-    - name: Auto-apply load and report any regressions here
-      run: |
-        MONITOR_ID="140974014"  # K8S DEVNET: COMPOSITE MONITOR ID
-        DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
-        DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
-
-        end_minute=$(( $(date +'%M') + 1))  # Calculate the end minute
-
-        while [ $(date +'%M') -lt $end_minute ]; do
-          STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
-                    -H "Content-Type: application/json" \
-                    -H "DD-API-KEY: ${DATADOG_API_KEY}" \
-                    -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
-                    -s \
-                    | jq -r '.overall_state')
-          echo "K8S DEVNET HEALTH STATUS: $STATUS"
-          echo "Review your devnet health dashboard here: https://app.datadoghq.com/dashboard/fqu-nh2-bzd?fromUser=false&refresh_mode=sliding&view=spans&from_ts=1709886652761&to_ts=1709890252761&live=true"
-          sleep 60
-        done
-        
-    - name: After stress tests, cleanup all ephemeral K8S devnet resources
-      run: |
-        # remove panoptichain, prometheus, and grafana resources
-        cd panoptichain
-        export KUBECONFIG=~/.kube/regression-cluster.yaml
-        kubectl delete -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
-        
-        # remove all datadog agent resources
+    - name: Remove all datadog agent resources
+      run: |        
         cd panoptichain/terraform
         terraform init
         terraform delete -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
-        
-        # finally, remove all pos devnet resources to return to clean slate GKE environment
-        export KUBECONFIG=~/.kube/regression-cluster.yaml
+
+    - name: Finally, remove pos devnet resources and return to clean slate GKE environment
+      run: |     
         cd polygon-devnets/kubernetes/pos
+        export KUBECONFIG=~/.kube/regression-cluster.yaml
         kubectl delete -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
 

From f727e25d5b189c3ed90c8ba0c515915a3596cc02 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 10:15:29 -0600
Subject: [PATCH 077/144] typo

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 82e92346be..d305aa911b 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -99,7 +99,7 @@ jobs:
       run: |        
         cd panoptichain/terraform
         terraform init
-        terraform delete -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
+        terraform destroy -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
 
     - name: Finally, remove pos devnet resources and return to clean slate GKE environment
       run: |     

From 478619ad25928248b3dc038634108cc41204dd16 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 10:18:23 -0600
Subject: [PATCH 078/144] cleanup

---
 .github/workflows/k8s-regression-tester.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index d305aa911b..285484f2f2 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -48,11 +48,11 @@ jobs:
         ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY_PANOPTICHAIN }}'
         git clone -b dan/k8s_panoptichain git@github.com:0xPolygon/panoptichain.git
 
-    # - name: Deploy PoS devnet to isolated GKE namespace
-    #   run: |
-    #     mkdir -p ~/.kube
-    #     echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/regression-cluster.yaml
-    #     export KUBECONFIG=~/.kube/regression-cluster.yaml
+    - name: Deploy PoS devnet to isolated GKE namespace
+      run: |
+        mkdir -p ~/.kube
+        echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/regression-cluster.yaml
+        export KUBECONFIG=~/.kube/regression-cluster.yaml
     #     cd polygon-devnets/kubernetes/pos
     #     kubectl apply -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
     # 

From a41996a45fd780de9e920f6a4c308adf98f0bf68 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 10:21:11 -0600
Subject: [PATCH 079/144] tf auto approve

---
 .github/workflows/k8s-regression-tester.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 285484f2f2..d573b84ea0 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -66,7 +66,7 @@ jobs:
     #   run: |
     #     cd panoptichain/terraform
     #     terraform init
-    #     terraform apply -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
+    #     terraform apply -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
     # 
     # - name: Auto-apply load and report any regressions here
     #   run: |
@@ -99,7 +99,7 @@ jobs:
       run: |        
         cd panoptichain/terraform
         terraform init
-        terraform destroy -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
+        terraform destroy -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
 
     - name: Finally, remove pos devnet resources and return to clean slate GKE environment
       run: |     

From ce029e1a75b070bac1750737bd06c48f9a329927 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 10:25:15 -0600
Subject: [PATCH 080/144] again

---
 .github/workflows/k8s-regression-tester.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index d573b84ea0..338fbf4811 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -96,7 +96,12 @@ jobs:
     #     kubectl delete -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
 
     - name: Remove all datadog agent resources
-      run: |        
+      register: tf_result
+      retries: 3
+      delay: 5
+      until: tf_result is not failed
+      ansible.builtin.shell:
+        cmd: |
         cd panoptichain/terraform
         terraform init
         terraform destroy -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
@@ -106,4 +111,3 @@ jobs:
         cd polygon-devnets/kubernetes/pos
         export KUBECONFIG=~/.kube/regression-cluster.yaml
         kubectl delete -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
-

From ea09ad448df11a5fff0dca7fd51993834afe0307 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 10:27:01 -0600
Subject: [PATCH 081/144] fmt

---
 .github/workflows/k8s-regression-tester.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 338fbf4811..c99c8d5298 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -102,9 +102,9 @@ jobs:
       until: tf_result is not failed
       ansible.builtin.shell:
         cmd: |
-        cd panoptichain/terraform
-        terraform init
-        terraform destroy -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
+          cd panoptichain/terraform
+          terraform init
+          terraform destroy -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
 
     - name: Finally, remove pos devnet resources and return to clean slate GKE environment
       run: |     

From 8ccbb463eec0e4c2a2391a621a9261e1c2119bf6 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 10:29:52 -0600
Subject: [PATCH 082/144] cmon

---
 .github/workflows/k8s-regression-tester.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index c99c8d5298..1939647329 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -100,14 +100,13 @@ jobs:
       retries: 3
       delay: 5
       until: tf_result is not failed
-      ansible.builtin.shell:
-        cmd: |
+      run: |    
           cd panoptichain/terraform
           terraform init
           terraform destroy -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
 
     - name: Finally, remove pos devnet resources and return to clean slate GKE environment
-      run: |     
+      run: |
         cd polygon-devnets/kubernetes/pos
         export KUBECONFIG=~/.kube/regression-cluster.yaml
         kubectl delete -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1

From 76f163fb217a27d516ea48cdb470ed12933f34ce Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 10:32:00 -0600
Subject: [PATCH 083/144] tests

---
 .github/workflows/k8s-regression-tester.yml | 27 ++++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 1939647329..462fe6a160 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -95,16 +95,25 @@ jobs:
     #     export KUBECONFIG=~/.kube/regression-cluster.yaml
     #     kubectl delete -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
 
-    - name: Remove all datadog agent resources
-      register: tf_result
-      retries: 3
-      delay: 5
-      until: tf_result is not failed
-      run: |    
-          cd panoptichain/terraform
+    - name: Remove all Datadog agent resources
+      env:
+        DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
+        CORALOGIX_API_KEY: ${{ secrets.CORALOGIX_API_KEY }}
+      run: |
+        cd panoptichain/terraform
+        retries=3
+        delay=5
+        count=0
+        while [ $count -lt $retries ]; do
           terraform init
-          terraform destroy -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
-
+          terraform destroy -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${DATADOG_API_KEY}" -var="coralogix_api_key=${CORALOGIX_API_KEY}"
+          if [ $? -eq 0 ]; then
+            break
+          fi
+          count=$((count+1))
+          sleep $delay
+        done
+    
     - name: Finally, remove pos devnet resources and return to clean slate GKE environment
       run: |
         cd polygon-devnets/kubernetes/pos

From c94ed5ea5ea1f7fb6ac2e45fa04da9efd1489096 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 10:37:01 -0600
Subject: [PATCH 084/144] cmon

---
 .github/workflows/k8s-regression-tester.yml | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 462fe6a160..760dcdab89 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -52,7 +52,7 @@ jobs:
       run: |
         mkdir -p ~/.kube
         echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/regression-cluster.yaml
-        export KUBECONFIG=~/.kube/regression-cluster.yaml
+    #     export KUBECONFIG=~/.kube/regression-cluster.yaml
     #     cd polygon-devnets/kubernetes/pos
     #     kubectl apply -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
     # 
@@ -102,16 +102,22 @@ jobs:
       run: |
         cd panoptichain/terraform
         retries=3
-        delay=5
+        delay=180
         count=0
         while [ $count -lt $retries ]; do
           terraform init
-          terraform destroy -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${DATADOG_API_KEY}" -var="coralogix_api_key=${CORALOGIX_API_KEY}"
-          if [ $? -eq 0 ]; then
+          destroy_output=$(terraform destroy -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${DATADOG_API_KEY}" -var="coralogix_api_key=${CORALOGIX_API_KEY}" 2>&1)
+          if echo "$destroy_output" | grep -q "Still destroying"; then
+            echo "Destroy operation is still in progress. Retrying in $delay seconds..."
+            sleep $delay
+          elif echo "$destroy_output" | grep -q "Apply complete! Resources: 0 added, 0 changed, 1 destroyed"; then
+            echo "Datadog agent resources successfully destroyed."
             break
+          else
+            echo "Destroy operation failed. Retrying in $delay seconds..."
+            count=$((count+1))
+            sleep $delay
           fi
-          count=$((count+1))
-          sleep $delay
         done
     
     - name: Finally, remove pos devnet resources and return to clean slate GKE environment

From dc45ed4d717e081cfa8a08d1e0cad28609cddf9d Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 10:44:42 -0600
Subject: [PATCH 085/144] pls

---
 .github/workflows/k8s-regression-tester.yml | 30 +++++++++------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 760dcdab89..8f5ce0b753 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -62,12 +62,12 @@ jobs:
     #     export KUBECONFIG=~/.kube/regression-cluster.yaml
     #     kubectl apply -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
     # 
-    # - name: Finally, deploy datadog-agent via helm chart
-    #   run: |
-    #     cd panoptichain/terraform
-    #     terraform init
-    #     terraform apply -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
-    # 
+    - name: Finally, deploy datadog-agent via helm chart
+      run: |
+        cd panoptichain/terraform
+        terraform init
+        terraform apply -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
+    
     # - name: Auto-apply load and report any regressions here
     #   run: |
     #     MONITOR_ID="140974014"  # K8S DEVNET: COMPOSITE MONITOR ID
@@ -87,7 +87,7 @@ jobs:
     #       echo "Review your devnet health dashboard here: https://app.datadoghq.com/dashboard/fqu-nh2-bzd?fromUser=false&refresh_mode=sliding&view=spans&from_ts=1709886652761&to_ts=1709890252761&live=true"
     #       sleep 60
     #     done
-    # 
+    
     #   # After stress tests, cleanup all ephemeral K8S devnet resources    
     # - name: Remove panoptichain, prometheus, and grafana resources
     #   run: |
@@ -102,22 +102,16 @@ jobs:
       run: |
         cd panoptichain/terraform
         retries=3
-        delay=180
+        delay=5
         count=0
         while [ $count -lt $retries ]; do
           terraform init
-          destroy_output=$(terraform destroy -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${DATADOG_API_KEY}" -var="coralogix_api_key=${CORALOGIX_API_KEY}" 2>&1)
-          if echo "$destroy_output" | grep -q "Still destroying"; then
-            echo "Destroy operation is still in progress. Retrying in $delay seconds..."
-            sleep $delay
-          elif echo "$destroy_output" | grep -q "Apply complete! Resources: 0 added, 0 changed, 1 destroyed"; then
-            echo "Datadog agent resources successfully destroyed."
+          terraform destroy -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${DATADOG_API_KEY}" -var="coralogix_api_key=${CORALOGIX_API_KEY}"
+          if [ $? -eq 0 ]; then
             break
-          else
-            echo "Destroy operation failed. Retrying in $delay seconds..."
-            count=$((count+1))
-            sleep $delay
           fi
+          count=$((count+1))
+          sleep $delay
         done
     
     - name: Finally, remove pos devnet resources and return to clean slate GKE environment

From 8092c417e71a4c04a7b53c878d6936a294613a1c Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 10:49:31 -0600
Subject: [PATCH 086/144] cmon

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 8f5ce0b753..747968faaf 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -95,7 +95,7 @@ jobs:
     #     export KUBECONFIG=~/.kube/regression-cluster.yaml
     #     kubectl delete -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
 
-    - name: Remove all Datadog agent resources
+    - name: Remove all Datadog agent resources 
       env:
         DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
         CORALOGIX_API_KEY: ${{ secrets.CORALOGIX_API_KEY }}

From 6002df9b6ed27ca9c3c07baa691723922f0c5cf4 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 11:12:19 -0600
Subject: [PATCH 087/144] full e2e k8s devnet tests

---
 .github/workflows/k8s-regression-tester.yml | 70 ++++++++++-----------
 1 file changed, 35 insertions(+), 35 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 747968faaf..261b812865 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -52,48 +52,48 @@ jobs:
       run: |
         mkdir -p ~/.kube
         echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/regression-cluster.yaml
-    #     export KUBECONFIG=~/.kube/regression-cluster.yaml
-    #     cd polygon-devnets/kubernetes/pos
-    #     kubectl apply -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
-    # 
-    # - name: Deploy panoptichain, prometheus, grafana stack for metric tracking
-    #   run: |
-    #     cd panoptichain
-    #     export KUBECONFIG=~/.kube/regression-cluster.yaml
-    #     kubectl apply -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
-    # 
+        export KUBECONFIG=~/.kube/regression-cluster.yaml
+        cd polygon-devnets/kubernetes/pos
+        kubectl apply -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+    
+    - name: Deploy panoptichain, prometheus, grafana stack for metric tracking
+      run: |
+        cd panoptichain
+        export KUBECONFIG=~/.kube/regression-cluster.yaml
+        kubectl apply -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+    
     - name: Finally, deploy datadog-agent via helm chart
       run: |
         cd panoptichain/terraform
         terraform init
         terraform apply -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
     
-    # - name: Auto-apply load and report any regressions here
-    #   run: |
-    #     MONITOR_ID="140974014"  # K8S DEVNET: COMPOSITE MONITOR ID
-    #     DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
-    #     DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
-    # 
-    #     end_minute=$(( $(date +'%M') + 1))  # Calculate the end minute
-    # 
-    #     while [ $(date +'%M') -lt $end_minute ]; do
-    #       STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
-    #                 -H "Content-Type: application/json" \
-    #                 -H "DD-API-KEY: ${DATADOG_API_KEY}" \
-    #                 -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
-    #                 -s \
-    #                 | jq -r '.overall_state')
-    #       echo "K8S DEVNET HEALTH STATUS: $STATUS"
-    #       echo "Review your devnet health dashboard here: https://app.datadoghq.com/dashboard/fqu-nh2-bzd?fromUser=false&refresh_mode=sliding&view=spans&from_ts=1709886652761&to_ts=1709890252761&live=true"
-    #       sleep 60
-    #     done
+    - name: Auto-apply load and report any regressions here
+      run: |
+        MONITOR_ID="140974014"  # K8S DEVNET: COMPOSITE MONITOR ID
+        DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
+        DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
+    
+        end_minute=$(( $(date +'%M') + 1))  # Calculate the end minute
+    
+        while [ $(date +'%M') -lt $end_minute ]; do
+          STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
+                    -H "Content-Type: application/json" \
+                    -H "DD-API-KEY: ${DATADOG_API_KEY}" \
+                    -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
+                    -s \
+                    | jq -r '.overall_state')
+          echo "K8S DEVNET HEALTH STATUS: $STATUS"
+          echo "Review your devnet health dashboard here: https://app.datadoghq.com/dashboard/fqu-nh2-bzd?fromUser=false&refresh_mode=sliding&view=spans&from_ts=1709886652761&to_ts=1709890252761&live=true"
+          sleep 60
+        done
     
-    #   # After stress tests, cleanup all ephemeral K8S devnet resources    
-    # - name: Remove panoptichain, prometheus, and grafana resources
-    #   run: |
-    #     cd panoptichain
-    #     export KUBECONFIG=~/.kube/regression-cluster.yaml
-    #     kubectl delete -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+      # After stress tests, cleanup all ephemeral K8S devnet resources    
+    - name: Remove panoptichain, prometheus, and grafana resources
+      run: |
+        cd panoptichain
+        export KUBECONFIG=~/.kube/regression-cluster.yaml
+        kubectl delete -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
 
     - name: Remove all Datadog agent resources 
       env:

From 167566ec3715b4565d773fd0dae80f1f77d3b531 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 11:18:04 -0600
Subject: [PATCH 088/144] e2e k8s pos devnet

---
 .github/workflows/k8s-regression-tester.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 261b812865..233850f891 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -55,18 +55,21 @@ jobs:
         export KUBECONFIG=~/.kube/regression-cluster.yaml
         cd polygon-devnets/kubernetes/pos
         kubectl apply -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+        sleep 60
     
     - name: Deploy panoptichain, prometheus, grafana stack for metric tracking
       run: |
         cd panoptichain
         export KUBECONFIG=~/.kube/regression-cluster.yaml
         kubectl apply -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+        sleep 60
     
     - name: Finally, deploy datadog-agent via helm chart
       run: |
         cd panoptichain/terraform
         terraform init
         terraform apply -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
+        sleep 60
     
     - name: Auto-apply load and report any regressions here
       run: |

From 0cdd3880b52c40116e2403833c9ebe06015aa647 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 8 Mar 2024 11:42:11 -0600
Subject: [PATCH 089/144] full e2e demo w/ john

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 233850f891..1598c039ce 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -77,7 +77,7 @@ jobs:
         DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
         DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
     
-        end_minute=$(( $(date +'%M') + 1))  # Calculate the end minute
+        end_minute=$(( $(date +'%M') + 10))  # Calculate the end minute, bake time, make configurable..
     
         while [ $(date +'%M') -lt $end_minute ]; do
           STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \

From bb0c86b44c2c2196f92193d779c13211b1c650d1 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Sat, 9 Mar 2024 06:20:10 -0600
Subject: [PATCH 090/144] feat: github ui dispatch

---
 .github/workflows/k8s-regression-tester.yml | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 1598c039ce..16b35b5f31 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -1,8 +1,14 @@
 name: K8S DEVNET - REGRESSION DETECTOR
 on:
-  push:
-  release:
-    types: [prereleased]
+  workflow_dispatch:
+    inputs:
+      bor_version:
+        description: 'bor version (default: latest)'
+        required: false
+        default: 'latest'
+      bake_time:
+        description: 'bake time in minutes'
+        required: true
 
 # forward-looking, devs could manually trigger this action (via github UI) and easily spin up / stress test devnets for ad hoc testing
 jobs:
@@ -76,9 +82,10 @@ jobs:
         MONITOR_ID="140974014"  # K8S DEVNET: COMPOSITE MONITOR ID
         DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
         DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
-    
-        end_minute=$(( $(date +'%M') + 10))  # Calculate the end minute, bake time, make configurable..
-    
+
+        bake_time="${{ github.event.inputs.bake_time }}"
+        end_minute=$(( $(date +'%M') + bake_time))
+
         while [ $(date +'%M') -lt $end_minute ]; do
           STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
                     -H "Content-Type: application/json" \

From 74101a63e498e5ad0ae12034599f185103276588 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Sat, 9 Mar 2024 06:33:44 -0600
Subject: [PATCH 091/144] fix: final cleanup

---
 .github/workflows/k8s-regression-tester.yml | 1 -
 .github/workflows/make-k8s-images.yml       | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 16b35b5f31..5f97b1476f 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -10,7 +10,6 @@ on:
         description: 'bake time in minutes'
         required: true
 
-# forward-looking, devs could manually trigger this action (via github UI) and easily spin up / stress test devnets for ad hoc testing
 jobs:
   deploy_devnet:
     runs-on: ubuntu-latest
diff --git a/.github/workflows/make-k8s-images.yml b/.github/workflows/make-k8s-images.yml
index 948334eb88..ee12cb9f0a 100644
--- a/.github/workflows/make-k8s-images.yml
+++ b/.github/workflows/make-k8s-images.yml
@@ -2,7 +2,7 @@ name: Image Sync
 
 on:
   schedule:
-    - cron: '*/15 * * * *'
+    - cron: '*/30 * * * *'
 
 jobs:
   sync-images:

From de25b5df31757391cdef8f209cc5ecb11f038477 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Sat, 9 Mar 2024 06:34:41 -0600
Subject: [PATCH 092/144] fix: workflow title

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 5f97b1476f..edf4d060b4 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -1,4 +1,4 @@
-name: K8S DEVNET - REGRESSION DETECTOR
+name: K8S DEVNETS - REGRESSION DETECTOR
 on:
   workflow_dispatch:
     inputs:

From 15ef265e721f362f0f86415b65dd0bc0680db0d0 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 20 Mar 2024 12:38:31 -0500
Subject: [PATCH 093/144] feature: kurtosis ci devnet deploys

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 74 +++++++++++++++++++
 kurtosis-config.yml                           | 17 +++++
 2 files changed, 91 insertions(+)
 create mode 100644 .github/workflows/cdk-kurtosis-gke-devnet.yml
 create mode 100644 kurtosis-config.yml

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
new file mode 100644
index 0000000000..6525df3fa6
--- /dev/null
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -0,0 +1,74 @@
+name: K8S DEVNETS - CDK
+on:
+  push
+
+jobs:
+  deploy_devnet:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.before }}
+
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v1
+
+    - name: Setup Google Cloud SDK
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        version: '>= 363.0.0'
+
+    - uses: 'google-github-actions/auth@v2'
+      with:
+        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+        project_id: 'prj-polygonlabs-devtools-dev'
+
+    - name: Install kubectl and gcloud kubectl auth plugin
+      run: |
+        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
+        && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && gcloud components install gke-gcloud-auth-plugin
+
+    - name: Install tomlq for config file preparation
+      run: |
+        pip3 install tomlq
+
+    - name: Clone private kurtosis-cdk repo
+      run: |
+        eval `ssh-agent -s`
+        ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
+        git clone -b main git@github.com:0xPolygon/kurtosis-cdk.git
+        
+## add necessary manipulation steps here...
+    - name: Install kurtosis
+      run: |
+        echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+        sudo apt update
+        sudo apt install kurtosis-cli
+
+    - name: Setup necessary gke custom kurtosis netorking configs
+      run: |        
+        export KURTOSIS_CONFIG_PATH="$(kurtosis config path)"
+        cp /home/runner/work/bor/kurtosis-config.yaml $KURTOSIS_CONFIG_PATH
+        
+    - name: Set kurtosis cloud env
+      run: |       
+        mkdir -p ~/.kube
+        echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/config     
+        export KUBECONFIG=~/.kube/config && kurtosis cluster set cloud
+
+    - name: Run kurtosis agent in background
+      run: |  
+        kurtosis gateway &  # Run the command in the background
+        sleep 30
+
+    - name: Deploy CDK devnet to isolated GKE namespace
+      run: |
+        cd kurtosis-cdk
+        kurtosis run --enclave cdk --args-file cdk/params.yml cdk
+        sleep 60
+    
+    - name: Finally, remove all devnet resources
+      run: |
+        cd kurtosis-cdk
+        kurtosis clean -a
diff --git a/kurtosis-config.yml b/kurtosis-config.yml
new file mode 100644
index 0000000000..81e132cc95
--- /dev/null
+++ b/kurtosis-config.yml
@@ -0,0 +1,17 @@
+config-version: 2
+should-send-metrics: true
+kurtosis-clusters:
+  docker:
+    type: "docker"
+  minikube:
+    type: "kubernetes"
+    config:
+      kubernetes-cluster-name: "minikube"
+      storage-class: "standard"
+      enclave-size-in-megabytes: 10
+  cloud:
+    type: "kubernetes"
+    config:
+      kubernetes-cluster-name: "gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1"
+      storage-class: "standard"
+      enclave-size-in-megabytes: 5000

From d2769df7028caadb53f2e0b210fed2cc3ef0dbc0 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Thu, 21 Mar 2024 04:07:41 -0500
Subject: [PATCH 094/144] test

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index 6525df3fa6..04724fe9ec 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -49,6 +49,10 @@ jobs:
     - name: Setup necessary gke custom kurtosis netorking configs
       run: |        
         export KURTOSIS_CONFIG_PATH="$(kurtosis config path)"
+        pwd
+        ls -a
+        cd bor
+        ls -a
         cp /home/runner/work/bor/kurtosis-config.yaml $KURTOSIS_CONFIG_PATH
         
     - name: Set kurtosis cloud env

From ad13f96dd7f6798b238a10c8263909b2355d3ab2 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Thu, 21 Mar 2024 04:22:39 -0500
Subject: [PATCH 095/144] fix repo path

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index 04724fe9ec..0667933635 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -53,7 +53,7 @@ jobs:
         ls -a
         cd bor
         ls -a
-        cp /home/runner/work/bor/kurtosis-config.yaml $KURTOSIS_CONFIG_PATH
+        cp /home/runner/work/bor/bor/kurtosis-config.yaml $KURTOSIS_CONFIG_PATH
         
     - name: Set kurtosis cloud env
       run: |       

From 5d19a5111dcf115ec1bbb4dcc6577b3a7b01ab1d Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Thu, 21 Mar 2024 04:29:04 -0500
Subject: [PATCH 096/144] finetune repo path

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index 0667933635..fb20d70a01 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -51,8 +51,6 @@ jobs:
         export KURTOSIS_CONFIG_PATH="$(kurtosis config path)"
         pwd
         ls -a
-        cd bor
-        ls -a
         cp /home/runner/work/bor/bor/kurtosis-config.yaml $KURTOSIS_CONFIG_PATH
         
     - name: Set kurtosis cloud env

From f940ba062baf1b70b6a04f7621a40c5a52be4b79 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Thu, 21 Mar 2024 04:31:21 -0500
Subject: [PATCH 097/144] fix: filename

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index fb20d70a01..83c32571af 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -51,7 +51,7 @@ jobs:
         export KURTOSIS_CONFIG_PATH="$(kurtosis config path)"
         pwd
         ls -a
-        cp /home/runner/work/bor/bor/kurtosis-config.yaml $KURTOSIS_CONFIG_PATH
+        cp /home/runner/work/bor/bor/kurtosis-config.yml $KURTOSIS_CONFIG_PATH
         
     - name: Set kurtosis cloud env
       run: |       

From 4cc2abb24a4c3c588055839dd16402827e955078 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Thu, 21 Mar 2024 04:40:16 -0500
Subject: [PATCH 098/144] fix: kurtosis engine restart

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index 83c32571af..a0562c97f0 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -49,8 +49,6 @@ jobs:
     - name: Setup necessary gke custom kurtosis netorking configs
       run: |        
         export KURTOSIS_CONFIG_PATH="$(kurtosis config path)"
-        pwd
-        ls -a
         cp /home/runner/work/bor/bor/kurtosis-config.yml $KURTOSIS_CONFIG_PATH
         
     - name: Set kurtosis cloud env
@@ -67,6 +65,7 @@ jobs:
     - name: Deploy CDK devnet to isolated GKE namespace
       run: |
         cd kurtosis-cdk
+        kurtosis engine restart
         kurtosis run --enclave cdk --args-file cdk/params.yml cdk
         sleep 60
     

From 4d9e2c10380b991590db408d29d2d672e1f0ec62 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Thu, 21 Mar 2024 05:07:33 -0500
Subject: [PATCH 099/144] debug: reduce size of enclave

---
 kurtosis-config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kurtosis-config.yml b/kurtosis-config.yml
index 81e132cc95..9d2da62899 100644
--- a/kurtosis-config.yml
+++ b/kurtosis-config.yml
@@ -14,4 +14,4 @@ kurtosis-clusters:
     config:
       kubernetes-cluster-name: "gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1"
       storage-class: "standard"
-      enclave-size-in-megabytes: 5000
+      enclave-size-in-megabytes: 500

From f7dd70e2aa9285a1f1ea098ff715cdf8a2e134b4 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Thu, 21 Mar 2024 05:17:14 -0500
Subject: [PATCH 100/144] debug: pvcs

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index a0562c97f0..f3debf6968 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -61,6 +61,11 @@ jobs:
       run: |  
         kurtosis gateway &  # Run the command in the background
         sleep 30
+        
+    - name: TEMP remove all devnet resources
+      run: |
+        cd kurtosis-cdk
+        kurtosis clean -a
 
     - name: Deploy CDK devnet to isolated GKE namespace
       run: |

From 2e79f194f73cb51261715a770bd8e83d88243310 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Thu, 21 Mar 2024 16:59:11 -0500
Subject: [PATCH 101/144] refactor kurtosis run cmd per latest cdk-devnets main
 branch

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index f3debf6968..3e0e4d3671 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -59,20 +59,15 @@ jobs:
 
     - name: Run kurtosis agent in background
       run: |  
-        kurtosis gateway &  # Run the command in the background
-        sleep 30
-        
-    - name: TEMP remove all devnet resources
-      run: |
-        cd kurtosis-cdk
-        kurtosis clean -a
+        kurtosis gateway &  # Run cmd in background
+        sleep 10
 
     - name: Deploy CDK devnet to isolated GKE namespace
       run: |
         cd kurtosis-cdk
         kurtosis engine restart
-        kurtosis run --enclave cdk --args-file cdk/params.yml cdk
-        sleep 60
+        kurtosis run --enclave cdk-v1 --args-file params.yml .
+        sleep 600
     
     - name: Finally, remove all devnet resources
       run: |

From f1b55e0da5a2bbc7086c3be7a0c8330a2cafd285 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 26 Mar 2024 03:59:59 -0500
Subject: [PATCH 102/144] debug: contracts container state error

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 2 +-
 kurtosis-config.yml                           | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index 3e0e4d3671..11a2c9e02f 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -37,7 +37,7 @@ jobs:
       run: |
         eval `ssh-agent -s`
         ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
-        git clone -b main git@github.com:0xPolygon/kurtosis-cdk.git
+        git clone -b dan/contracts_running_state git@github.com:0xPolygon/kurtosis-cdk.git
         
 ## add necessary manipulation steps here...
     - name: Install kurtosis
diff --git a/kurtosis-config.yml b/kurtosis-config.yml
index 9d2da62899..ef9f6a2410 100644
--- a/kurtosis-config.yml
+++ b/kurtosis-config.yml
@@ -8,10 +8,10 @@ kurtosis-clusters:
     config:
       kubernetes-cluster-name: "minikube"
       storage-class: "standard"
-      enclave-size-in-megabytes: 10
+      enclave-size-in-megabytes: 2000
   cloud:
     type: "kubernetes"
     config:
       kubernetes-cluster-name: "gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1"
       storage-class: "standard"
-      enclave-size-in-megabytes: 500
+      enclave-size-in-megabytes: 2000

From ba8035a2a4d02f4dfff6b8d78ec507a3ae3261d4 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 26 Mar 2024 04:10:30 -0500
Subject: [PATCH 103/144] debug: container fail

---
 kurtosis-config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/kurtosis-config.yml b/kurtosis-config.yml
index ef9f6a2410..2f227b16ef 100644
--- a/kurtosis-config.yml
+++ b/kurtosis-config.yml
@@ -8,10 +8,10 @@ kurtosis-clusters:
     config:
       kubernetes-cluster-name: "minikube"
       storage-class: "standard"
-      enclave-size-in-megabytes: 2000
+      enclave-size-in-megabytes: 400
   cloud:
     type: "kubernetes"
     config:
       kubernetes-cluster-name: "gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1"
       storage-class: "standard"
-      enclave-size-in-megabytes: 2000
+      enclave-size-in-megabytes: 400

From 31c605612138d0ec1eaa46fdd546c91cd73d8919 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 26 Mar 2024 04:25:58 -0500
Subject: [PATCH 104/144] debug

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index 11a2c9e02f..2d50449574 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -61,6 +61,11 @@ jobs:
       run: |  
         kurtosis gateway &  # Run cmd in background
         sleep 10
+        
+    - name: REMOVE, for CI testing, remove all devnet resources
+      run: |
+        cd kurtosis-cdk
+        kurtosis clean -a
 
     - name: Deploy CDK devnet to isolated GKE namespace
       run: |

From d34db133660bc2956a404037717802e5a7dfe64f Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 26 Mar 2024 04:32:16 -0500
Subject: [PATCH 105/144] debug: cdk starlark

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index 2d50449574..ad135748de 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -62,10 +62,11 @@ jobs:
         kurtosis gateway &  # Run cmd in background
         sleep 10
         
-    - name: REMOVE, for CI testing, remove all devnet resources
+    - name: REMOVE, tmp for CI testing remove all devnet resources
       run: |
         cd kurtosis-cdk
         kurtosis clean -a
+        sleep 180
 
     - name: Deploy CDK devnet to isolated GKE namespace
       run: |

From c9ead2c51f09aa1d542447a5a7023a9acd7f396a Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 26 Mar 2024 04:57:58 -0500
Subject: [PATCH 106/144] debug

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index ad135748de..6baab6c848 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -62,7 +62,7 @@ jobs:
         kurtosis gateway &  # Run cmd in background
         sleep 10
         
-    - name: REMOVE, tmp for CI testing remove all devnet resources
+    - name: REMOVE, temp for CI testing remove all devnet resources
       run: |
         cd kurtosis-cdk
         kurtosis clean -a

From 913960d6fabc567b80aad6dc009af375e523b6be Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 5 Apr 2024 07:26:20 -0500
Subject: [PATCH 107/144] debug JIT agglayer images

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml |   2 +-
 .github/workflows/k8s-regression-tester.yml   | 126 ++++++------------
 2 files changed, 45 insertions(+), 83 deletions(-)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index 6baab6c848..894d9a2dd4 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -1,4 +1,4 @@
-name: K8S DEVNETS - CDK
+name: Regression Detector - Ephemeral CDK Devnet
 on:
   push
 
diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index edf4d060b4..40201b2050 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -1,13 +1,12 @@
-name: K8S DEVNETS - REGRESSION DETECTOR
+name: REGRESSION DETECTOR - EPHEMERAL DEVNETS
 on:
   workflow_dispatch:
     inputs:
-      bor_version:
-        description: 'bor version (default: latest)'
-        required: false
-        default: 'latest'
+      commit_id:
+        description: 'agglayer version (commit id)'
+        required: true
       bake_time:
-        description: 'bake time in minutes'
+        description: 'bake time (minutes)'
         required: true
 
 jobs:
@@ -41,90 +40,53 @@ jobs:
       run: |
         pip3 install tomlq
 
-    - name: Clone private polygon-devnets repo
+    - name: Clone internal kurtosis-cdk repo
       run: |
-        eval `ssh-agent -s`
-        ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
-        git clone -b fix/pos_k8s_devnets git@github.com:maticnetwork/polygon-devnets.git
+        git clone git@github.com:0xPolygon/kurtosis-cdk.git
+        cd kurtosis-cdk
+        git checkout dan/contracts_running_state
         
-    - name: Clone private panoptichain repo
-      run: |
-        eval `ssh-agent -s`
-        ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY_PANOPTICHAIN }}'
-        git clone -b dan/k8s_panoptichain git@github.com:0xPolygon/panoptichain.git
-
-    - name: Deploy PoS devnet to isolated GKE namespace
+    - name: Clone internal agglayer repo
       run: |
-        mkdir -p ~/.kube
-        echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/regression-cluster.yaml
-        export KUBECONFIG=~/.kube/regression-cluster.yaml
-        cd polygon-devnets/kubernetes/pos
-        kubectl apply -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
-        sleep 60
-    
-    - name: Deploy panoptichain, prometheus, grafana stack for metric tracking
-      run: |
-        cd panoptichain
-        export KUBECONFIG=~/.kube/regression-cluster.yaml
-        kubectl apply -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
-        sleep 60
-    
-    - name: Finally, deploy datadog-agent via helm chart
+        git clone git@github.com:0xPolygon/agglayer.git
+        cd agglayer
+        git checkout "${{ github.event.inputs.commit_id }}"
+        
+    - name: Build JIT agglayer image to be used in ephemeral devnet
       run: |
-        cd panoptichain/terraform
-        terraform init
-        terraform apply -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${{ secrets.DATADOG_API_KEY }}" -var="coralogix_api_key=${{ secrets.CORALOGIX_API_KEY }}"
-        sleep 60
-    
-    - name: Auto-apply load and report any regressions here
+        cd agglayer
+        make build-docker
+        
+    - name: Install kurtosis
       run: |
-        MONITOR_ID="140974014"  # K8S DEVNET: COMPOSITE MONITOR ID
-        DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
-        DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
+        echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+        sudo apt update
+        sudo apt install kurtosis-cli
 
-        bake_time="${{ github.event.inputs.bake_time }}"
-        end_minute=$(( $(date +'%M') + bake_time))
+    - name: Setup necessary gke custom kurtosis netorking configs
+      run: |        
+        export KURTOSIS_CONFIG_PATH="$(kurtosis config path)"
+        cp /home/runner/work/bor/bor/kurtosis-config.yml $KURTOSIS_CONFIG_PATH
+        
+    - name: Set kurtosis cloud env
+      run: |       
+        mkdir -p ~/.kube
+        echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/config     
+        export KUBECONFIG=~/.kube/config && kurtosis cluster set cloud
 
-        while [ $(date +'%M') -lt $end_minute ]; do
-          STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
-                    -H "Content-Type: application/json" \
-                    -H "DD-API-KEY: ${DATADOG_API_KEY}" \
-                    -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
-                    -s \
-                    | jq -r '.overall_state')
-          echo "K8S DEVNET HEALTH STATUS: $STATUS"
-          echo "Review your devnet health dashboard here: https://app.datadoghq.com/dashboard/fqu-nh2-bzd?fromUser=false&refresh_mode=sliding&view=spans&from_ts=1709886652761&to_ts=1709890252761&live=true"
-          sleep 60
-        done
-    
-      # After stress tests, cleanup all ephemeral K8S devnet resources    
-    - name: Remove panoptichain, prometheus, and grafana resources
-      run: |
-        cd panoptichain
-        export KUBECONFIG=~/.kube/regression-cluster.yaml
-        kubectl delete -f grafana-service.yaml,panoptichain-service.yaml,prometheus-service.yaml,grafana-deployment.yaml,grafana-claim0-persistentvolumeclaim.yaml,panoptichain-deployment.yaml,prometheus-deployment.yaml,prometheus-claim0-persistentvolumeclaim.yaml --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+    - name: Run kurtosis agent in background
+      run: |  
+        kurtosis gateway &  # Run cmd in background
+        sleep 10
 
-    - name: Remove all Datadog agent resources 
-      env:
-        DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
-        CORALOGIX_API_KEY: ${{ secrets.CORALOGIX_API_KEY }}
+    - name: Deploy CDK devnet to isolated GKE namespace
       run: |
-        cd panoptichain/terraform
-        retries=3
-        delay=5
-        count=0
-        while [ $count -lt $retries ]; do
-          terraform init
-          terraform destroy -auto-approve -target=helm_release.datadog_agent -var="datadog_api_key=${DATADOG_API_KEY}" -var="coralogix_api_key=${CORALOGIX_API_KEY}"
-          if [ $? -eq 0 ]; then
-            break
-          fi
-          count=$((count+1))
-          sleep $delay
-        done
+        cd kurtosis-cdk
+        kurtosis engine restart
+        kurtosis run --enclave cdk-v1 --args-file params.yml .
+        sleep 300
     
-    - name: Finally, remove pos devnet resources and return to clean slate GKE environment
+    - name: Finally, remove all devnet resources
       run: |
-        cd polygon-devnets/kubernetes/pos
-        export KUBECONFIG=~/.kube/regression-cluster.yaml
-        kubectl delete -k overlays/gcr --namespace=pos --context=gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1
+        cd kurtosis-cdk
+        kurtosis clean -a

From 131557d325f1d9831361f4b7e198e201c6663502 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 5 Apr 2024 07:40:21 -0500
Subject: [PATCH 108/144] work

---
 .github/workflows/cdk-kurtosis-gke-devnet.yml | 2 --
 .github/workflows/k8s-regression-tester.yml   | 4 ++--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/cdk-kurtosis-gke-devnet.yml b/.github/workflows/cdk-kurtosis-gke-devnet.yml
index 894d9a2dd4..43eae863ee 100644
--- a/.github/workflows/cdk-kurtosis-gke-devnet.yml
+++ b/.github/workflows/cdk-kurtosis-gke-devnet.yml
@@ -1,6 +1,4 @@
 name: Regression Detector - Ephemeral CDK Devnet
-on:
-  push
 
 jobs:
   deploy_devnet:
diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 40201b2050..ae67f3f43c 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -42,13 +42,13 @@ jobs:
 
     - name: Clone internal kurtosis-cdk repo
       run: |
-        git clone git@github.com:0xPolygon/kurtosis-cdk.git
+        git clone https://github.com/0xPolygon/kurtosis-cdk.git
         cd kurtosis-cdk
         git checkout dan/contracts_running_state
         
     - name: Clone internal agglayer repo
       run: |
-        git clone git@github.com:0xPolygon/agglayer.git
+        git clone https://github.com/0xPolygon/agglayer.git
         cd agglayer
         git checkout "${{ github.event.inputs.commit_id }}"
         

From e6d8dad8aa92a6292ef659306c9a2c2d97c94d8c Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 5 Apr 2024 10:08:19 -0500
Subject: [PATCH 109/144] run local

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index ae67f3f43c..955f47c0c8 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -44,7 +44,7 @@ jobs:
       run: |
         git clone https://github.com/0xPolygon/kurtosis-cdk.git
         cd kurtosis-cdk
-        git checkout dan/contracts_running_state
+        git checkout dan/jit_containers
         
     - name: Clone internal agglayer repo
       run: |

From 361ddddedd2367302b5e969271b8238e31dc29a9 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Fri, 5 Apr 2024 10:30:50 -0500
Subject: [PATCH 110/144] again

---
 .github/workflows/k8s-regression-tester.yml | 22 ++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 955f47c0c8..2987ef5252 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -1,4 +1,4 @@
-name: REGRESSION DETECTOR - EPHEMERAL DEVNETS
+name: REGRESSION DETECTOR - DEVNET
 on:
   workflow_dispatch:
     inputs:
@@ -63,16 +63,16 @@ jobs:
         sudo apt update
         sudo apt install kurtosis-cli
 
-    - name: Setup necessary gke custom kurtosis netorking configs
-      run: |        
-        export KURTOSIS_CONFIG_PATH="$(kurtosis config path)"
-        cp /home/runner/work/bor/bor/kurtosis-config.yml $KURTOSIS_CONFIG_PATH
-        
-    - name: Set kurtosis cloud env
-      run: |       
-        mkdir -p ~/.kube
-        echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/config     
-        export KUBECONFIG=~/.kube/config && kurtosis cluster set cloud
+    # - name: Setup necessary gke custom kurtosis netorking configs
+    #   run: |        
+    #     export KURTOSIS_CONFIG_PATH="$(kurtosis config path)"
+    #     cp /home/runner/work/bor/bor/kurtosis-config.yml $KURTOSIS_CONFIG_PATH
+    # 
+    # - name: Set kurtosis cloud env
+    #   run: |       
+    #     mkdir -p ~/.kube
+    #     echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/config     
+    #     export KUBECONFIG=~/.kube/config && kurtosis cluster set cloud
 
     - name: Run kurtosis agent in background
       run: |  

From 9a4f046ecb9906425dae6a3bd35325b60f572661 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 05:53:56 -0500
Subject: [PATCH 111/144] debug

---
 .github/workflows/k8s-regression-tester.yml | 19 ++++---------------
 1 file changed, 4 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 2987ef5252..b5254f12e8 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -54,8 +54,8 @@ jobs:
         
     - name: Build JIT agglayer image to be used in ephemeral devnet
       run: |
-        cd agglayer
-        make build-docker
+        docker compose -f agglayer/docker/docker-compose.yaml build --no-cache agglayer
+        sleep 10
         
     - name: Install kurtosis
       run: |
@@ -63,30 +63,19 @@ jobs:
         sudo apt update
         sudo apt install kurtosis-cli
 
-    # - name: Setup necessary gke custom kurtosis netorking configs
-    #   run: |        
-    #     export KURTOSIS_CONFIG_PATH="$(kurtosis config path)"
-    #     cp /home/runner/work/bor/bor/kurtosis-config.yml $KURTOSIS_CONFIG_PATH
-    # 
-    # - name: Set kurtosis cloud env
-    #   run: |       
-    #     mkdir -p ~/.kube
-    #     echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/config     
-    #     export KUBECONFIG=~/.kube/config && kurtosis cluster set cloud
-
     - name: Run kurtosis agent in background
       run: |  
         kurtosis gateway &  # Run cmd in background
         sleep 10
 
-    - name: Deploy CDK devnet to isolated GKE namespace
+    - name: Deploy CDK devnet on local github runner
       run: |
         cd kurtosis-cdk
         kurtosis engine restart
         kurtosis run --enclave cdk-v1 --args-file params.yml .
         sleep 300
     
-    - name: Finally, remove all devnet resources
+    - name: Finally, remove all devnet resources locally
       run: |
         cd kurtosis-cdk
         kurtosis clean -a

From 9083ecb9d1f8a155c5a5cff2511abe810a2a0a2b Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 05:56:03 -0500
Subject: [PATCH 112/144] debu

---
 .github/workflows/k8s-regression-tester.yml | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index b5254f12e8..8ce407ad6a 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -1,13 +1,14 @@
 name: REGRESSION DETECTOR - DEVNET
 on:
-  workflow_dispatch:
-    inputs:
-      commit_id:
-        description: 'agglayer version (commit id)'
-        required: true
-      bake_time:
-        description: 'bake time (minutes)'
-        required: true
+  push:
+  # workflow_dispatch:
+  #   inputs:
+  #     commit_id:
+  #       description: 'agglayer version (commit id)'
+  #       required: true
+  #     bake_time:
+  #       description: 'bake time (minutes)'
+  #       required: true
 
 jobs:
   deploy_devnet:

From bea256b722cefb65a64c5be9ade35e0f15c4cae4 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 05:59:29 -0500
Subject: [PATCH 113/144] debug

---
 .github/workflows/k8s-regression-tester.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 8ce407ad6a..74eeffd477 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -51,7 +51,8 @@ jobs:
       run: |
         git clone https://github.com/0xPolygon/agglayer.git
         cd agglayer
-        git checkout "${{ github.event.inputs.commit_id }}"
+        # git checkout "${{ github.event.inputs.commit_id }}"
+        git checkout c0a88f1
         
     - name: Build JIT agglayer image to be used in ephemeral devnet
       run: |

From 70d5eb4bd098fbf9b1b5cab2d54054a2532108cb Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 06:24:41 -0500
Subject: [PATCH 114/144] degug

---
 .github/workflows/k8s-regression-tester.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/k8s-regression-tester.yml
index 74eeffd477..236277fcab 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/k8s-regression-tester.yml
@@ -54,7 +54,7 @@ jobs:
         # git checkout "${{ github.event.inputs.commit_id }}"
         git checkout c0a88f1
         
-    - name: Build JIT agglayer image to be used in ephemeral devnet
+    - name: Build JIT agglayer image to be used in ephemeral devnets
       run: |
         docker compose -f agglayer/docker/docker-compose.yaml build --no-cache agglayer
         sleep 10

From 4a2483706cd586d2bf5ef11ff812385549b6301e Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 16:52:14 -0500
Subject: [PATCH 115/144] superuser cdk devnet tool

---
 ...ster.yml => agglayer-regression-tests.yml} |  46 +++--
 .../cdk-superuser-regression-tests.yml        | 168 ++++++++++++++++++
 2 files changed, 200 insertions(+), 14 deletions(-)
 rename .github/workflows/{k8s-regression-tester.yml => agglayer-regression-tests.yml} (61%)
 create mode 100644 .github/workflows/cdk-superuser-regression-tests.yml

diff --git a/.github/workflows/k8s-regression-tester.yml b/.github/workflows/agglayer-regression-tests.yml
similarity index 61%
rename from .github/workflows/k8s-regression-tester.yml
rename to .github/workflows/agglayer-regression-tests.yml
index 236277fcab..645a144f5f 100644
--- a/.github/workflows/k8s-regression-tester.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -1,14 +1,13 @@
-name: REGRESSION DETECTOR - DEVNET
+name: REGRESSION TESTER - AGGLAYER SPECIFIC
 on:
-  push:
-  # workflow_dispatch:
-  #   inputs:
-  #     commit_id:
-  #       description: 'agglayer version (commit id)'
-  #       required: true
-  #     bake_time:
-  #       description: 'bake time (minutes)'
-  #       required: true
+  workflow_dispatch:
+    inputs:
+      commit_id:
+        description: 'agglayer version (commit id)'
+        required: true
+      bake_time:
+        description: 'bake time (minutes)'
+        required: true
 
 jobs:
   deploy_devnet:
@@ -51,8 +50,7 @@ jobs:
       run: |
         git clone https://github.com/0xPolygon/agglayer.git
         cd agglayer
-        # git checkout "${{ github.event.inputs.commit_id }}"
-        git checkout c0a88f1
+        git checkout "${{ github.event.inputs.commit_id }}"
         
     - name: Build JIT agglayer image to be used in ephemeral devnets
       run: |
@@ -75,8 +73,28 @@ jobs:
         cd kurtosis-cdk
         kurtosis engine restart
         kurtosis run --enclave cdk-v1 --args-file params.yml .
-        sleep 300
-    
+
+    - name: Auto-apply load and report any regressions here
+      run: |
+        MONITOR_ID="140974014"  # K8S DEVNET: COMPOSITE MONITOR ID
+        DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
+        DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
+
+        bake_time="${{ github.event.inputs.bake_time }}"
+        end_minute=$(( $(date +'%M') + bake_time))
+
+        while [ $(date +'%M') -lt $end_minute ]; do
+          STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
+                    -H "Content-Type: application/json" \
+                    -H "DD-API-KEY: ${DATADOG_API_KEY}" \
+                    -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
+                    -s \
+                    | jq -r '.overall_state')
+          echo "K8S DEVNET HEALTH STATUS: $STATUS"
+          echo "Review your devnet health dashboard here: https://app.datadoghq.com/dashboard/fqu-nh2-bzd?fromUser=false&refresh_mode=sliding&view=spans&from_ts=1709886652761&to_ts=1709890252761&live=true"
+          sleep 60
+        done
+
     - name: Finally, remove all devnet resources locally
       run: |
         cd kurtosis-cdk
diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
new file mode 100644
index 0000000000..5b843a41a3
--- /dev/null
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -0,0 +1,168 @@
+name: REGRESSION TESTER - SUPERUSERS
+on:
+  workflow_dispatch:
+    inputs:
+      zkevm_agglayer_commit_id:
+        description: 'zkevm agglayer repo (commit id)'
+        required: true
+      zkevm_bridge_service_commit_id:
+        description: 'zkevm bridge service repo (commit id)'
+        required: true
+      zkevm_bridge_ui_commit_id:
+        description: 'zkevm bridge ui repo (commit id)'
+        required: true
+      zkevm_contracts_commit_id:
+        description: 'zkevm contracts repo (commit id)'
+        required: true
+      zkevm_dac_commit_id:
+        description: 'zkevm dac repo (commit id)'
+        required: true
+      zkevm_node_commit_id:
+        description: 'zkevm node repo (commit id)'
+        required: true
+      zkevm_prover_commit_id:
+        description: 'zkevm prover repo (commit id)'
+        required: true
+      bake_time:
+        description: 'bake time (minutes)'
+        required: true
+
+jobs:
+  deploy_devnet:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.before }}
+
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v1
+
+    - name: Setup Google Cloud SDK
+      uses: 'google-github-actions/setup-gcloud@v2'
+      with:
+        version: '>= 363.0.0'
+
+    - uses: 'google-github-actions/auth@v2'
+      with:
+        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+        project_id: 'prj-polygonlabs-devtools-dev'
+
+    - name: Install kubectl and gcloud kubectl auth plugin
+      run: |
+        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
+        && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && gcloud components install gke-gcloud-auth-plugin
+
+    - name: Install tomlq for config file preparation
+      run: |
+        pip3 install tomlq
+
+    - name: Clone internal kurtosis-cdk repo
+      run: |
+        git clone https://github.com/0xPolygon/kurtosis-cdk.git
+        cd kurtosis-cdk
+        git checkout dan/jit_containers_superusers
+        
+    - name: Clone internal agglayer repo
+      run: |
+        git clone https://github.com/0xPolygon/agglayer.git
+        cd agglayer
+        git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
+    - name: Build JIT agglayer image for ephemeral devnet
+      run: |
+        docker compose -f agglayer/docker/docker-compose.yaml build --no-cache agglayer
+        sleep 10
+        
+    - name: Clone internal bridge service repo
+      run: |
+        git clone https://github.com/0xPolygonHermez/zkevm-bridge-service.git
+        cd zkevm-bridge-service
+        git checkout "${{ github.event.inputs.zkevm_bridge_service_commit_id }}"
+    - name: Build JIT zkevm-bridge-service image for ephemeral devnet
+      run: |
+        docker build -t zkevm-bridge-service:local -f ./Dockerfile .
+        sleep 10
+        
+    - name: Clone internal bridge ui repo
+      run: |
+        git clone https://github.com/0xPolygonHermez/zkevm-bridge-ui.git
+        cd zkevm-bridge-ui
+        git checkout "${{ github.event.inputs.zkevm_bridge_ui_commit_id }}"
+    - name: Build JIT zkevm-bridge-ui image for ephemeral devnet
+      run: |
+        docker build -t zkevm-bridge-ui:local -f ./Dockerfile .
+        sleep 10
+        
+    - name: Clone internal dac repo
+      run: |
+        git clone https://github.com/0xPolygon/cdk-data-availability.git
+        cd cdk-data-availability
+        git checkout "${{ github.event.inputs.zkevm_dac_commit_id }}"
+    - name: Build JIT cdk-data-availability image for ephemeral devnet
+      run: |
+        docker build -t cdk-data-availability:local -f ./Dockerfile .
+        sleep 10      
+        
+    - name: Clone internal validium node repo
+      run: |
+        git clone https://github.com/0xPolygon/cdk-validium-node.git
+        cd cdk-validium-node
+        git checkout "${{ github.event.inputs.zkevm_node_commit_id }}"
+    - name: Build JIT cdk-validium-node image for ephemeral devnet
+      run: |
+        docker build -t cdk-validium-node:local -f ./Dockerfile .
+        sleep 10       
+
+    - name: Clone internal zk prover repo
+      run: |
+        git clone https://github.com/0xPolygonHermez/zkevm-prover.git
+        cd zkevm-prover
+        git checkout "${{ github.event.inputs.zkevm_prover_commit_id }}"
+    - name: Build JIT zkevm-prover image for ephemeral devnet
+      run: |
+        docker build -t zkevm-prover:local -f ./Dockerfile-GHA .
+        sleep 10
+        
+    - name: Install kurtosis
+      run: |
+        echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+        sudo apt update
+        sudo apt install kurtosis-cli
+
+    - name: Run kurtosis agent in background
+      run: |  
+        kurtosis gateway &  # Run cmd in background
+        sleep 10
+
+    - name: Deploy CDK devnet on local github runner
+      run: |
+        cd kurtosis-cdk
+        kurtosis engine restart
+        kurtosis run --enclave cdk-v1 --args-file params.yml .
+
+    - name: Auto-apply load and report any regressions here
+      run: |
+        MONITOR_ID="140974014"  # K8S DEVNET: COMPOSITE MONITOR ID
+        DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
+        DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
+
+        bake_time="${{ github.event.inputs.bake_time }}"
+        end_minute=$(( $(date +'%M') + bake_time))
+
+        while [ $(date +'%M') -lt $end_minute ]; do
+          STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
+                    -H "Content-Type: application/json" \
+                    -H "DD-API-KEY: ${DATADOG_API_KEY}" \
+                    -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
+                    -s \
+                    | jq -r '.overall_state')
+          echo "K8S DEVNET HEALTH STATUS: $STATUS"
+          echo "Review your devnet health dashboard here: https://app.datadoghq.com/dashboard/fqu-nh2-bzd?fromUser=false&refresh_mode=sliding&view=spans&from_ts=1709886652761&to_ts=1709890252761&live=true"
+          sleep 60
+        done
+    
+    - name: Finally, remove all devnet resources locally
+      run: |
+        cd kurtosis-cdk
+        kurtosis clean -a

From 8ba439a53e53b01e2265620f16c41c6aac989e1d Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 17:18:04 -0500
Subject: [PATCH 116/144] cleanup unused code

---
 .../workflows/agglayer-regression-tests.yml   | 19 ----------------
 .../cdk-superuser-regression-tests.yml        | 22 -------------------
 2 files changed, 41 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index 645a144f5f..9e29c1b571 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -21,25 +21,6 @@ jobs:
     - name: Set up Docker
       uses: docker/setup-buildx-action@v1
 
-    - name: Setup Google Cloud SDK
-      uses: 'google-github-actions/setup-gcloud@v2'
-      with:
-        version: '>= 363.0.0'
-
-    - uses: 'google-github-actions/auth@v2'
-      with:
-        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-        project_id: 'prj-polygonlabs-devtools-dev'
-
-    - name: Install kubectl and gcloud kubectl auth plugin
-      run: |
-        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
-        && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && gcloud components install gke-gcloud-auth-plugin
-
-    - name: Install tomlq for config file preparation
-      run: |
-        pip3 install tomlq
-
     - name: Clone internal kurtosis-cdk repo
       run: |
         git clone https://github.com/0xPolygon/kurtosis-cdk.git
diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 5b843a41a3..714fa0fa84 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -11,9 +11,6 @@ on:
       zkevm_bridge_ui_commit_id:
         description: 'zkevm bridge ui repo (commit id)'
         required: true
-      zkevm_contracts_commit_id:
-        description: 'zkevm contracts repo (commit id)'
-        required: true
       zkevm_dac_commit_id:
         description: 'zkevm dac repo (commit id)'
         required: true
@@ -39,25 +36,6 @@ jobs:
     - name: Set up Docker
       uses: docker/setup-buildx-action@v1
 
-    - name: Setup Google Cloud SDK
-      uses: 'google-github-actions/setup-gcloud@v2'
-      with:
-        version: '>= 363.0.0'
-
-    - uses: 'google-github-actions/auth@v2'
-      with:
-        credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-        project_id: 'prj-polygonlabs-devtools-dev'
-
-    - name: Install kubectl and gcloud kubectl auth plugin
-      run: |
-        curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
-        && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && gcloud components install gke-gcloud-auth-plugin
-
-    - name: Install tomlq for config file preparation
-      run: |
-        pip3 install tomlq
-
     - name: Clone internal kurtosis-cdk repo
       run: |
         git clone https://github.com/0xPolygon/kurtosis-cdk.git

From 717641f60bca787ba9abf2f71eedde0e59109b50 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 17:35:28 -0500
Subject: [PATCH 117/144] consolidate workflow logic

---
 .../workflows/agglayer-regression-tests.yml   | 11 ++++-----
 .../cdk-superuser-regression-tests.yml        | 24 +++++--------------
 2 files changed, 10 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index 9e29c1b571..da25e0d0a9 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -2,8 +2,8 @@ name: REGRESSION TESTER - AGGLAYER SPECIFIC
 on:
   workflow_dispatch:
     inputs:
-      commit_id:
-        description: 'agglayer version (commit id)'
+      zkevm_agglayer_commit_id:
+        description: 'zkevm agglayer repo (commit id)'
         required: true
       bake_time:
         description: 'bake time (minutes)'
@@ -27,14 +27,11 @@ jobs:
         cd kurtosis-cdk
         git checkout dan/jit_containers
         
-    - name: Clone internal agglayer repo
+    - name: Clone and build agglayer
       run: |
         git clone https://github.com/0xPolygon/agglayer.git
         cd agglayer
-        git checkout "${{ github.event.inputs.commit_id }}"
-        
-    - name: Build JIT agglayer image to be used in ephemeral devnets
-      run: |
+        git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
         docker compose -f agglayer/docker/docker-compose.yaml build --no-cache agglayer
         sleep 10
         
diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 714fa0fa84..104d479086 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -42,63 +42,51 @@ jobs:
         cd kurtosis-cdk
         git checkout dan/jit_containers_superusers
         
-    - name: Clone internal agglayer repo
+    - name: Clone and build agglayer
       run: |
         git clone https://github.com/0xPolygon/agglayer.git
         cd agglayer
         git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
-    - name: Build JIT agglayer image for ephemeral devnet
-      run: |
         docker compose -f agglayer/docker/docker-compose.yaml build --no-cache agglayer
         sleep 10
         
-    - name: Clone internal bridge service repo
+    - name: Clone and build zkevm-bridge-service
       run: |
         git clone https://github.com/0xPolygonHermez/zkevm-bridge-service.git
         cd zkevm-bridge-service
         git checkout "${{ github.event.inputs.zkevm_bridge_service_commit_id }}"
-    - name: Build JIT zkevm-bridge-service image for ephemeral devnet
-      run: |
         docker build -t zkevm-bridge-service:local -f ./Dockerfile .
         sleep 10
         
-    - name: Clone internal bridge ui repo
+    - name: Clone and build zkevm-bridge-ui
       run: |
         git clone https://github.com/0xPolygonHermez/zkevm-bridge-ui.git
         cd zkevm-bridge-ui
         git checkout "${{ github.event.inputs.zkevm_bridge_ui_commit_id }}"
-    - name: Build JIT zkevm-bridge-ui image for ephemeral devnet
-      run: |
         docker build -t zkevm-bridge-ui:local -f ./Dockerfile .
         sleep 10
         
-    - name: Clone internal dac repo
+    - name: Clone and build cdk-data-availability
       run: |
         git clone https://github.com/0xPolygon/cdk-data-availability.git
         cd cdk-data-availability
         git checkout "${{ github.event.inputs.zkevm_dac_commit_id }}"
-    - name: Build JIT cdk-data-availability image for ephemeral devnet
-      run: |
         docker build -t cdk-data-availability:local -f ./Dockerfile .
         sleep 10      
         
-    - name: Clone internal validium node repo
+    - name: Clone and build cdk-validium-node
       run: |
         git clone https://github.com/0xPolygon/cdk-validium-node.git
         cd cdk-validium-node
         git checkout "${{ github.event.inputs.zkevm_node_commit_id }}"
-    - name: Build JIT cdk-validium-node image for ephemeral devnet
-      run: |
         docker build -t cdk-validium-node:local -f ./Dockerfile .
         sleep 10       
 
-    - name: Clone internal zk prover repo
+    - name: Clone and build zkevm-prover
       run: |
         git clone https://github.com/0xPolygonHermez/zkevm-prover.git
         cd zkevm-prover
         git checkout "${{ github.event.inputs.zkevm_prover_commit_id }}"
-    - name: Build JIT zkevm-prover image for ephemeral devnet
-      run: |
         docker build -t zkevm-prover:local -f ./Dockerfile-GHA .
         sleep 10
         

From 928cc31bf2d78fe13596cae82fc09434a552fa9b Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 17:43:08 -0500
Subject: [PATCH 118/144] more clarity in ci input descriptions

---
 .github/workflows/agglayer-regression-tests.yml      |  2 +-
 .github/workflows/cdk-superuser-regression-tests.yml | 12 ++++++------
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index da25e0d0a9..c281a69f05 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -3,7 +3,7 @@ on:
   workflow_dispatch:
     inputs:
       zkevm_agglayer_commit_id:
-        description: 'zkevm agglayer repo (commit id)'
+        description: '0xPolygon/agglayer (commit id)'
         required: true
       bake_time:
         description: 'bake time (minutes)'
diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 104d479086..1449907a6d 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -3,22 +3,22 @@ on:
   workflow_dispatch:
     inputs:
       zkevm_agglayer_commit_id:
-        description: 'zkevm agglayer repo (commit id)'
+        description: '0xPolygon/agglayer (commit id)'
         required: true
       zkevm_bridge_service_commit_id:
-        description: 'zkevm bridge service repo (commit id)'
+        description: '0xPolygonHermez/zkevm-bridge-service (commit id)'
         required: true
       zkevm_bridge_ui_commit_id:
-        description: 'zkevm bridge ui repo (commit id)'
+        description: '0xPolygonHermez/zkevm-bridge-ui (commit id)'
         required: true
       zkevm_dac_commit_id:
-        description: 'zkevm dac repo (commit id)'
+        description: '0xPolygon/cdk-data-availability (commit id)'
         required: true
       zkevm_node_commit_id:
-        description: 'zkevm node repo (commit id)'
+        description: '0xPolygon/cdk-validium-node (commit id)'
         required: true
       zkevm_prover_commit_id:
-        description: 'zkevm prover repo (commit id)'
+        description: '0xPolygonHermez/zkevm-prover (commit id)'
         required: true
       bake_time:
         description: 'bake time (minutes)'

From 610f0e3b9d5debece7932deb1839b8cdc698293b Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 17:56:03 -0500
Subject: [PATCH 119/144] cleanup wokflow

---
 .github/workflows/agglayer-regression-tests.yml      | 2 +-
 .github/workflows/cdk-superuser-regression-tests.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index c281a69f05..3ab4d20f01 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -32,7 +32,7 @@ jobs:
         git clone https://github.com/0xPolygon/agglayer.git
         cd agglayer
         git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
-        docker compose -f agglayer/docker/docker-compose.yaml build --no-cache agglayer
+        docker compose -f docker/docker-compose.yaml build --no-cache agglayer
         sleep 10
         
     - name: Install kurtosis
diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 1449907a6d..6a8c74e4b7 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -47,7 +47,7 @@ jobs:
         git clone https://github.com/0xPolygon/agglayer.git
         cd agglayer
         git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
-        docker compose -f agglayer/docker/docker-compose.yaml build --no-cache agglayer
+        docker compose -f docker/docker-compose.yaml build --no-cache agglayer
         sleep 10
         
     - name: Clone and build zkevm-bridge-service

From 43e56b19547f659cfb108899a113ad7908719988 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 18:12:39 -0500
Subject: [PATCH 120/144] debug

---
 .../cdk-superuser-regression-tests.yml        | 133 +++++++++---------
 1 file changed, 69 insertions(+), 64 deletions(-)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 6a8c74e4b7..25bbff0af7 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -1,28 +1,29 @@
 name: REGRESSION TESTER - SUPERUSERS
 on:
-  workflow_dispatch:
-    inputs:
-      zkevm_agglayer_commit_id:
-        description: '0xPolygon/agglayer (commit id)'
-        required: true
-      zkevm_bridge_service_commit_id:
-        description: '0xPolygonHermez/zkevm-bridge-service (commit id)'
-        required: true
-      zkevm_bridge_ui_commit_id:
-        description: '0xPolygonHermez/zkevm-bridge-ui (commit id)'
-        required: true
-      zkevm_dac_commit_id:
-        description: '0xPolygon/cdk-data-availability (commit id)'
-        required: true
-      zkevm_node_commit_id:
-        description: '0xPolygon/cdk-validium-node (commit id)'
-        required: true
-      zkevm_prover_commit_id:
-        description: '0xPolygonHermez/zkevm-prover (commit id)'
-        required: true
-      bake_time:
-        description: 'bake time (minutes)'
-        required: true
+  push:
+  # workflow_dispatch:
+  #   inputs:
+  #     zkevm_agglayer_commit_id:
+  #       description: '0xPolygon/agglayer (commit id)'
+  #       required: true
+  #     zkevm_bridge_service_commit_id:
+  #       description: '0xPolygonHermez/zkevm-bridge-service (commit id)'
+  #       required: true
+  #     zkevm_bridge_ui_commit_id:
+  #       description: '0xPolygonHermez/zkevm-bridge-ui (commit id)'
+  #       required: true
+  #     zkevm_dac_commit_id:
+  #       description: '0xPolygon/cdk-data-availability (commit id)'
+  #       required: true
+  #     zkevm_node_commit_id:
+  #       description: '0xPolygon/cdk-validium-node (commit id)'
+  #       required: true
+  #     zkevm_prover_commit_id:
+  #       description: '0xPolygonHermez/zkevm-prover (commit id)'
+  #       required: true
+  #     bake_time:
+  #       description: 'bake time (minutes)'
+  #       required: true
 
 jobs:
   deploy_devnet:
@@ -41,52 +42,56 @@ jobs:
         git clone https://github.com/0xPolygon/kurtosis-cdk.git
         cd kurtosis-cdk
         git checkout dan/jit_containers_superusers
-        
-    - name: Clone and build agglayer
-      run: |
-        git clone https://github.com/0xPolygon/agglayer.git
-        cd agglayer
-        git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
-        docker compose -f docker/docker-compose.yaml build --no-cache agglayer
-        sleep 10
-        
-    - name: Clone and build zkevm-bridge-service
-      run: |
-        git clone https://github.com/0xPolygonHermez/zkevm-bridge-service.git
-        cd zkevm-bridge-service
-        git checkout "${{ github.event.inputs.zkevm_bridge_service_commit_id }}"
-        docker build -t zkevm-bridge-service:local -f ./Dockerfile .
-        sleep 10
-        
-    - name: Clone and build zkevm-bridge-ui
-      run: |
-        git clone https://github.com/0xPolygonHermez/zkevm-bridge-ui.git
-        cd zkevm-bridge-ui
-        git checkout "${{ github.event.inputs.zkevm_bridge_ui_commit_id }}"
-        docker build -t zkevm-bridge-ui:local -f ./Dockerfile .
-        sleep 10
-        
-    - name: Clone and build cdk-data-availability
-      run: |
-        git clone https://github.com/0xPolygon/cdk-data-availability.git
-        cd cdk-data-availability
-        git checkout "${{ github.event.inputs.zkevm_dac_commit_id }}"
-        docker build -t cdk-data-availability:local -f ./Dockerfile .
-        sleep 10      
-        
-    - name: Clone and build cdk-validium-node
-      run: |
-        git clone https://github.com/0xPolygon/cdk-validium-node.git
-        cd cdk-validium-node
-        git checkout "${{ github.event.inputs.zkevm_node_commit_id }}"
-        docker build -t cdk-validium-node:local -f ./Dockerfile .
-        sleep 10       
+
+    # - name: Clone and build agglayer
+    #   run: |
+    #     git clone https://github.com/0xPolygon/agglayer.git
+    #     cd agglayer
+    #     git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
+    #     docker compose -f docker/docker-compose.yaml build --no-cache agglayer
+    #     sleep 10
+    # 
+    # - name: Clone and build zkevm-bridge-service
+    #   run: |
+    #     git clone https://github.com/0xPolygonHermez/zkevm-bridge-service.git
+    #     cd zkevm-bridge-service
+    #     git checkout "${{ github.event.inputs.zkevm_bridge_service_commit_id }}"
+    #     docker build -t zkevm-bridge-service:local -f ./Dockerfile .
+    #     sleep 10
+    # 
+    # - name: Clone and build zkevm-bridge-ui
+    #   run: |
+    #     git clone https://github.com/0xPolygonHermez/zkevm-bridge-ui.git
+    #     cd zkevm-bridge-ui
+    #     git checkout "${{ github.event.inputs.zkevm_bridge_ui_commit_id }}"
+    #     docker build -t zkevm-bridge-ui:local -f ./Dockerfile .
+    #     sleep 10
+    # 
+    # - name: Clone and build cdk-data-availability
+    #   run: |
+    #     git clone https://github.com/0xPolygon/cdk-data-availability.git
+    #     cd cdk-data-availability
+    #     git checkout "${{ github.event.inputs.zkevm_dac_commit_id }}"
+    #     docker build -t cdk-data-availability:local -f ./Dockerfile .
+    #     sleep 10      
+    # 
+    # - name: Clone and build cdk-validium-node
+    #   run: |
+    #     git clone https://github.com/0xPolygon/cdk-validium-node.git
+    #     cd cdk-validium-node
+    #     git checkout "${{ github.event.inputs.zkevm_node_commit_id }}"
+    #     docker build -t cdk-validium-node:local -f ./Dockerfile .
+    #     sleep 10       
 
     - name: Clone and build zkevm-prover
       run: |
         git clone https://github.com/0xPolygonHermez/zkevm-prover.git
         cd zkevm-prover
-        git checkout "${{ github.event.inputs.zkevm_prover_commit_id }}"
+        # git checkout "${{ github.event.inputs.zkevm_prover_commit_id }}"
+        git checkout d37e826
+        sudo apt-get install libgtest-dev
+        g++ tests/tests.cpp src/* -lgtest -lgmp -lomp -o test -g  -Wall -pthread -fopenmp -mavx2 -L$(find /usr/lib/llvm-* -name "libomp.so" | sed 's/libomp.so//')
+        g++ benchs/bench.cpp src/* -lbenchmark -lomp -lpthread -lgmp  -std=c++17 -Wall -pthread -fopenmp -mavx2 -L$(find /usr/lib/llvm-* -name "libomp.so" | sed 's/libomp.so//') -O3 -o bench
         docker build -t zkevm-prover:local -f ./Dockerfile-GHA .
         sleep 10
         

From 01e7944976c6efa2e94173fa4a8441a5b7cf8f87 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 18:20:17 -0500
Subject: [PATCH 121/144] prover build debug

---
 .../workflows/cdk-superuser-regression-tests.yml  | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 25bbff0af7..f6bd06602f 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -85,14 +85,19 @@ jobs:
 
     - name: Clone and build zkevm-prover
       run: |
-        git clone https://github.com/0xPolygonHermez/zkevm-prover.git
+        git clone --recursive https://github.com/0xPolygonHermez/zkevm-prover.git
         cd zkevm-prover
         # git checkout "${{ github.event.inputs.zkevm_prover_commit_id }}"
         git checkout d37e826
-        sudo apt-get install libgtest-dev
-        g++ tests/tests.cpp src/* -lgtest -lgmp -lomp -o test -g  -Wall -pthread -fopenmp -mavx2 -L$(find /usr/lib/llvm-* -name "libomp.so" | sed 's/libomp.so//')
-        g++ benchs/bench.cpp src/* -lbenchmark -lomp -lpthread -lgmp  -std=c++17 -Wall -pthread -fopenmp -mavx2 -L$(find /usr/lib/llvm-* -name "libomp.so" | sed 's/libomp.so//') -O3 -o bench
-        docker build -t zkevm-prover:local -f ./Dockerfile-GHA .
+        # install archive deps
+        ./tools/download_archive.sh
+        # install pkg deps 
+        apt update
+        apt install build-essential libbenchmark-dev libomp-dev libgmp-dev nlohmann-json3-dev postgresql libpqxx-dev libpqxx-doc nasm libsecp256k1-dev grpc-proto libsodium-dev libprotobuf-dev libssl-dev cmake libgrpc++-dev protobuf-compiler protobuf-compiler-grpc uuid-dev
+        cd src/grpc
+        make
+        cd ../..
+        docker build -t zkevm-prover:local -f ./Dockerfile .
         sleep 10
         
     - name: Install kurtosis

From 54d1f1f9576139c34e99a3085a8c026c5fe35d14 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 18:25:07 -0500
Subject: [PATCH 122/144] debug

---
 .github/workflows/cdk-superuser-regression-tests.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index f6bd06602f..44a3648024 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -90,13 +90,17 @@ jobs:
         # git checkout "${{ github.event.inputs.zkevm_prover_commit_id }}"
         git checkout d37e826
         # install archive deps
+        pwd
         ./tools/download_archive.sh
+        pwd
         # install pkg deps 
         apt update
-        apt install build-essential libbenchmark-dev libomp-dev libgmp-dev nlohmann-json3-dev postgresql libpqxx-dev libpqxx-doc nasm libsecp256k1-dev grpc-proto libsodium-dev libprotobuf-dev libssl-dev cmake libgrpc++-dev protobuf-compiler protobuf-compiler-grpc uuid-dev
+        apt install -y build-essential libbenchmark-dev libomp-dev libgmp-dev nlohmann-json3-dev postgresql libpqxx-dev libpqxx-doc nasm libsecp256k1-dev grpc-proto libsodium-dev libprotobuf-dev libssl-dev cmake libgrpc++-dev protobuf-compiler protobuf-compiler-grpc uuid-dev
         cd src/grpc
+        pwd
         make
         cd ../..
+        pwd
         docker build -t zkevm-prover:local -f ./Dockerfile .
         sleep 10
         

From 35ee74f4a9e3e00db42006c7f0f8d4f9b19adc63 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Mon, 8 Apr 2024 18:29:49 -0500
Subject: [PATCH 123/144] debuggin

---
 .github/workflows/cdk-superuser-regression-tests.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 44a3648024..6b67686701 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -91,9 +91,16 @@ jobs:
         git checkout d37e826
         # install archive deps
         pwd
-        ./tools/download_archive.sh
+        ARCHIVE_NAME="v3.0.0-RC3-fork.6"
+        ARCHIVE_EXTENSION=".tgz"
+        ARCHIVE_URL="https://storage.googleapis.com/zkevm/zkproverc/${ARCHIVE_NAME}${ARCHIVE_EXTENSION}"
+        wget -c ${ARCHIVE_URL}
+        tar -xzvf ${ARCHIVE_NAME}${ARCHIVE_EXTENSION}
+        rm -rf config
+        cp -R ${ARCHIVE_NAME}/config .
+        rm ${ARCHIVE_NAME}${ARCHIVE_EXTENSION}
         pwd
-        # install pkg deps 
+        # install pkg deps
         apt update
         apt install -y build-essential libbenchmark-dev libomp-dev libgmp-dev nlohmann-json3-dev postgresql libpqxx-dev libpqxx-doc nasm libsecp256k1-dev grpc-proto libsodium-dev libprotobuf-dev libssl-dev cmake libgrpc++-dev protobuf-compiler protobuf-compiler-grpc uuid-dev
         cd src/grpc

From 8f213f55371f8d924203326c777da4834162db8a Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 9 Apr 2024 09:33:39 -0500
Subject: [PATCH 124/144] debug

---
 .../cdk-superuser-regression-tests.yml        | 199 ++++++++----------
 1 file changed, 85 insertions(+), 114 deletions(-)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 6b67686701..b7d25ad750 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -1,115 +1,91 @@
 name: REGRESSION TESTER - SUPERUSERS
 on:
   push:
-  # workflow_dispatch:
-  #   inputs:
-  #     zkevm_agglayer_commit_id:
-  #       description: '0xPolygon/agglayer (commit id)'
-  #       required: true
-  #     zkevm_bridge_service_commit_id:
-  #       description: '0xPolygonHermez/zkevm-bridge-service (commit id)'
-  #       required: true
-  #     zkevm_bridge_ui_commit_id:
-  #       description: '0xPolygonHermez/zkevm-bridge-ui (commit id)'
-  #       required: true
-  #     zkevm_dac_commit_id:
-  #       description: '0xPolygon/cdk-data-availability (commit id)'
-  #       required: true
-  #     zkevm_node_commit_id:
-  #       description: '0xPolygon/cdk-validium-node (commit id)'
-  #       required: true
-  #     zkevm_prover_commit_id:
-  #       description: '0xPolygonHermez/zkevm-prover (commit id)'
-  #       required: true
-  #     bake_time:
-  #       description: 'bake time (minutes)'
-  #       required: true
+#   workflow_dispatch:
+#     inputs:
+#       zkevm_agglayer_commit_id:
+#         description: '0xPolygon/agglayer (commit id)'
+#         required: true
+#       zkevm_bridge_service_commit_id:
+#         description: '0xPolygonHermez/zkevm-bridge-service (commit id)'
+#         required: true
+#       zkevm_bridge_ui_commit_id:
+#         description: '0xPolygonHermez/zkevm-bridge-ui (commit id)'
+#         required: true
+#       zkevm_dac_commit_id:
+#         description: '0xPolygon/cdk-data-availability (commit id)'
+#         required: true
+#       zkevm_node_commit_id:
+#         description: '0xPolygon/cdk-validium-node (commit id)'
+#         required: true
+#       bake_time:
+#         description: 'bake time (minutes)'
+#         required: true
+# 
+# jobs:
+#   deploy_devnet:
+#     runs-on: ubuntu-latest
+#     steps:
+#     - name: Checkout repository
+#       uses: actions/checkout@v2
+#       with:
+#         ref: ${{ github.event.before }}
+# 
+#     - name: Set up Docker
+#       uses: docker/setup-buildx-action@v1
+# 
+#     - name: Clone internal kurtosis-cdk repo
+#       run: |
+#         git clone https://github.com/0xPolygon/kurtosis-cdk.git
+#         cd kurtosis-cdk
+#         git checkout dan/jit_containers_superusers
+# 
+#     - name: Clone and build agglayer
+#       run: |
+#         git clone https://github.com/0xPolygon/agglayer.git
+#         cd agglayer
+#         git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
+#         docker compose -f docker/docker-compose.yaml build --no-cache agglayer
+#         sleep 10
+# 
+#     - name: Clone and build zkevm-bridge-service
+#       run: |
+#         git clone https://github.com/0xPolygonHermez/zkevm-bridge-service.git
+#         cd zkevm-bridge-service
+#         git checkout "${{ github.event.inputs.zkevm_bridge_service_commit_id }}"
+#         docker build -t zkevm-bridge-service:local -f ./Dockerfile .
+#         sleep 10
+# 
+#     - name: Clone and build zkevm-bridge-ui
+#       run: |
+#         git clone https://github.com/0xPolygonHermez/zkevm-bridge-ui.git
+#         cd zkevm-bridge-ui
+#         git checkout "${{ github.event.inputs.zkevm_bridge_ui_commit_id }}"
+#         docker build -t zkevm-bridge-ui:local -f ./Dockerfile .
+#         sleep 10
+# 
+#     - name: Clone and build cdk-data-availability
+#       run: |
+#         git clone https://github.com/0xPolygon/cdk-data-availability.git
+#         cd cdk-data-availability
+#         git checkout "${{ github.event.inputs.zkevm_dac_commit_id }}"
+#         docker build -t cdk-data-availability:local -f ./Dockerfile .
+#         sleep 10      
+# 
+#     - name: Clone and build cdk-validium-node
+#       run: |
+#         git clone https://github.com/0xPolygon/cdk-validium-node.git
+#         cd cdk-validium-node
+#         git checkout "${{ github.event.inputs.zkevm_node_commit_id }}"
+#         docker build -t cdk-validium-node:local -f ./Dockerfile .
+#         sleep 10
 
-jobs:
-  deploy_devnet:
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ github.event.before }}
-
-    - name: Set up Docker
-      uses: docker/setup-buildx-action@v1
-
-    - name: Clone internal kurtosis-cdk repo
-      run: |
-        git clone https://github.com/0xPolygon/kurtosis-cdk.git
-        cd kurtosis-cdk
-        git checkout dan/jit_containers_superusers
-
-    # - name: Clone and build agglayer
-    #   run: |
-    #     git clone https://github.com/0xPolygon/agglayer.git
-    #     cd agglayer
-    #     git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
-    #     docker compose -f docker/docker-compose.yaml build --no-cache agglayer
-    #     sleep 10
-    # 
-    # - name: Clone and build zkevm-bridge-service
-    #   run: |
-    #     git clone https://github.com/0xPolygonHermez/zkevm-bridge-service.git
-    #     cd zkevm-bridge-service
-    #     git checkout "${{ github.event.inputs.zkevm_bridge_service_commit_id }}"
-    #     docker build -t zkevm-bridge-service:local -f ./Dockerfile .
-    #     sleep 10
-    # 
-    # - name: Clone and build zkevm-bridge-ui
-    #   run: |
-    #     git clone https://github.com/0xPolygonHermez/zkevm-bridge-ui.git
-    #     cd zkevm-bridge-ui
-    #     git checkout "${{ github.event.inputs.zkevm_bridge_ui_commit_id }}"
-    #     docker build -t zkevm-bridge-ui:local -f ./Dockerfile .
-    #     sleep 10
-    # 
-    # - name: Clone and build cdk-data-availability
-    #   run: |
-    #     git clone https://github.com/0xPolygon/cdk-data-availability.git
-    #     cd cdk-data-availability
-    #     git checkout "${{ github.event.inputs.zkevm_dac_commit_id }}"
-    #     docker build -t cdk-data-availability:local -f ./Dockerfile .
-    #     sleep 10      
-    # 
-    # - name: Clone and build cdk-validium-node
-    #   run: |
-    #     git clone https://github.com/0xPolygon/cdk-validium-node.git
-    #     cd cdk-validium-node
-    #     git checkout "${{ github.event.inputs.zkevm_node_commit_id }}"
-    #     docker build -t cdk-validium-node:local -f ./Dockerfile .
-    #     sleep 10       
-
-    - name: Clone and build zkevm-prover
-      run: |
-        git clone --recursive https://github.com/0xPolygonHermez/zkevm-prover.git
-        cd zkevm-prover
-        # git checkout "${{ github.event.inputs.zkevm_prover_commit_id }}"
-        git checkout d37e826
-        # install archive deps
-        pwd
-        ARCHIVE_NAME="v3.0.0-RC3-fork.6"
-        ARCHIVE_EXTENSION=".tgz"
-        ARCHIVE_URL="https://storage.googleapis.com/zkevm/zkproverc/${ARCHIVE_NAME}${ARCHIVE_EXTENSION}"
-        wget -c ${ARCHIVE_URL}
-        tar -xzvf ${ARCHIVE_NAME}${ARCHIVE_EXTENSION}
-        rm -rf config
-        cp -R ${ARCHIVE_NAME}/config .
-        rm ${ARCHIVE_NAME}${ARCHIVE_EXTENSION}
-        pwd
-        # install pkg deps
-        apt update
-        apt install -y build-essential libbenchmark-dev libomp-dev libgmp-dev nlohmann-json3-dev postgresql libpqxx-dev libpqxx-doc nasm libsecp256k1-dev grpc-proto libsodium-dev libprotobuf-dev libssl-dev cmake libgrpc++-dev protobuf-compiler protobuf-compiler-grpc uuid-dev
-        cd src/grpc
-        pwd
-        make
-        cd ../..
-        pwd
-        docker build -t zkevm-prover:local -f ./Dockerfile .
-        sleep 10
+    - name: Install foundry (i.e. cast)
+      run: |    
+        curl -L https://foundry.paradigm.xyz | bash 
+        . .bashrc
+        foundryup
+        cast
         
     - name: Install kurtosis
       run: |
@@ -130,10 +106,6 @@ jobs:
 
     - name: Auto-apply load and report any regressions here
       run: |
-        MONITOR_ID="140974014"  # K8S DEVNET: COMPOSITE MONITOR ID
-        DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
-        DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
-
         bake_time="${{ github.event.inputs.bake_time }}"
         end_minute=$(( $(date +'%M') + bake_time))
 
@@ -144,8 +116,7 @@ jobs:
                     -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
                     -s \
                     | jq -r '.overall_state')
-          echo "K8S DEVNET HEALTH STATUS: $STATUS"
-          echo "Review your devnet health dashboard here: https://app.datadoghq.com/dashboard/fqu-nh2-bzd?fromUser=false&refresh_mode=sliding&view=spans&from_ts=1709886652761&to_ts=1709890252761&live=true"
+          echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
           sleep 60
         done
     

From cf89deaee4f723b93044dc3ad645311cbc7db565 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 9 Apr 2024 09:34:59 -0500
Subject: [PATCH 125/144] test

---
 .../cdk-superuser-regression-tests.yml        | 110 +++++++++---------
 1 file changed, 55 insertions(+), 55 deletions(-)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index b7d25ad750..9de910e82e 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -22,23 +22,23 @@ on:
 #         description: 'bake time (minutes)'
 #         required: true
 # 
-# jobs:
-#   deploy_devnet:
-#     runs-on: ubuntu-latest
-#     steps:
-#     - name: Checkout repository
-#       uses: actions/checkout@v2
-#       with:
-#         ref: ${{ github.event.before }}
-# 
-#     - name: Set up Docker
-#       uses: docker/setup-buildx-action@v1
-# 
-#     - name: Clone internal kurtosis-cdk repo
-#       run: |
-#         git clone https://github.com/0xPolygon/kurtosis-cdk.git
-#         cd kurtosis-cdk
-#         git checkout dan/jit_containers_superusers
+jobs:
+  deploy_devnet:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        ref: ${{ github.event.before }}
+
+    - name: Set up Docker
+      uses: docker/setup-buildx-action@v1
+
+    - name: Clone internal kurtosis-cdk repo
+      run: |
+        git clone https://github.com/0xPolygon/kurtosis-cdk.git
+        cd kurtosis-cdk
+        git checkout dan/jit_containers_superusers
 # 
 #     - name: Clone and build agglayer
 #       run: |
@@ -86,41 +86,41 @@ on:
         . .bashrc
         foundryup
         cast
-        
-    - name: Install kurtosis
-      run: |
-        echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
-        sudo apt update
-        sudo apt install kurtosis-cli
-
-    - name: Run kurtosis agent in background
-      run: |  
-        kurtosis gateway &  # Run cmd in background
-        sleep 10
-
-    - name: Deploy CDK devnet on local github runner
-      run: |
-        cd kurtosis-cdk
-        kurtosis engine restart
-        kurtosis run --enclave cdk-v1 --args-file params.yml .
-
-    - name: Auto-apply load and report any regressions here
-      run: |
-        bake_time="${{ github.event.inputs.bake_time }}"
-        end_minute=$(( $(date +'%M') + bake_time))
-
-        while [ $(date +'%M') -lt $end_minute ]; do
-          STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
-                    -H "Content-Type: application/json" \
-                    -H "DD-API-KEY: ${DATADOG_API_KEY}" \
-                    -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
-                    -s \
-                    | jq -r '.overall_state')
-          echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
-          sleep 60
-        done
-    
-    - name: Finally, remove all devnet resources locally
-      run: |
-        cd kurtosis-cdk
-        kurtosis clean -a
+    # 
+    # - name: Install kurtosis
+    #   run: |
+    #     echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+    #     sudo apt update
+    #     sudo apt install kurtosis-cli
+    # 
+    # - name: Run kurtosis agent in background
+    #   run: |  
+    #     kurtosis gateway &  # Run cmd in background
+    #     sleep 10
+    # 
+    # - name: Deploy CDK devnet on local github runner
+    #   run: |
+    #     cd kurtosis-cdk
+    #     kurtosis engine restart
+    #     kurtosis run --enclave cdk-v1 --args-file params.yml .
+    # 
+    # - name: Auto-apply load and report any regressions here
+    #   run: |
+    #     bake_time="${{ github.event.inputs.bake_time }}"
+    #     end_minute=$(( $(date +'%M') + bake_time))
+    # 
+    #     while [ $(date +'%M') -lt $end_minute ]; do
+    #       STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
+    #                 -H "Content-Type: application/json" \
+    #                 -H "DD-API-KEY: ${DATADOG_API_KEY}" \
+    #                 -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
+    #                 -s \
+    #                 | jq -r '.overall_state')
+    #       echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
+    #       sleep 60
+    #     done
+    # 
+    # - name: Finally, remove all devnet resources locally
+    #   run: |
+    #     cd kurtosis-cdk
+    #     kurtosis clean -a

From f5289ee0ea097728327544150b37dd50b1b2e493 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 9 Apr 2024 09:36:21 -0500
Subject: [PATCH 126/144] tester

---
 .github/workflows/cdk-superuser-regression-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 9de910e82e..42e630e37b 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -83,7 +83,7 @@ jobs:
     - name: Install foundry (i.e. cast)
       run: |    
         curl -L https://foundry.paradigm.xyz | bash 
-        . .bashrc
+        source /home/runner/.bashrc
         foundryup
         cast
     # 

From 2a9d03bb2a8c1d4f5e43952dfb11a5028a0945ff Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 9 Apr 2024 09:41:44 -0500
Subject: [PATCH 127/144] debug

---
 .../cdk-superuser-regression-tests.yml        | 202 +++++++++---------
 1 file changed, 96 insertions(+), 106 deletions(-)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 42e630e37b..a0e52f8c55 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -1,27 +1,26 @@
 name: REGRESSION TESTER - SUPERUSERS
 on:
-  push:
-#   workflow_dispatch:
-#     inputs:
-#       zkevm_agglayer_commit_id:
-#         description: '0xPolygon/agglayer (commit id)'
-#         required: true
-#       zkevm_bridge_service_commit_id:
-#         description: '0xPolygonHermez/zkevm-bridge-service (commit id)'
-#         required: true
-#       zkevm_bridge_ui_commit_id:
-#         description: '0xPolygonHermez/zkevm-bridge-ui (commit id)'
-#         required: true
-#       zkevm_dac_commit_id:
-#         description: '0xPolygon/cdk-data-availability (commit id)'
-#         required: true
-#       zkevm_node_commit_id:
-#         description: '0xPolygon/cdk-validium-node (commit id)'
-#         required: true
-#       bake_time:
-#         description: 'bake time (minutes)'
-#         required: true
-# 
+  workflow_dispatch:
+    inputs:
+      zkevm_agglayer_commit_id:
+        description: '0xPolygon/agglayer (commit id)'
+        required: true
+      zkevm_bridge_service_commit_id:
+        description: '0xPolygonHermez/zkevm-bridge-service (commit id)'
+        required: true
+      zkevm_bridge_ui_commit_id:
+        description: '0xPolygonHermez/zkevm-bridge-ui (commit id)'
+        required: true
+      zkevm_dac_commit_id:
+        description: '0xPolygon/cdk-data-availability (commit id)'
+        required: true
+      zkevm_node_commit_id:
+        description: '0xPolygon/cdk-validium-node (commit id)'
+        required: true
+      bake_time:
+        description: 'bake time (minutes)'
+        required: true
+
 jobs:
   deploy_devnet:
     runs-on: ubuntu-latest
@@ -33,94 +32,85 @@ jobs:
 
     - name: Set up Docker
       uses: docker/setup-buildx-action@v1
+      
+    - name: Install Foundry
+      uses: foundry-rs/foundry-toolchain@v1
 
     - name: Clone internal kurtosis-cdk repo
       run: |
         git clone https://github.com/0xPolygon/kurtosis-cdk.git
         cd kurtosis-cdk
         git checkout dan/jit_containers_superusers
-# 
-#     - name: Clone and build agglayer
-#       run: |
-#         git clone https://github.com/0xPolygon/agglayer.git
-#         cd agglayer
-#         git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
-#         docker compose -f docker/docker-compose.yaml build --no-cache agglayer
-#         sleep 10
-# 
-#     - name: Clone and build zkevm-bridge-service
-#       run: |
-#         git clone https://github.com/0xPolygonHermez/zkevm-bridge-service.git
-#         cd zkevm-bridge-service
-#         git checkout "${{ github.event.inputs.zkevm_bridge_service_commit_id }}"
-#         docker build -t zkevm-bridge-service:local -f ./Dockerfile .
-#         sleep 10
-# 
-#     - name: Clone and build zkevm-bridge-ui
-#       run: |
-#         git clone https://github.com/0xPolygonHermez/zkevm-bridge-ui.git
-#         cd zkevm-bridge-ui
-#         git checkout "${{ github.event.inputs.zkevm_bridge_ui_commit_id }}"
-#         docker build -t zkevm-bridge-ui:local -f ./Dockerfile .
-#         sleep 10
-# 
-#     - name: Clone and build cdk-data-availability
-#       run: |
-#         git clone https://github.com/0xPolygon/cdk-data-availability.git
-#         cd cdk-data-availability
-#         git checkout "${{ github.event.inputs.zkevm_dac_commit_id }}"
-#         docker build -t cdk-data-availability:local -f ./Dockerfile .
-#         sleep 10      
-# 
-#     - name: Clone and build cdk-validium-node
-#       run: |
-#         git clone https://github.com/0xPolygon/cdk-validium-node.git
-#         cd cdk-validium-node
-#         git checkout "${{ github.event.inputs.zkevm_node_commit_id }}"
-#         docker build -t cdk-validium-node:local -f ./Dockerfile .
-#         sleep 10
 
-    - name: Install foundry (i.e. cast)
-      run: |    
-        curl -L https://foundry.paradigm.xyz | bash 
-        source /home/runner/.bashrc
-        foundryup
-        cast
-    # 
-    # - name: Install kurtosis
-    #   run: |
-    #     echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
-    #     sudo apt update
-    #     sudo apt install kurtosis-cli
-    # 
-    # - name: Run kurtosis agent in background
-    #   run: |  
-    #     kurtosis gateway &  # Run cmd in background
-    #     sleep 10
-    # 
-    # - name: Deploy CDK devnet on local github runner
-    #   run: |
-    #     cd kurtosis-cdk
-    #     kurtosis engine restart
-    #     kurtosis run --enclave cdk-v1 --args-file params.yml .
-    # 
-    # - name: Auto-apply load and report any regressions here
-    #   run: |
-    #     bake_time="${{ github.event.inputs.bake_time }}"
-    #     end_minute=$(( $(date +'%M') + bake_time))
-    # 
-    #     while [ $(date +'%M') -lt $end_minute ]; do
-    #       STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
-    #                 -H "Content-Type: application/json" \
-    #                 -H "DD-API-KEY: ${DATADOG_API_KEY}" \
-    #                 -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
-    #                 -s \
-    #                 | jq -r '.overall_state')
-    #       echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
-    #       sleep 60
-    #     done
-    # 
-    # - name: Finally, remove all devnet resources locally
-    #   run: |
-    #     cd kurtosis-cdk
-    #     kurtosis clean -a
+    - name: Clone and build agglayer
+      run: |
+        git clone https://github.com/0xPolygon/agglayer.git
+        cd agglayer
+        git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
+        docker compose -f docker/docker-compose.yaml build --no-cache agglayer
+        sleep 10
+
+    - name: Clone and build zkevm-bridge-service
+      run: |
+        git clone https://github.com/0xPolygonHermez/zkevm-bridge-service.git
+        cd zkevm-bridge-service
+        git checkout "${{ github.event.inputs.zkevm_bridge_service_commit_id }}"
+        docker build -t zkevm-bridge-service:local -f ./Dockerfile .
+        sleep 10
+
+    - name: Clone and build zkevm-bridge-ui
+      run: |
+        git clone https://github.com/0xPolygonHermez/zkevm-bridge-ui.git
+        cd zkevm-bridge-ui
+        git checkout "${{ github.event.inputs.zkevm_bridge_ui_commit_id }}"
+        docker build -t zkevm-bridge-ui:local -f ./Dockerfile .
+        sleep 10
+
+    - name: Clone and build cdk-data-availability
+      run: |
+        git clone https://github.com/0xPolygon/cdk-data-availability.git
+        cd cdk-data-availability
+        git checkout "${{ github.event.inputs.zkevm_dac_commit_id }}"
+        docker build -t cdk-data-availability:local -f ./Dockerfile .
+        sleep 10      
+
+    - name: Clone and build cdk-validium-node
+      run: |
+        git clone https://github.com/0xPolygon/cdk-validium-node.git
+        cd cdk-validium-node
+        git checkout "${{ github.event.inputs.zkevm_node_commit_id }}"
+        docker build -t cdk-validium-node:local -f ./Dockerfile .
+        sleep 10
+    
+    - name: Install kurtosis
+      run: |
+        echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+        sudo apt update
+        sudo apt install kurtosis-cli
+    
+    - name: Run kurtosis agent in background
+      run: |  
+        kurtosis gateway &  # Run cmd in background
+        sleep 10
+    
+    - name: Deploy CDK devnet on local github runner
+      run: |
+        cd kurtosis-cdk
+        kurtosis engine restart
+        kurtosis run --enclave cdk-v1 --args-file params.yml .
+    
+    - name: Auto-apply load and report any regressions here
+      run: |
+        bake_time="${{ github.event.inputs.bake_time }}"
+        end_minute=$(( $(date +'%M') + bake_time))
+    
+        while [ $(date +'%M') -lt $end_minute ]; do
+          STATUS=$(cast rpc zkevm_verifiedBatchNumber)
+          echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
+          sleep 60
+        done
+    
+    - name: Finally, remove all devnet resources locally
+      run: |
+        cd kurtosis-cdk
+        kurtosis clean -a

From 4c658d06aeec88ce0c1cb5dda54ed0ce1323be13 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 9 Apr 2024 11:52:17 -0500
Subject: [PATCH 128/144] debug

---
 .github/workflows/cdk-superuser-regression-tests.yml | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index a0e52f8c55..1d284e19c3 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -105,8 +105,15 @@ jobs:
         end_minute=$(( $(date +'%M') + bake_time))
     
         while [ $(date +'%M') -lt $end_minute ]; do
-          STATUS=$(cast rpc zkevm_verifiedBatchNumber)
-          echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
+          # Attempt to connect to the service
+          if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
+            echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
+          else
+            # If connection fails, wait and retry after some time
+            echo "Failed to connect, waiting and retrying..."
+            sleep 10
+            continue
+          fi
           sleep 60
         done
     

From 1c97ef793f2ab5bc66c7c98b94113be51e59307d Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 9 Apr 2024 11:53:55 -0500
Subject: [PATCH 129/144] debugin

---
 .github/workflows/cdk-superuser-regression-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 1d284e19c3..f81a9f1d55 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -111,7 +111,7 @@ jobs:
           else
             # If connection fails, wait and retry after some time
             echo "Failed to connect, waiting and retrying..."
-            sleep 10
+            sleep 20
             continue
           fi
           sleep 60

From 953bab4e8e8b4438b8912550c99392b03c35cde9 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 9 Apr 2024 23:10:14 -0500
Subject: [PATCH 130/144] debug

---
 .github/workflows/cdk-superuser-regression-tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index f81a9f1d55..66f9b83f72 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -103,6 +103,7 @@ jobs:
       run: |
         bake_time="${{ github.event.inputs.bake_time }}"
         end_minute=$(( $(date +'%M') + bake_time))
+        export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
     
         while [ $(date +'%M') -lt $end_minute ]; do
           # Attempt to connect to the service

From 8e2f814e7eba7450ce53666e5747f49a9a8352a4 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 10 Apr 2024 00:01:55 -0500
Subject: [PATCH 131/144] fix

---
 .../workflows/agglayer-regression-tests.yml   | 56 ++++++++++++-------
 .../cdk-superuser-regression-tests.yml        | 23 ++++++--
 2 files changed, 53 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index 3ab4d20f01..9151f4da7e 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -1,4 +1,4 @@
-name: REGRESSION TESTER - AGGLAYER SPECIFIC
+name: REGRESSION TESTER - AGGLAYER
 on:
   workflow_dispatch:
     inputs:
@@ -20,13 +20,16 @@ jobs:
 
     - name: Set up Docker
       uses: docker/setup-buildx-action@v1
+      
+    - name: Install Foundry
+      uses: foundry-rs/foundry-toolchain@v1
 
     - name: Clone internal kurtosis-cdk repo
       run: |
         git clone https://github.com/0xPolygon/kurtosis-cdk.git
         cd kurtosis-cdk
         git checkout dan/jit_containers
-        
+
     - name: Clone and build agglayer
       run: |
         git clone https://github.com/0xPolygon/agglayer.git
@@ -34,45 +37,56 @@ jobs:
         git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
         docker compose -f docker/docker-compose.yaml build --no-cache agglayer
         sleep 10
-        
+
     - name: Install kurtosis
       run: |
         echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
         sudo apt update
         sudo apt install kurtosis-cli
-
+    
     - name: Run kurtosis agent in background
       run: |  
         kurtosis gateway &  # Run cmd in background
         sleep 10
-
+    
     - name: Deploy CDK devnet on local github runner
       run: |
         cd kurtosis-cdk
         kurtosis engine restart
         kurtosis run --enclave cdk-v1 --args-file params.yml .
-
-    - name: Auto-apply load and report any regressions here
+    
+    - name: Monitor and report any potential regressions to CI logs
       run: |
-        MONITOR_ID="140974014"  # K8S DEVNET: COMPOSITE MONITOR ID
-        DATADOG_API_KEY="${{ secrets.DATADOG_API_KEY }}"
-        DATADOG_APP_KEY="${{ secrets.DATADOG_APP_KEY }}"
-
         bake_time="${{ github.event.inputs.bake_time }}"
         end_minute=$(( $(date +'%M') + bake_time))
-
+        export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
+        greater_than_zero=false
+        
         while [ $(date +'%M') -lt $end_minute ]; do
-          STATUS=$(curl -X GET "https://api.datadoghq.com/api/v1/monitor/${MONITOR_ID}" \
-                    -H "Content-Type: application/json" \
-                    -H "DD-API-KEY: ${DATADOG_API_KEY}" \
-                    -H "DD-APPLICATION-KEY: ${DATADOG_APP_KEY}" \
-                    -s \
-                    | jq -r '.overall_state')
-          echo "K8S DEVNET HEALTH STATUS: $STATUS"
-          echo "Review your devnet health dashboard here: https://app.datadoghq.com/dashboard/fqu-nh2-bzd?fromUser=false&refresh_mode=sliding&view=spans&from_ts=1709886652761&to_ts=1709890252761&live=true"
+          # Attempt connecting to service
+          if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
+            # Convert hexadecimal to decimal
+            DEC_STATUS=$(printf "%d" "$STATUS")
+            echo "ZKEVM_VERIFIED_BATCH_NUMBER: $DEC_STATUS"
+            
+            # Check if STATUS is greater than 0
+            if [ $DEC_STATUS -gt 0 ]; then
+              greater_than_zero=true
+            fi
+          else
+            echo "Failed to connect, waiting and retrying..."
+            sleep 60
+            continue
+          fi
           sleep 60
         done
-
+        
+        # Check that zkevm_verifiedBatchNumber increments, indicating e2e success
+        if ! $greater_than_zero; then
+          echo "zkevm_verifiedBatchNumber failed to increment past 0. marking CI job failed accordingly"
+          exit 1
+        fi
+    
     - name: Finally, remove all devnet resources locally
       run: |
         cd kurtosis-cdk
diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 66f9b83f72..6a33b0f67b 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -104,19 +104,32 @@ jobs:
         bake_time="${{ github.event.inputs.bake_time }}"
         end_minute=$(( $(date +'%M') + bake_time))
         export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
-    
+        greater_than_zero=false
+        
         while [ $(date +'%M') -lt $end_minute ]; do
-          # Attempt to connect to the service
+          # Attempt connecting to service
           if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
-            echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
+            # Convert hexadecimal to decimal
+            DEC_STATUS=$(printf "%d" "$STATUS")
+            echo "ZKEVM_VERIFIED_BATCH_NUMBER: $DEC_STATUS"
+            
+            # Check if STATUS is greater than 0
+            if [ $DEC_STATUS -gt 0 ]; then
+              greater_than_zero=true
+            fi
           else
-            # If connection fails, wait and retry after some time
             echo "Failed to connect, waiting and retrying..."
-            sleep 20
+            sleep 60
             continue
           fi
           sleep 60
         done
+        
+        # Check if zkevm_verifiedBatchNumber becomes greater than 0, indicating e2e success
+        if ! $greater_than_zero; then
+          echo "zkevm_verifiedBatchNumber failed to increment past 0. marking CI job failed accordingly"
+          exit 1
+        fi
     
     - name: Finally, remove all devnet resources locally
       run: |

From 49f350d1d5025c689cac1a680aaec9c4d469d14a Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 10 Apr 2024 00:04:52 -0500
Subject: [PATCH 132/144] cleanup

---
 .github/workflows/cdk-superuser-regression-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 6a33b0f67b..b3647b93e7 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -99,7 +99,7 @@ jobs:
         kurtosis engine restart
         kurtosis run --enclave cdk-v1 --args-file params.yml .
     
-    - name: Auto-apply load and report any regressions here
+    - name: Monitor and report any potential regressions to CI logs
       run: |
         bake_time="${{ github.event.inputs.bake_time }}"
         end_minute=$(( $(date +'%M') + bake_time))

From f7ddd52ae8e46374aa320295e3ec51de889d3dd9 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 10 Apr 2024 00:26:47 -0500
Subject: [PATCH 133/144] debug

---
 .github/workflows/agglayer-regression-tests.yml      | 9 +++++----
 .github/workflows/cdk-superuser-regression-tests.yml | 9 +++++----
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index 9151f4da7e..487b1e3138 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -63,10 +63,10 @@ jobs:
         greater_than_zero=false
         
         while [ $(date +'%M') -lt $end_minute ]; do
-          # Attempt connecting to service
+          # Attempt to connect to the service
           if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
             # Convert hexadecimal to decimal
-            DEC_STATUS=$(printf "%d" "$STATUS")
+            DEC_STATUS=$((16#$STATUS))
             echo "ZKEVM_VERIFIED_BATCH_NUMBER: $DEC_STATUS"
             
             # Check if STATUS is greater than 0
@@ -74,6 +74,7 @@ jobs:
               greater_than_zero=true
             fi
           else
+            # If connection fails, wait and retry after some time
             echo "Failed to connect, waiting and retrying..."
             sleep 60
             continue
@@ -81,9 +82,9 @@ jobs:
           sleep 60
         done
         
-        # Check that zkevm_verifiedBatchNumber increments, indicating e2e success
+        # Check if STATUS field never becomes greater than 0
         if ! $greater_than_zero; then
-          echo "zkevm_verifiedBatchNumber failed to increment past 0. marking CI job failed accordingly"
+          echo "STATUS field never becomes greater than 0. CI job failed."
           exit 1
         fi
     
diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index b3647b93e7..e05f1728d6 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -107,10 +107,10 @@ jobs:
         greater_than_zero=false
         
         while [ $(date +'%M') -lt $end_minute ]; do
-          # Attempt connecting to service
+          # Attempt to connect to the service
           if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
             # Convert hexadecimal to decimal
-            DEC_STATUS=$(printf "%d" "$STATUS")
+            DEC_STATUS=$((16#$STATUS))
             echo "ZKEVM_VERIFIED_BATCH_NUMBER: $DEC_STATUS"
             
             # Check if STATUS is greater than 0
@@ -118,6 +118,7 @@ jobs:
               greater_than_zero=true
             fi
           else
+            # If connection fails, wait and retry after some time
             echo "Failed to connect, waiting and retrying..."
             sleep 60
             continue
@@ -125,9 +126,9 @@ jobs:
           sleep 60
         done
         
-        # Check if zkevm_verifiedBatchNumber becomes greater than 0, indicating e2e success
+        # Check if STATUS field never becomes greater than 0
         if ! $greater_than_zero; then
-          echo "zkevm_verifiedBatchNumber failed to increment past 0. marking CI job failed accordingly"
+          echo "STATUS field never becomes greater than 0. CI job failed."
           exit 1
         fi
     

From 191349dfa883b099ceb511c57759a4a37bdf6f17 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 10 Apr 2024 00:59:28 -0500
Subject: [PATCH 134/144] cmon

---
 .../workflows/agglayer-regression-tests.yml   | 19 +++++++++----------
 .../cdk-superuser-regression-tests.yml        | 19 +++++++++----------
 2 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index 487b1e3138..3304c1fa1b 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -60,18 +60,18 @@ jobs:
         bake_time="${{ github.event.inputs.bake_time }}"
         end_minute=$(( $(date +'%M') + bake_time))
         export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
-        greater_than_zero=false
+        incremented=false
         
         while [ $(date +'%M') -lt $end_minute ]; do
           # Attempt to connect to the service
           if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
-            # Convert hexadecimal to decimal
-            DEC_STATUS=$((16#$STATUS))
-            echo "ZKEVM_VERIFIED_BATCH_NUMBER: $DEC_STATUS"
+            echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
             
-            # Check if STATUS is greater than 0
-            if [ $DEC_STATUS -gt 0 ]; then
-              greater_than_zero=true
+            # Check if STATUS is not 0x0
+            if [ "$STATUS" != "0x0" ]; then
+              incremented=true
+              echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
+              exit 0
             fi
           else
             # If connection fails, wait and retry after some time
@@ -82,9 +82,8 @@ jobs:
           sleep 60
         done
         
-        # Check if STATUS field never becomes greater than 0
-        if ! $greater_than_zero; then
-          echo "STATUS field never becomes greater than 0. CI job failed."
+        if ! $incremented; then
+          echo "ZKEVM_VERIFIED_BATCH_NUMBER remained at 0x0 till the end. CI job failed."
           exit 1
         fi
     
diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index e05f1728d6..a6addacda1 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -104,18 +104,18 @@ jobs:
         bake_time="${{ github.event.inputs.bake_time }}"
         end_minute=$(( $(date +'%M') + bake_time))
         export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
-        greater_than_zero=false
+        incremented=false
         
         while [ $(date +'%M') -lt $end_minute ]; do
           # Attempt to connect to the service
           if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
-            # Convert hexadecimal to decimal
-            DEC_STATUS=$((16#$STATUS))
-            echo "ZKEVM_VERIFIED_BATCH_NUMBER: $DEC_STATUS"
+            echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
             
-            # Check if STATUS is greater than 0
-            if [ $DEC_STATUS -gt 0 ]; then
-              greater_than_zero=true
+            # Check if STATUS is not 0x0
+            if [ "$STATUS" != "0x0" ]; then
+              incremented=true
+              echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
+              exit 0
             fi
           else
             # If connection fails, wait and retry after some time
@@ -126,9 +126,8 @@ jobs:
           sleep 60
         done
         
-        # Check if STATUS field never becomes greater than 0
-        if ! $greater_than_zero; then
-          echo "STATUS field never becomes greater than 0. CI job failed."
+        if ! $incremented; then
+          echo "ZKEVM_VERIFIED_BATCH_NUMBER remained at 0x0 till the end. CI job failed."
           exit 1
         fi
     

From 4bbf0eca3bddd6fabe2ca262cbb3e519a37317cf Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 10 Apr 2024 01:16:22 -0500
Subject: [PATCH 135/144] refine verifiedBatch check

---
 .github/workflows/agglayer-regression-tests.yml      | 5 +++--
 .github/workflows/cdk-superuser-regression-tests.yml | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index 3304c1fa1b..0b06c96e35 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -60,6 +60,7 @@ jobs:
         bake_time="${{ github.event.inputs.bake_time }}"
         end_minute=$(( $(date +'%M') + bake_time))
         export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
+        INITIAL_STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null)
         incremented=false
         
         while [ $(date +'%M') -lt $end_minute ]; do
@@ -67,8 +68,8 @@ jobs:
           if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
             echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
             
-            # Check if STATUS is not 0x0
-            if [ "$STATUS" != "0x0" ]; then
+            # Check if STATUS has incremented
+            if [ "$STATUS" != "$INITIAL_STATUS" ]; then
               incremented=true
               echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
               exit 0
diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index a6addacda1..bedf47b468 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -104,6 +104,7 @@ jobs:
         bake_time="${{ github.event.inputs.bake_time }}"
         end_minute=$(( $(date +'%M') + bake_time))
         export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
+        INITIAL_STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null)
         incremented=false
         
         while [ $(date +'%M') -lt $end_minute ]; do
@@ -111,8 +112,8 @@ jobs:
           if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
             echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
             
-            # Check if STATUS is not 0x0
-            if [ "$STATUS" != "0x0" ]; then
+            # Check if STATUS has incremented
+            if [ "$STATUS" != "$INITIAL_STATUS" ]; then
               incremented=true
               echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
               exit 0

From 420b3f0ea4e2ebdfcca1b2efbf44ad5d5a0c0147 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 10 Apr 2024 01:23:35 -0500
Subject: [PATCH 136/144] debug

---
 .github/workflows/agglayer-regression-tests.yml      | 5 +----
 .github/workflows/cdk-superuser-regression-tests.yml | 5 +----
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index 0b06c96e35..f0c7945b01 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -71,11 +71,8 @@ jobs:
             # Check if STATUS has incremented
             if [ "$STATUS" != "$INITIAL_STATUS" ]; then
               incremented=true
-              echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
-              exit 0
             fi
           else
-            # If connection fails, wait and retry after some time
             echo "Failed to connect, waiting and retrying..."
             sleep 60
             continue
@@ -84,7 +81,7 @@ jobs:
         done
         
         if ! $incremented; then
-          echo "ZKEVM_VERIFIED_BATCH_NUMBER remained at 0x0 till the end. CI job failed."
+          echo "ZKEVM_VERIFIED_BATCH_NUMBER did not increment. This may indicate chain experienced a regression. Please investigate."
           exit 1
         fi
     
diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index bedf47b468..f56b8eb2b0 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -115,11 +115,8 @@ jobs:
             # Check if STATUS has incremented
             if [ "$STATUS" != "$INITIAL_STATUS" ]; then
               incremented=true
-              echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
-              exit 0
             fi
           else
-            # If connection fails, wait and retry after some time
             echo "Failed to connect, waiting and retrying..."
             sleep 60
             continue
@@ -128,7 +125,7 @@ jobs:
         done
         
         if ! $incremented; then
-          echo "ZKEVM_VERIFIED_BATCH_NUMBER remained at 0x0 till the end. CI job failed."
+          echo "ZKEVM_VERIFIED_BATCH_NUMBER did not increment. This may indicate chain experienced a regression. Please investigate."
           exit 1
         fi
     

From 959e8b16ad1bf9a9c574623b36ab399655a97331 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 10 Apr 2024 06:27:08 -0500
Subject: [PATCH 137/144] pr cleanup

---
 .../workflows/agglayer-regression-tests.yml   | 152 +++++++++---------
 1 file changed, 76 insertions(+), 76 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index f0c7945b01..0e23b9cc88 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -1,91 +1,91 @@
 name: REGRESSION TESTER - AGGLAYER
-on:
+'on':
   workflow_dispatch:
     inputs:
       zkevm_agglayer_commit_id:
-        description: '0xPolygon/agglayer (commit id)'
-        required: true
+        description: 0xPolygon/agglayer (commit id)
+        required: false
+        default: b9d79c6
       bake_time:
-        description: 'bake time (minutes)'
-        required: true
-
+        description: bake time (minutes)
+        required: false
+        default: 30
 jobs:
   deploy_devnet:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ github.event.before }}
+      - name: Checkout repository
+        uses: actions/checkout@v2
+        with:
+          ref: '${{ github.event.before }}'
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v1
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+      - name: Clone internal kurtosis-cdk repo
+        run: |
+          git clone https://github.com/0xPolygon/kurtosis-cdk.git
+          cd kurtosis-cdk
+          git checkout dan/jit_containers
+      - name: Clone and build agglayer
+        run: |
+          git clone https://github.com/0xPolygon/agglayer.git
+          cd agglayer
+          git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
+          docker compose -f docker/docker-compose.yaml build --no-cache agglayer
+      - name: Install kurtosis
+        run: >
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo
+          tee /etc/apt/sources.list.d/kurtosis.list
+
+          sudo apt update
+
+          sudo apt install kurtosis-cli
+      - name: Run kurtosis agent in background
+        run: |
+          kurtosis gateway &  # Run cmd in background
+      - name: Deploy CDK devnet on local github runner
+        run: |
+          cd kurtosis-cdk
+          kurtosis engine restart
+          kurtosis run --enclave cdk-v1 --args-file params.yml .
+      - name: Monitor and report any potential regressions to CI logs
+        run: >
+          bake_time="${{ github.event.inputs.bake_time }}"
+
+          end_minute=$(( $(date +'%M') + bake_time))
 
-    - name: Set up Docker
-      uses: docker/setup-buildx-action@v1
-      
-    - name: Install Foundry
-      uses: foundry-rs/foundry-toolchain@v1
+          export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001
+          http-rpc)"
 
-    - name: Clone internal kurtosis-cdk repo
-      run: |
-        git clone https://github.com/0xPolygon/kurtosis-cdk.git
-        cd kurtosis-cdk
-        git checkout dan/jit_containers
+          INITIAL_STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null)
 
-    - name: Clone and build agglayer
-      run: |
-        git clone https://github.com/0xPolygon/agglayer.git
-        cd agglayer
-        git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
-        docker compose -f docker/docker-compose.yaml build --no-cache agglayer
-        sleep 10
+          incremented=false
 
-    - name: Install kurtosis
-      run: |
-        echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
-        sudo apt update
-        sudo apt install kurtosis-cli
-    
-    - name: Run kurtosis agent in background
-      run: |  
-        kurtosis gateway &  # Run cmd in background
-        sleep 10
-    
-    - name: Deploy CDK devnet on local github runner
-      run: |
-        cd kurtosis-cdk
-        kurtosis engine restart
-        kurtosis run --enclave cdk-v1 --args-file params.yml .
-    
-    - name: Monitor and report any potential regressions to CI logs
-      run: |
-        bake_time="${{ github.event.inputs.bake_time }}"
-        end_minute=$(( $(date +'%M') + bake_time))
-        export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
-        INITIAL_STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null)
-        incremented=false
-        
-        while [ $(date +'%M') -lt $end_minute ]; do
-          # Attempt to connect to the service
-          if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
-            echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
-            
-            # Check if STATUS has incremented
-            if [ "$STATUS" != "$INITIAL_STATUS" ]; then
-              incremented=true
+
+          while [ $(date +'%M') -lt $end_minute ]; do
+            # Attempt to connect to the service
+            if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
+              echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
+              
+              # Check if STATUS has incremented
+              if [ "$STATUS" != "$INITIAL_STATUS" ]; then
+                incremented=true
+              fi
+            else
+              echo "Failed to connect, waiting and retrying..."
+              sleep 30
+              continue
             fi
-          else
-            echo "Failed to connect, waiting and retrying..."
-            sleep 60
-            continue
+            sleep 30
+          done
+
+
+          if ! $incremented; then
+            echo "ZKEVM_VERIFIED_BATCH_NUMBER did not increment. This may indicate chain experienced a regression. Please investigate."
+            exit 1
           fi
-          sleep 60
-        done
-        
-        if ! $incremented; then
-          echo "ZKEVM_VERIFIED_BATCH_NUMBER did not increment. This may indicate chain experienced a regression. Please investigate."
-          exit 1
-        fi
-    
-    - name: Finally, remove all devnet resources locally
-      run: |
-        cd kurtosis-cdk
-        kurtosis clean -a
+      - name: Finally, remove all devnet resources locally
+        run: |
+          cd kurtosis-cdk
+          kurtosis clean -a

From dad7a0dfefbc32f657b361c294ff90c48ee97cc3 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 10 Apr 2024 07:34:13 -0500
Subject: [PATCH 138/144] debug bake time logic

---
 .../workflows/agglayer-regression-tests.yml   | 31 ++++++++-----------
 .../cdk-superuser-regression-tests.yml        |  2 ++
 2 files changed, 15 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index 0e23b9cc88..8087379033 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -1,4 +1,4 @@
-name: REGRESSION TESTER - AGGLAYER
+name: Regression Tests
 'on':
   workflow_dispatch:
     inputs:
@@ -20,6 +20,12 @@ jobs:
           ref: '${{ github.event.before }}'
       - name: Set up Docker
         uses: docker/setup-buildx-action@v1
+      - name: Clone and build agglayer image locally
+        run: |
+          git clone https://github.com/0xPolygon/agglayer.git
+          cd agglayer
+          git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
+          docker compose -f docker/docker-compose.yaml build --no-cache agglayer
       - name: Install Foundry
         uses: foundry-rs/foundry-toolchain@v1
       - name: Clone internal kurtosis-cdk repo
@@ -27,19 +33,11 @@ jobs:
           git clone https://github.com/0xPolygon/kurtosis-cdk.git
           cd kurtosis-cdk
           git checkout dan/jit_containers
-      - name: Clone and build agglayer
-        run: |
-          git clone https://github.com/0xPolygon/agglayer.git
-          cd agglayer
-          git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
-          docker compose -f docker/docker-compose.yaml build --no-cache agglayer
       - name: Install kurtosis
-        run: >
+        run: |
           echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo
           tee /etc/apt/sources.list.d/kurtosis.list
-
           sudo apt update
-
           sudo apt install kurtosis-cli
       - name: Run kurtosis agent in background
         run: |
@@ -50,19 +48,15 @@ jobs:
           kurtosis engine restart
           kurtosis run --enclave cdk-v1 --args-file params.yml .
       - name: Monitor and report any potential regressions to CI logs
-        run: >
+        run: |
           bake_time="${{ github.event.inputs.bake_time }}"
-
           end_minute=$(( $(date +'%M') + bake_time))
 
           export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001
           http-rpc)"
-
           INITIAL_STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null)
-
           incremented=false
 
-
           while [ $(date +'%M') -lt $end_minute ]; do
             # Attempt to connect to the service
             if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
@@ -71,16 +65,17 @@ jobs:
               # Check if STATUS has incremented
               if [ "$STATUS" != "$INITIAL_STATUS" ]; then
                 incremented=true
+                echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
+                exit 0
               fi
             else
               echo "Failed to connect, waiting and retrying..."
-              sleep 30
+              sleep 60
               continue
             fi
-            sleep 30
+            sleep 60
           done
 
-
           if ! $incremented; then
             echo "ZKEVM_VERIFIED_BATCH_NUMBER did not increment. This may indicate chain experienced a regression. Please investigate."
             exit 1
diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index f56b8eb2b0..ffeb9055da 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -115,6 +115,8 @@ jobs:
             # Check if STATUS has incremented
             if [ "$STATUS" != "$INITIAL_STATUS" ]; then
               incremented=true
+              echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
+              exit 0
             fi
           else
             echo "Failed to connect, waiting and retrying..."

From 8eb1f65d0b3a3ed5f062c742cce314faaa95e8b3 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 10 Apr 2024 07:41:09 -0500
Subject: [PATCH 139/144] fix: kurtosis install

---
 .github/workflows/agglayer-regression-tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index 8087379033..bb77512c42 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -35,8 +35,7 @@ jobs:
           git checkout dan/jit_containers
       - name: Install kurtosis
         run: |
-          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo
-          tee /etc/apt/sources.list.d/kurtosis.list
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
           sudo apt update
           sudo apt install kurtosis-cli
       - name: Run kurtosis agent in background

From 0231676c49f890ba9b2d1bd4ee2bdcc624d5f4a7 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 10 Apr 2024 07:56:21 -0500
Subject: [PATCH 140/144] fix: typo

---
 .github/workflows/agglayer-regression-tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index bb77512c42..ee42d15e9d 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -51,8 +51,7 @@ jobs:
           bake_time="${{ github.event.inputs.bake_time }}"
           end_minute=$(( $(date +'%M') + bake_time))
 
-          export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001
-          http-rpc)"
+          export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
           INITIAL_STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null)
           incremented=false
 

From 578c7841500b90a6e3016d617e70a01444f00f26 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 10 Apr 2024 09:33:18 -0500
Subject: [PATCH 141/144] run on merge to main

---
 .github/workflows/agglayer-regression-tests.yml | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index ee42d15e9d..99b7e3f09f 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -4,12 +4,25 @@ name: Regression Tests
     inputs:
       zkevm_agglayer_commit_id:
         description: 0xPolygon/agglayer (commit id)
-        required: false
-        default: b9d79c6
+        required: true
       bake_time:
         description: bake time (minutes)
         required: false
         default: 30
+  push:
+    branches:
+      - main
+    paths:
+      - 'agglayer/**'
+    inputs:
+      zkevm_agglayer_commit_id:
+        description: 0xPolygon/agglayer commit id
+        required: false
+        default: '${{ github.event.before }}' # Set default value to commit ID being merged to main
+      bake_time:
+        description: bake time (minutes)
+        required: false
+        default: 20
 jobs:
   deploy_devnet:
     runs-on: ubuntu-latest

From 93db54004d71693d893727051121d932a5bc2971 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Wed, 10 Apr 2024 09:39:20 -0500
Subject: [PATCH 142/144] test auto main tests

---
 .github/workflows/agglayer-regression-tests.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index 99b7e3f09f..e5880daec1 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -12,8 +12,6 @@ name: Regression Tests
   push:
     branches:
       - main
-    paths:
-      - 'agglayer/**'
     inputs:
       zkevm_agglayer_commit_id:
         description: 0xPolygon/agglayer commit id

From 7120961fdf148a21d9336fc3cfb482108980de27 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 16 Apr 2024 12:04:50 -0500
Subject: [PATCH 143/144] final test

---
 .../cdk-superuser-regression-tests.yml        | 203 ++++++++----------
 1 file changed, 88 insertions(+), 115 deletions(-)

diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index ffeb9055da..64e11bf7f1 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -1,137 +1,110 @@
-name: REGRESSION TESTER - SUPERUSERS
-on:
+name: Regression Tests
+'on':
   workflow_dispatch:
     inputs:
       zkevm_agglayer_commit_id:
-        description: '0xPolygon/agglayer (commit id)'
+        description: 0xPolygon/agglayer (commit id)
         required: true
       zkevm_bridge_service_commit_id:
-        description: '0xPolygonHermez/zkevm-bridge-service (commit id)'
+        description: 0xPolygonHermez/zkevm-bridge-service (commit id)
         required: true
       zkevm_bridge_ui_commit_id:
-        description: '0xPolygonHermez/zkevm-bridge-ui (commit id)'
+        description: 0xPolygonHermez/zkevm-bridge-ui (commit id)
         required: true
       zkevm_dac_commit_id:
-        description: '0xPolygon/cdk-data-availability (commit id)'
+        description: 0xPolygon/cdk-data-availability (commit id)
         required: true
       zkevm_node_commit_id:
-        description: '0xPolygon/cdk-validium-node (commit id)'
+        description: 0xPolygon/cdk-validium-node (commit id)
         required: true
       bake_time:
-        description: 'bake time (minutes)'
-        required: true
-
+        description: bake time (minutes)
+        required: false
+        default: 30
 jobs:
   deploy_devnet:
     runs-on: ubuntu-latest
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        ref: ${{ github.event.before }}
-
-    - name: Set up Docker
-      uses: docker/setup-buildx-action@v1
-      
-    - name: Install Foundry
-      uses: foundry-rs/foundry-toolchain@v1
-
-    - name: Clone internal kurtosis-cdk repo
-      run: |
-        git clone https://github.com/0xPolygon/kurtosis-cdk.git
-        cd kurtosis-cdk
-        git checkout dan/jit_containers_superusers
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Set up Docker
+        uses: docker/setup-buildx-action@v1
+      - name: Clone and build agglayer
+        run: |
+          git clone https://github.com/0xPolygon/agglayer.git
+          cd agglayer
+          git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
+          docker compose -f docker/docker-compose.yaml build --no-cache agglayer
+      - name: Clone and build zkevm-bridge-service
+        run: |
+          git clone https://github.com/0xPolygonHermez/zkevm-bridge-service.git
+          cd zkevm-bridge-service
+          git checkout "${{ github.event.inputs.zkevm_bridge_service_commit_id }}"
+          docker build -t zkevm-bridge-service:local -f ./Dockerfile .
+      - name: Clone and build zkevm-bridge-ui
+        run: |
+          git clone https://github.com/0xPolygonHermez/zkevm-bridge-ui.git
+          cd zkevm-bridge-ui
+          git checkout "${{ github.event.inputs.zkevm_bridge_ui_commit_id }}"
+          docker build -t zkevm-bridge-ui:local -f ./Dockerfile .
+      - name: Clone and build cdk-data-availability
+        run: |
+          git clone https://github.com/0xPolygon/cdk-data-availability.git
+          cd cdk-data-availability
+          git checkout "${{ github.event.inputs.zkevm_dac_commit_id }}"
+          docker build -t cdk-data-availability:local -f ./Dockerfile .    
+      - name: Clone and build cdk-validium-node
+        run: |
+          git clone https://github.com/0xPolygon/cdk-validium-node.git
+          cd cdk-validium-node
+          git checkout "${{ github.event.inputs.zkevm_node_commit_id }}"
+          docker build -t cdk-validium-node:local -f ./Dockerfile .
+      - name: Install Foundry
+        uses: foundry-rs/foundry-toolchain@v1
+      - name: Clone internal kurtosis-cdk repo
+        run: |
+          git clone https://github.com/0xPolygon/kurtosis-cdk.git
+          cd kurtosis-cdk
+          git checkout dan/jit_containers_superusers
+      - name: Install kurtosis
+        run: |
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt update
+          sudo apt install kurtosis-cli
+          kurtosis analytics disable
+      - name: Deploy CDK devnet on local github runner
+        run: |
+          cd kurtosis-cdk
+          kurtosis run --enclave cdk-v1 --args-file params.yml .
+      - name: Monitor and report any potential regressions to CI logs
+        run: |
+          bake_time="${{ github.event.inputs.bake_time }}"
+          end_minute=$(( $(date +'%M') + bake_time))
 
-    - name: Clone and build agglayer
-      run: |
-        git clone https://github.com/0xPolygon/agglayer.git
-        cd agglayer
-        git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
-        docker compose -f docker/docker-compose.yaml build --no-cache agglayer
-        sleep 10
+          export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
+          INITIAL_STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null)
+          incremented=false
 
-    - name: Clone and build zkevm-bridge-service
-      run: |
-        git clone https://github.com/0xPolygonHermez/zkevm-bridge-service.git
-        cd zkevm-bridge-service
-        git checkout "${{ github.event.inputs.zkevm_bridge_service_commit_id }}"
-        docker build -t zkevm-bridge-service:local -f ./Dockerfile .
-        sleep 10
-
-    - name: Clone and build zkevm-bridge-ui
-      run: |
-        git clone https://github.com/0xPolygonHermez/zkevm-bridge-ui.git
-        cd zkevm-bridge-ui
-        git checkout "${{ github.event.inputs.zkevm_bridge_ui_commit_id }}"
-        docker build -t zkevm-bridge-ui:local -f ./Dockerfile .
-        sleep 10
-
-    - name: Clone and build cdk-data-availability
-      run: |
-        git clone https://github.com/0xPolygon/cdk-data-availability.git
-        cd cdk-data-availability
-        git checkout "${{ github.event.inputs.zkevm_dac_commit_id }}"
-        docker build -t cdk-data-availability:local -f ./Dockerfile .
-        sleep 10      
-
-    - name: Clone and build cdk-validium-node
-      run: |
-        git clone https://github.com/0xPolygon/cdk-validium-node.git
-        cd cdk-validium-node
-        git checkout "${{ github.event.inputs.zkevm_node_commit_id }}"
-        docker build -t cdk-validium-node:local -f ./Dockerfile .
-        sleep 10
-    
-    - name: Install kurtosis
-      run: |
-        echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
-        sudo apt update
-        sudo apt install kurtosis-cli
-    
-    - name: Run kurtosis agent in background
-      run: |  
-        kurtosis gateway &  # Run cmd in background
-        sleep 10
-    
-    - name: Deploy CDK devnet on local github runner
-      run: |
-        cd kurtosis-cdk
-        kurtosis engine restart
-        kurtosis run --enclave cdk-v1 --args-file params.yml .
-    
-    - name: Monitor and report any potential regressions to CI logs
-      run: |
-        bake_time="${{ github.event.inputs.bake_time }}"
-        end_minute=$(( $(date +'%M') + bake_time))
-        export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
-        INITIAL_STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null)
-        incremented=false
-        
-        while [ $(date +'%M') -lt $end_minute ]; do
-          # Attempt to connect to the service
-          if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
-            echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
-            
-            # Check if STATUS has incremented
-            if [ "$STATUS" != "$INITIAL_STATUS" ]; then
-              incremented=true
-              echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
-              exit 0
+          while [ $(date +'%M') -lt $end_minute ]; do
+            # Attempt to connect to the service
+            if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
+              echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
+              
+              # Check if STATUS has incremented
+              if [ "$STATUS" != "$INITIAL_STATUS" ]; then
+                incremented=true
+                echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
+                exit 0
+              fi
+            else
+              echo "Failed to connect, waiting and retrying..."
+              sleep 60
+              continue
             fi
-          else
-            echo "Failed to connect, waiting and retrying..."
             sleep 60
-            continue
+          done
+
+          if ! $incremented; then
+            echo "ZKEVM_VERIFIED_BATCH_NUMBER did not increment. This may indicate chain experienced a regression. Please investigate."
+            exit 1
           fi
-          sleep 60
-        done
-        
-        if ! $incremented; then
-          echo "ZKEVM_VERIFIED_BATCH_NUMBER did not increment. This may indicate chain experienced a regression. Please investigate."
-          exit 1
-        fi
-    
-    - name: Finally, remove all devnet resources locally
-      run: |
-        cd kurtosis-cdk
-        kurtosis clean -a

From 357211691913b2bab096e1068f55c8e0eaee3695 Mon Sep 17 00:00:00 2001
From: dan moore <dmoore@polygon.technology>
Date: Tue, 16 Apr 2024 12:07:46 -0500
Subject: [PATCH 144/144] debug

---
 .github/workflows/agglayer-regression-tests.yml      | 2 +-
 .github/workflows/cdk-superuser-regression-tests.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/agglayer-regression-tests.yml b/.github/workflows/agglayer-regression-tests.yml
index e5880daec1..aa9cc1a7a2 100644
--- a/.github/workflows/agglayer-regression-tests.yml
+++ b/.github/workflows/agglayer-regression-tests.yml
@@ -1,4 +1,4 @@
-name: Regression Tests
+name: Regression Tests - Agglayer
 'on':
   workflow_dispatch:
     inputs:
diff --git a/.github/workflows/cdk-superuser-regression-tests.yml b/.github/workflows/cdk-superuser-regression-tests.yml
index 64e11bf7f1..1de2177830 100644
--- a/.github/workflows/cdk-superuser-regression-tests.yml
+++ b/.github/workflows/cdk-superuser-regression-tests.yml
@@ -1,4 +1,4 @@
-name: Regression Tests
+name: Regression Tests - Superuser
 'on':
   workflow_dispatch:
     inputs: