diff --git a/.github/workflows/bench_run.yaml b/.github/workflows/bench_run.yaml
index 4cee0a0f1..4b3c3f3ba 100644
--- a/.github/workflows/bench_run.yaml
+++ b/.github/workflows/bench_run.yaml
@@ -1,42 +1,34 @@
-name: Benchmarks
+name: MicroBenchmarks
 
 on:
+  push:
+    branches:
+      - "main"
   pull_request:
-    branches: [ main ]
-
-# Cancel already running jobs
-concurrency:
-  group: benchmark_run_${{ github.head_ref }}
-  cancel-in-progress: true
-
-env:
-  CARGO_TERM_COLOR: always
-  RUST_BACKTRACE: 1
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
 
 jobs:
   benchmarks:
-    runs-on: ubuntu-20.04
-    name: Benchmarks
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v4
-    - uses: Swatinem/rust-cache@v2
-      with:
-        # PRs do not share caches, instead each PR initially pulls from the cache of the main branch for the first run.
-        # This workflow does not run on main, so to make use of a cache before this workflow has completed once on a PR,
-        # we need to manually recreate the key used by ubuntu-20.04 release builds.
-        shared-key: "ubuntu-20.04 - --release-build_check_and_upload"
-        save-if: false
-    - name: cache custom ubuntu packages
-      uses: actions/cache@v4
-      with:
-        path: shotover-proxy/build/packages
-        key: ubuntu-20.04-packages
-    - name: Install ubuntu packages
-      run: shotover-proxy/build/install_ubuntu_packages.sh
-    - name: Run benchmarks
-      run: shotover-proxy/tests/scripts/bench_against_master.sh ${{ github.event.number }}
-    - name: Upload comment artifact
-      uses: actions/upload-artifact@v4
-      with:
-        name: comment_info
-        path: comment_info/
+      - uses: actions/checkout@v4
+      - uses: Swatinem/rust-cache@v2
+        with:
+          # this line means that only the main branch writes to the cache
+          # benefits:
+          # * prevents main branch caches from being evicted in favor of a PR cache
+          # * saves about 1min per workflow by skipping the actual cache write
+          # downsides:
+          # * PRs that update rust version or changes deps will be slow to iterate on due to changes not being cached.
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+      - uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-codspeed
+      - name: Build the benchmark target(s)
+        run: cargo codspeed build --features alpha-transforms
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@v2
+        with:
+          run: cargo codspeed run --features alpha-transforms
\ No newline at end of file
diff --git a/Cargo.lock b/Cargo.lock
index 16f23c525..86ca4664a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1126,12 +1126,46 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "codspeed"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a104ac948e0188b921eb3fcbdd55dcf62e542df4c7ab7e660623f6288302089"
+dependencies = [
+ "colored",
+ "libc",
+ "serde_json",
+]
+
+[[package]]
+name = "codspeed-criterion-compat"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "722c36bdc62d9436d027256ce2627af81ac7a596dfc7d13d849d0d212448d7fe"
+dependencies = [
+ "codspeed",
+ "colored",
+ "criterion",
+ "futures",
+ "tokio",
+]
+
 [[package]]
 name = "colorchoice"
 version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
 
+[[package]]
+name = "colored"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8"
+dependencies = [
+ "lazy_static",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "combine"
 version = "4.6.7"
@@ -4574,9 +4608,9 @@ dependencies = [
  "cassandra-protocol",
  "chacha20poly1305",
  "clap",
+ "codspeed-criterion-compat",
  "cql3-parser",
  "crc16",
- "criterion",
  "csv",
  "dashmap",
  "derivative",
diff --git a/shotover-proxy/tests/scripts/bench_against_master.sh b/shotover-proxy/tests/scripts/bench_against_master.sh
deleted file mode 100755
index 8671ac8e8..000000000
--- a/shotover-proxy/tests/scripts/bench_against_master.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-
-set -e; set -u
-set -o pipefail # Needed to make the cargo bench fail early on e.g. compiler error
-
-GITHUB_EVENT_NUMBER=$1
-LOG_PAGE=$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID
-
-# The extra complexity allows us to restore back to the original ref in more cases
-ORIGINAL_REF=`git rev-parse --abbrev-ref HEAD`
-if [ "$ORIGINAL_REF" == "HEAD" ]; then
-  ORIGINAL_REF=`git rev-parse HEAD`
-fi
-
-echo -e "\nBenchmarking main as a baseline"
-git fetch origin
-git checkout origin/$GITHUB_BASE_REF
-# Need to specify `--bench benches` becauses otherwise it tries to pass --save-baseline to something else (possibly a test) and blows up
-cargo bench --all-features --bench benches -- --save-baseline master --noplot
-
-echo -e "\nBenchmarking PR branch against main as a baseline"
-git checkout $ORIGINAL_REF
-cargo bench --all-features --bench benches -- --baseline-lenient master --noplot | tee benches_log.txt -a
-
-# grep returns non zero exit code when it doesnt find anything so we need to disable pipefail
-set +o pipefail
-COUNT_REGRESS=`grep -o "Performance has regressed." benches_log.txt | wc -l`
-COUNT_IMPROVE=`grep -o "Performance has improved." benches_log.txt | wc -l`
-set -o pipefail
-
-mkdir -p comment_info
-if [[ "$COUNT_REGRESS" != "0" || "$COUNT_IMPROVE" != "0" ]]; then
-  echo "$COUNT_REGRESS benchmark regressed. $COUNT_IMPROVE benchmark improved. Please check the benchmark workflow logs for full details: $LOG_PAGE" > comment_info/message.txt
-  echo "$GITHUB_EVENT_NUMBER" > ./comment_info/issue_number.txt
-
-  # grep returns non zero exit code when it doesnt find anything so we need to disable -e
-  set +e
-
-  if [[ "$COUNT_REGRESS" != "0" ]]; then
-    echo "\`\`\`" >> comment_info/message.txt
-    # Kind of brittle but -B5 includes the 5 lines prior which always happens to includes the bench name + results
-    grep -B6 "Performance has regressed." benches_log.txt >> comment_info/message.txt
-    echo "\`\`\`" >> comment_info/message.txt
-  fi
-
-  if [[ "$COUNT_IMPROVE" != "0" ]]; then
-    echo "\`\`\`" >> comment_info/message.txt
-    grep -B6 "Performance has improved." benches_log.txt >> comment_info/message.txt
-    echo "\`\`\`" >> comment_info/message.txt
-  fi
-
-  set -e
-fi
diff --git a/shotover-proxy/tests/scripts/bench_against_master_local.sh b/shotover-proxy/tests/scripts/bench_against_master_local.sh
deleted file mode 100755
index 6e7b9a106..000000000
--- a/shotover-proxy/tests/scripts/bench_against_master_local.sh
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env bash
-
-# Frontend script for bench_against_master.sh to run it on your local machine
-
-echo "Assuming that origin is the upstream remote"
-
-set -e; set -u
-
-SCRIPT_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-
-export LOG_PAGE="log is above"
-export GITHUB_SERVER_URL=""
-export GITHUB_REPOSITORY=""
-export GITHUB_RUN_ID=""
-export GITHUB_BASE_REF="main"
-
-$SCRIPT_DIR/bench_against_master.sh 1
diff --git a/shotover/Cargo.toml b/shotover/Cargo.toml
index 3be72d920..8d51c58d2 100644
--- a/shotover/Cargo.toml
+++ b/shotover/Cargo.toml
@@ -121,7 +121,7 @@ atoi = { version = "2.0.0", optional = true }
 fnv = "1.0.7"
 
 [dev-dependencies]
-criterion = { version = "0.5.0", features = ["async_tokio"] }
+criterion = { version = "2.6.0", features = ["async_tokio"], package = "codspeed-criterion-compat" }
 hex-literal.workspace = true
 
 # TODO: Optionally compiling benches is quite tricky with criterion, maybe it would be easier with divan?