diff --git a/.github/workflows/ci_build_test_cpp_linux.yml b/.github/workflows/ci_build_test_cpp_linux.yml
index 67b7dc89c..6391a7a21 100644
--- a/.github/workflows/ci_build_test_cpp_linux.yml
+++ b/.github/workflows/ci_build_test_cpp_linux.yml
@@ -31,7 +31,7 @@ jobs:
       - name: "Setting up Python"
         uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # v2.3.3
         with:
-          python-version: "3.11"
+          python-version: "3.10"
 
       - name: Sync source deps
         run: |
@@ -59,3 +59,7 @@ jobs:
           # Some things put stuff in cache with weird, root read-only
           # permissions. Take them back.
           sudo chown -R "$(whoami)" "${cache_dir}"
+
+      - name: Run generated tests
+        run: |
+          bash build_tools/ci/run_paritybench_test.sh
diff --git a/build_tools/ci/run_paritybench_test.sh b/build_tools/ci/run_paritybench_test.sh
new file mode 100644
index 000000000..a87463e49
--- /dev/null
+++ b/build_tools/ci/run_paritybench_test.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+git clone https://github.com/jansel/pytorch-jit-paritybench.git
+
+cd pytorch-jit-paritybench
+
+git checkout 7e55a422588c1d1e00f35a3d3a3ff896cce59e18
+pip install -r requirements.txt
+pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+pip install expecttest
+
+cd ..
+
+pip install ./compiler
+pip install ./runtime
+pip install . --no-deps
+source build/iree/.env
+
+python python/test/generated/main.py --tests-dir ./pytorch-jit-paritybench --limit 100 -j 8
+
+