diff --git a/.github/workflows/ci_build_test_cpp_linux.yml b/.github/workflows/ci_build_test_cpp_linux.yml
index 67b7dc89c..6391a7a21 100644
--- a/.github/workflows/ci_build_test_cpp_linux.yml
+++ b/.github/workflows/ci_build_test_cpp_linux.yml
@@ -31,7 +31,7 @@ jobs:
       - name: "Setting up Python"
         uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # v2.3.3
         with:
-          python-version: "3.11"
+          python-version: "3.10"
 
       - name: Sync source deps
         run: |
@@ -59,3 +59,7 @@ jobs:
           # Some things put stuff in cache with weird, root read-only
           # permissions. Take them back.
           sudo chown -R "$(whoami)" "${cache_dir}"
+
+      - name: Run generated tests
+        run: |
+          bash build_tools/ci/run_paritybench_test.sh
diff --git a/build_tools/ci/run_paritybench_test.sh b/build_tools/ci/run_paritybench_test.sh
new file mode 100644
index 000000000..7d4b404aa
--- /dev/null
+++ b/build_tools/ci/run_paritybench_test.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+git clone https://github.com/jansel/pytorch-jit-paritybench.git
+
+cd pytorch-jit-paritybench
+
+git checkout 7e55a422588c1d1e00f35a3d3a3ff896cce59e18
+pip install -r requirements.txt
+pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
+pip install expecttest
+
+cd ..
+
+pip install ./compiler
+pip install ./runtime
+pip install . --no-deps
+source build/iree/.env
+
+python python/test/generated/main.py --tests-dir ./pytorch-jit-paritybench --limit 100 -j 4 --no-log
+
+
diff --git a/python/test/generated/main.py b/python/test/generated/main.py
index 9ea3fd4c3..1c21ca159 100644
--- a/python/test/generated/main.py
+++ b/python/test/generated/main.py
@@ -10,7 +10,6 @@
 import logging
 
 log = logging.getLogger("turbine-test")
-logging.basicConfig(level=logging.INFO)
 
 ENV_FILE = "JITPARITYBENCH_PATH.txt"
 
@@ -41,6 +40,11 @@ def get_args(raw_args=None):
         help="jit-paritybench location (i.e. /path/to/pytorch-jit-paritybench)",
     )
     # parser.add_argument("--device", default="cuda", type=str, help="evaluate modules using cuda or cpu") # excluded for now as we only have turbine-cpu, can use this later
+    parser.add_argument(
+        "--no-log",
+        action='store_true',
+        help="disable logging during execution",
+    )
 
     args = parser.parse_args(raw_args)
     return args
@@ -60,6 +64,11 @@ def read_path() -> str:
 if __name__ == "__main__":
     args = get_args()
 
+    if args.no_log:
+        logging.basicConfig(level=logging.ERROR)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
     if args.tests_dir is not None:
         pb = args.tests_dir
         write_path(pb)  # store this path for next time