update benchmarks to work with splitting

.github/workflows/benchmark.yml

@@ -59,7 +59,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r devtools/dev-requirements.txt
 
-      - name: Benchmark with pytest-benchmark
+      - name: Benchmark with pytest-benchmark (PR)
         run: |
           source .venv-${{ matrix.python-version }}/bin/activate
           pwd
@@ -81,7 +81,8 @@ jobs:
 
       - name: Checkout benchmarks from PR head
         run: git checkout ${{ github.event.pull_request.head.sha }} -- tests/benchmarks
-      - name: Benchmark with pytest-benchmark
+
+      - name: Benchmark with pytest-benchmark (MASTER)
         run: |
           source .venv-${{ matrix.python-version }}/bin/activate
           pwd
@@ -95,7 +96,7 @@ jobs:
             --group ${{ matrix.group }} \
             --splitting-algorithm least_duration
 
-      - name: put benchmark results in same folder
+      - name: Put benchmark results in same folder
         run: |
           source .venv-${{ matrix.python-version }}/bin/activate
           pwd
@@ -108,15 +109,32 @@ jobs:
           cp $t1 compare_results
           cp $t2 compare_results
 
+      - name: Download artifact
+        if: always()
+        uses: actions/download-artifact@v4
+        with:
+          pattern: benchmark_artifact_*
+          path: tests/benchmarks
+
+      - name: Unzip artifacts if downloaded
+        run: |
+          cd tests/benchmarks
+          ls
+          if [ -f tests/benchmarks/benchmark_artifact_*.zip ]; then
+            unzip tests/benchmarks/benchmark_artifact_*.zip -d tests/benchmarks
+          else
+            echo "No benchmark artifact file found."
+          fi
+
       - name: Compare latest commit results to the master branch results
         run: |
           source .venv-${{ matrix.python-version }}/bin/activate
-          pwd
           cd tests/benchmarks
+          pwd
           python compare_bench_results.py
           cat commit_msg.txt
 
-      - name: comment PR with the results
+      - name: Comment PR with the results
         uses: thollander/actions-comment-pull-request@v2
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -128,5 +146,5 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: benchmark_artifact
+          name: benchmark_artifact_${{ matrix.group }}
           path: tests/benchmarks/.benchmarks

tests/benchmarks/compare_bench_results.py

@@ -8,60 +8,87 @@
 cwd = os.getcwd()
 
 data = {}
-master_idx = 0
-latest_idx = 0
+master_idx = []
+latest_idx = []
 commit_ind = 0
-for diret in os.walk(cwd + "/compare_results"):
-    files = diret[2]
-    timing_file_exists = False
-
-    for filename in files:
-        if filename.find("json") != -1:  # check if json output file is present
-            try:
-                filepath = os.path.join(diret[0], filename)
-                with open(filepath) as f:
-                    print(filepath)
-                    curr_data = json.load(f)
-                    commit_id = curr_data["commit_info"]["id"][0:7]
-                    data[commit_id] = curr_data
-                    if filepath.find("master") != -1:
-                        master_idx = commit_ind
-                    elif filepath.find("Latest_Commit") != -1:
-                        latest_idx = commit_ind
-                    commit_ind += 1
-            except Exception as e:
-                print(e)
-                continue
-
+folder_names = []
+
+for root1, dirs1, files1 in os.walk(cwd):
+    for dir_name in dirs1:
+        if dir_name == "compare_results" or dir_name.startswith("benchmark_artifact"):
+            print("Including folder: " + dir_name)
+            # "compare_results" is the folder containing the benchmark results from this
+            # job "benchmark_artifact" is the folder containing the benchmark results
+            # from other jobs if in future we change the Python version of the
+            # benchmarks, we will need to update this
+            # "/Linux-CPython-<PYTHON_VERSION>-64bit"
+            files2walk = (
+                os.walk(cwd + "/" + dir_name)
+                if dir_name == "compare_results"
+                else os.walk(cwd + "/" + dir_name + "/Linux-CPython-3.9-64bit")
+            )
+            for root, dirs, files in files2walk:
+                for filename in files:
+                    if (
+                        filename.find("json") != -1
+                    ):  # check if json output file is present
+                        try:
+                            filepath = os.path.join(root, filename)
+                            with open(filepath) as f:
+                                curr_data = json.load(f)
+                                commit_id = curr_data["commit_info"]["id"][0:7]
+                                data[commit_ind] = curr_data["benchmarks"]
+                                if filepath.find("master") != -1:
+                                    master_idx.append(commit_ind)
+                                elif filepath.find("Latest_Commit") != -1:
+                                    latest_idx.append(commit_ind)
+                                commit_ind += 1
+                        except Exception as e:
+                            print(e)
+                            continue
 
 # need arrays of size [ num benchmarks x num commits ]
 # one for mean one for stddev
 # number of benchmark cases
-num_benchmarks = len(data[list(data.keys())[0]]["benchmarks"])
-num_commits = len(list(data.keys()))
+num_benchmarks = 0
+# sum number of benchmarks splitted into different jobs
+for split in master_idx:
+    num_benchmarks += len(data[split])
+num_commits = 2
+
 times = np.zeros([num_benchmarks, num_commits])
 stddevs = np.zeros([num_benchmarks, num_commits])
 commit_ids = []
 test_names = [None] * num_benchmarks
 
-for id_num, commit_id in enumerate(data.keys()):
-    commit_ids.append(commit_id)
-    for i, test in enumerate(data[commit_id]["benchmarks"]):
+id_num = 0
+for i in master_idx:
+    for test in data[i]:
         t_mean = test["stats"]["median"]
         t_stddev = test["stats"]["iqr"]
-        times[i, id_num] = t_mean
-        stddevs[i, id_num] = t_stddev
-        test_names[i] = test["name"]
-
+        times[id_num, 0] = t_mean
+        stddevs[id_num, 0] = t_stddev
+        test_names[id_num] = test["name"]
+        id_num = id_num + 1
+
+id_num = 0
+for i in latest_idx:
+    for test in data[i]:
+        t_mean = test["stats"]["median"]
+        t_stddev = test["stats"]["iqr"]
+        times[id_num, 1] = t_mean
+        stddevs[id_num, 1] = t_stddev
+        test_names[id_num] = test["name"]
+        id_num = id_num + 1
 
 # we say a slowdown/speedup has occurred if the mean time difference is greater than
 # n_sigma * (stdev of time delta)
 significance = 3  # n_sigmas of normal distribution, ie z score of 3
 colors = [" "] * num_benchmarks  # g if faster, w if similar, r if slower
-delta_times_ms = times[:, latest_idx] - times[:, master_idx]
-delta_stds_ms = np.sqrt(stddevs[:, latest_idx] ** 2 + stddevs[:, master_idx] ** 2)
-delta_times_pct = delta_times_ms / times[:, master_idx] * 100
-delta_stds_pct = delta_stds_ms / times[:, master_idx] * 100
+delta_times_ms = times[:, 1] - times[:, 0]
+delta_stds_ms = np.sqrt(stddevs[:, 1] ** 2 + stddevs[:, 0] ** 2)
+delta_times_pct = delta_times_ms / times[:, 0] * 100
+delta_stds_pct = delta_stds_ms / times[:, 0] * 100
 for i, (pct, spct) in enumerate(zip(delta_times_pct, delta_stds_pct)):
     if pct > 0 and pct > significance * spct:
         colors[i] = "-"  # this will make the line red
@@ -72,8 +99,6 @@
 
 # now make the commit message, save as a txt file
 # benchmark_name dt(%) dt(s) t_new(s) t_old(s)
-print(latest_idx)
-print(master_idx)
 commit_msg_lines = [
     "```diff",
     f"| {'benchmark_name':^38} | {'dt(%)':^22} | {'dt(s)':^22} |"
@@ -88,8 +113,8 @@
     line = f"{colors[i]:>1}{test_names[i]:<39} |"
     line += f" {f'{dpct:+6.2f} +/- {sdpct:4.2f}':^22} |"
     line += f" {f'{dt:+.2e} +/- {sdt:.2e}':^22} |"
-    line += f" {f'{times[i, latest_idx]:.2e} +/- {stddevs[i, latest_idx]:.1e}':^22} |"
-    line += f" {f'{times[i, master_idx]:.2e} +/- {stddevs[i, master_idx]:.1e}':^22} |"
+    line += f" {f'{times[i, 1]:.2e} +/- {stddevs[i, 1]:.1e}':^22} |"
+    line += f" {f'{times[i, 0]:.2e} +/- {stddevs[i, 0]:.1e}':^22} |"
 
     commit_msg_lines.append(line)