diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index ad00eeef5d..3655d67034 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -22,6 +22,12 @@ concurrency: jobs: benchmark: runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} + strategy: + matrix: + python-version: ['3.9'] + group: [1, 2] steps: # Enable tmate debugging of manually-triggered workflows if the input option was provided @@ -31,41 +37,68 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python 3.9 + + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.9 - - name: Install dependencies + python-version: ${{ matrix.python-version }} + + - name: Restore Python environment cache + id: restore-env + uses: actions/cache/restore@v4 + with: + path: .venv-${{ matrix.python-version }} + key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('devtools/dev-requirements.txt', 'requirements.txt') }} + + - name: Set up virtual environment if not restored from cache + if: steps.restore-env.outputs.cache-hit != 'true' run: | + gh cache list + python -m venv .venv-${{ matrix.python-version }} + source .venv-${{ matrix.python-version }}/bin/activate python -m pip install --upgrade pip pip install -r devtools/dev-requirements.txt - - name: Benchmark with pytest-benchmark + + - name: Benchmark with pytest-benchmark (PR) run: | + source .venv-${{ matrix.python-version }}/bin/activate pwd lscpu cd tests/benchmarks python -m pytest benchmark_cpu_small.py -vv \ --benchmark-save='Latest_Commit' \ --durations=0 \ - --benchmark-save-data + --benchmark-save-data \ + --splits 2 \ + --group ${{ matrix.group }} \ + --splitting-algorithm least_duration + - name: Checkout current master uses: actions/checkout@v4 with: ref: master clean: false + - name: Checkout benchmarks from PR head run: git checkout ${{ github.event.pull_request.head.sha }} -- tests/benchmarks - - name: Benchmark with pytest-benchmark + + - name: Benchmark with pytest-benchmark (MASTER) run: | + source .venv-${{ matrix.python-version }}/bin/activate pwd lscpu cd tests/benchmarks python -m pytest benchmark_cpu_small.py -vv \ --benchmark-save='master' \ --durations=0 \ - --benchmark-save-data - - name: put benchmark results in same folder + --benchmark-save-data \ + --splits 2 \ + --group ${{ matrix.group }} \ + --splitting-algorithm least_duration + + - name: Put benchmark results in same folder run: | + source .venv-${{ matrix.python-version }}/bin/activate pwd cd tests/benchmarks find .benchmarks/ -type f -printf "%T@ %p\n" | sort -n | cut -d' ' -f 2- | tail -n 1 > temp1 @@ -75,22 +108,43 @@ jobs: mkdir compare_results cp $t1 compare_results cp $t2 compare_results + + - name: Download artifact + if: always() + uses: actions/download-artifact@v4 + with: + pattern: benchmark_artifact_* + path: tests/benchmarks + + - name: Unzip artifacts if downloaded + run: | + cd tests/benchmarks + ls + if [ -f tests/benchmarks/benchmark_artifact_*.zip ]; then + unzip tests/benchmarks/benchmark_artifact_*.zip -d tests/benchmarks + else + echo "No benchmark artifact file found." + fi + - name: Compare latest commit results to the master branch results run: | - pwd + source .venv-${{ matrix.python-version }}/bin/activate cd tests/benchmarks + pwd python compare_bench_results.py cat commit_msg.txt - - name: comment PR with the results + + - name: Comment PR with the results uses: thollander/actions-comment-pull-request@v2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: filePath: tests/benchmarks/commit_msg.txt comment_tag: benchmark + - name: Upload benchmark data if: always() uses: actions/upload-artifact@v4 with: - name: benchmark_artifact + name: benchmark_artifact_${{ matrix.group }} path: tests/benchmarks/.benchmarks diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 993e56ee4b..ae0be3dbb7 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -6,26 +6,48 @@ jobs: black_format: runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} + strategy: + matrix: + python-version: ['3.10'] + steps: - uses: actions/checkout@v4 - - name: Set up Python 3.10 + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' - - name: Install dependencies + python-version: ${{ matrix.python-version }} + + - name: Restore Python environment cache + id: restore-env + uses: actions/cache/restore@v4 + with: + path: .venv-${{ matrix.python-version }} + key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('devtools/dev-requirements.txt', 'requirements.txt') }} + + - name: Set up virtual environment if not restored from cache + if: steps.restore-env.outputs.cache-hit != 'true' run: | + gh cache list + python -m venv .venv-${{ matrix.python-version }} + source .venv-${{ matrix.python-version }}/bin/activate python -m pip install --upgrade pip pip install -r devtools/dev-requirements.txt + - name: Check files using the black formatter run: | + source .venv-${{ matrix.python-version }}/bin/activate black --version black --check desc/ tests/ || black_return_code=$? echo "BLACK_RETURN_CODE=$black_return_code" >> $GITHUB_ENV black desc/ tests/ + - name: Annotate diff changes using reviewdog uses: reviewdog/action-suggester@v1 with: tool_name: blackfmt + - name: Fail if not formatted run: | exit ${{ env.BLACK_RETURN_CODE }} diff --git a/.github/workflows/cache_dependencies.yml b/.github/workflows/cache_dependencies.yml new file mode 100644 index 0000000000..d1f63538c5 --- /dev/null +++ b/.github/workflows/cache_dependencies.yml @@ -0,0 +1,55 @@ +name: Cache dependencies +# This workflow is triggered every 2 days and updates the Python +# and pip dependencies cache +on: + schedule: + - cron: '30 4 */2 * *' # This triggers the workflow at 4:30 AM every 2 days + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} + strategy: + matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Delete old cached file with same python version + run: | + echo "Current Cached files list" + gh cache list + echo "Deleting cached files with pattern: ${{ runner.os }}-venv-${{ matrix.python-version }}-" + for cache_key in $(gh cache list --json key -q ".[] | select(.key | startswith(\"${{ runner.os }}-venv-${{ matrix.python-version }}-\")) | .key"); do + echo "Deleting cache with key: $cache_key" + gh cache delete "$cache_key" + done + + - name: Set up virtual environment + run: | + python -m venv .venv-${{ matrix.python-version }} + source .venv-${{ matrix.python-version }}/bin/activate + python -m pip install --upgrade pip + pip install -r devtools/dev-requirements.txt + + - name: Cache Python environment + id: cache-env + uses: actions/cache@v4 + with: + path: .venv-${{ matrix.python-version }} + key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('devtools/dev-requirements.txt', 'requirements.txt') }} + + - name: Verify virtual environment activation + run: | + source .venv-${{ matrix.python-version }}/bin/activate + python --version + pip --version + pip list diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index c3ea0f96e9..2eef77dcc0 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -5,17 +5,24 @@ on: [pull_request, workflow_dispatch] jobs: flake8_linting: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10'] + name: Linting steps: - uses: actions/checkout@v4 - - name: Set up Python 3.10 + - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: ${{ matrix.python-version }} + + # For some reason, loading venv makes this way slower - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r devtools/dev-requirements.txt + - name: flake8 Lint uses: reviewdog/action-flake8@v3 with: diff --git a/.github/workflows/nbtests.yml b/.github/workflows/notebook_tests.yml similarity index 56% rename from .github/workflows/nbtests.yml rename to .github/workflows/notebook_tests.yml index 6d1fc6ca24..4dcbe54e14 100644 --- a/.github/workflows/nbtests.yml +++ b/.github/workflows/notebook_tests.yml @@ -18,28 +18,46 @@ jobs: notebook_tests: runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} strategy: matrix: python-version: ['3.10'] - group: [1, 2] + group: [1, 2, 3] steps: - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + + - name: Restore Python environment cache + id: restore-env + uses: actions/cache/restore@v4 + with: + path: .venv-${{ matrix.python-version }} + key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('devtools/dev-requirements.txt', 'requirements.txt') }} + + - name: Set up virtual environment if not restored from cache + if: steps.restore-env.outputs.cache-hit != 'true' run: | + gh cache list + python -m venv .venv-${{ matrix.python-version }} + source .venv-${{ matrix.python-version }}/bin/activate python -m pip install --upgrade pip pip install -r devtools/dev-requirements.txt + - name: Test notebooks with pytest and nbmake run: | + source .venv-${{ matrix.python-version }}/bin/activate pwd lscpu export PYTHONPATH=$(pwd) pytest -v --nbmake "./docs/notebooks" \ --nbmake-timeout=2000 \ --ignore=./docs/notebooks/zernike_eval.ipynb \ - --splits 2 \ + --splits 3 \ --group ${{ matrix.group }} \ + --splitting-algorithm least_duration diff --git a/.github/workflows/regression_test.yml b/.github/workflows/regression_tests.yml similarity index 72% rename from .github/workflows/regression_test.yml rename to .github/workflows/regression_tests.yml index d9ef1072d6..021d25a7a3 100644 --- a/.github/workflows/regression_test.yml +++ b/.github/workflows/regression_tests.yml @@ -18,6 +18,8 @@ jobs: regression_tests: runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} strategy: matrix: python-version: ['3.10'] @@ -29,20 +31,36 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + + - name: Restore Python environment cache + id: restore-env + uses: actions/cache/restore@v4 + with: + path: .venv-${{ matrix.python-version }} + key: ${{ runner.os }}-venv-${{ matrix.python-version }}-${{ hashFiles('devtools/dev-requirements.txt', 'requirements.txt') }} + + - name: Set up virtual environment if not restored from cache + if: steps.restore-env.outputs.cache-hit != 'true' run: | + gh cache list + python -m venv .venv-${{ matrix.python-version }} + source .venv-${{ matrix.python-version }}/bin/activate python -m pip install --upgrade pip pip install -r devtools/dev-requirements.txt pip install matplotlib==3.7.2 + - name: Set Swap Space uses: pierotofy/set-swap-space@master with: swap-size-gb: 10 + - name: Test with pytest run: | + source .venv-${{ matrix.python-version }}/bin/activate + pip install matplotlib==3.7.2 pwd lscpu - python -m pytest -v -m regression \ + python -m pytest -v -m regression\ --durations=0 \ --cov-report xml:cov.xml \ --cov-config=setup.cfg \ @@ -54,6 +72,7 @@ jobs: --group ${{ matrix.group }} \ --splitting-algorithm least_duration \ --db ./prof.db + - name: save coverage file and plot comparison results if: always() uses: actions/upload-artifact@v4 @@ -63,6 +82,7 @@ jobs: ./cov.xml ./mpl_results.html ./prof.db + - name: Upload coverage id : codecov uses: Wandalen/wretry.action@v1.3.0 diff --git a/.github/workflows/unittest.yml b/.github/workflows/unit_tests.yml similarity index 67% rename from .github/workflows/unittest.yml rename to .github/workflows/unit_tests.yml index 57e5881a05..bf95d25b28 100644 --- a/.github/workflows/unittest.yml +++ b/.github/workflows/unit_tests.yml @@ -18,12 +18,18 @@ jobs: unit_tests: runs-on: ubuntu-latest + env: + GH_TOKEN: ${{ github.token }} strategy: matrix: combos: [{group: 1, python_version: '3.9'}, {group: 2, python_version: '3.10'}, {group: 3, python_version: '3.11'}, - {group: 4, python_version: '3.12'}] + {group: 4, python_version: '3.12'}, + {group: 5, python_version: '3.12'}, + {group: 6, python_version: '3.12'}, + {group: 7, python_version: '3.12'}, + {group: 8, python_version: '3.12'}] steps: - uses: actions/checkout@v4 @@ -31,17 +37,33 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.combos.python_version }} - - name: Install dependencies + + - name: Restore Python environment cache + id: restore-env + uses: actions/cache/restore@v4 + with: + path: .venv-${{ matrix.combos.python_version }} + key: ${{ runner.os }}-venv-${{ matrix.combos.python_version }}-${{ hashFiles('devtools/dev-requirements.txt', 'requirements.txt') }} + + - name: Set up virtual environment if not restored from cache + if: steps.restore-env.outputs.cache-hit != 'true' run: | + gh cache list + python -m venv .venv-${{ matrix.combos.python_version }} + source .venv-${{ matrix.combos.python_version }}/bin/activate python -m pip install --upgrade pip pip install -r devtools/dev-requirements.txt pip install matplotlib==3.7.2 + - name: Set Swap Space uses: pierotofy/set-swap-space@master with: swap-size-gb: 10 + - name: Test with pytest run: | + source .venv-${{ matrix.combos.python_version }}/bin/activate + pip install matplotlib==3.7.2 pwd lscpu python -m pytest -v -m unit \ @@ -52,10 +74,11 @@ jobs: --mpl \ --mpl-results-path=mpl_results.html \ --mpl-generate-summary=html \ - --splits 4 \ + --splits 8 \ --group ${{ matrix.combos.group }} \ --splitting-algorithm least_duration \ --db ./prof.db + - name: save coverage file and plot comparison results if: always() uses: actions/upload-artifact@v4 @@ -65,6 +88,7 @@ jobs: ./cov.xml ./mpl_results.html ./prof.db + - name: Upload coverage id : codecov uses: Wandalen/wretry.action@v1.3.0 diff --git a/.github/workflows/scheduled.yml b/.github/workflows/weekly_tests.yml similarity index 75% rename from .github/workflows/scheduled.yml rename to .github/workflows/weekly_tests.yml index a584db5cb8..2fb309bd8a 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/weekly_tests.yml @@ -11,11 +11,10 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - combos: [{group: 1, python_version: '3.8'}, - {group: 2, python_version: '3.9'}, - {group: 3, python_version: '3.10'}, - {group: 4, python_version: '3.11'}, - {group: 5, python_version: '3.12'}] + combos: [{group: 1, python_version: '3.9'}, + {group: 2, python_version: '3.10'}, + {group: 3, python_version: '3.11'}, + {group: 4, python_version: '3.12'}] steps: - uses: actions/checkout@v4 @@ -37,6 +36,6 @@ jobs: lscpu python -m pytest -v -m unit \ --durations=0 \ - --splits 5 \ + --splits 4 \ --group ${{ matrix.combos.group }} \ --splitting-algorithm least_duration diff --git a/tests/benchmarks/compare_bench_results.py b/tests/benchmarks/compare_bench_results.py index 09fc580e22..ab56816153 100644 --- a/tests/benchmarks/compare_bench_results.py +++ b/tests/benchmarks/compare_bench_results.py @@ -8,60 +8,87 @@ cwd = os.getcwd() data = {} -master_idx = 0 -latest_idx = 0 +master_idx = [] +latest_idx = [] commit_ind = 0 -for diret in os.walk(cwd + "/compare_results"): - files = diret[2] - timing_file_exists = False - - for filename in files: - if filename.find("json") != -1: # check if json output file is present - try: - filepath = os.path.join(diret[0], filename) - with open(filepath) as f: - print(filepath) - curr_data = json.load(f) - commit_id = curr_data["commit_info"]["id"][0:7] - data[commit_id] = curr_data - if filepath.find("master") != -1: - master_idx = commit_ind - elif filepath.find("Latest_Commit") != -1: - latest_idx = commit_ind - commit_ind += 1 - except Exception as e: - print(e) - continue - +folder_names = [] + +for root1, dirs1, files1 in os.walk(cwd): + for dir_name in dirs1: + if dir_name == "compare_results" or dir_name.startswith("benchmark_artifact"): + print("Including folder: " + dir_name) + # "compare_results" is the folder containing the benchmark results from this + # job "benchmark_artifact" is the folder containing the benchmark results + # from other jobs if in future we change the Python version of the + # benchmarks, we will need to update this + # "/Linux-CPython--64bit" + files2walk = ( + os.walk(cwd + "/" + dir_name) + if dir_name == "compare_results" + else os.walk(cwd + "/" + dir_name + "/Linux-CPython-3.9-64bit") + ) + for root, dirs, files in files2walk: + for filename in files: + if ( + filename.find("json") != -1 + ): # check if json output file is present + try: + filepath = os.path.join(root, filename) + with open(filepath) as f: + curr_data = json.load(f) + commit_id = curr_data["commit_info"]["id"][0:7] + data[commit_ind] = curr_data["benchmarks"] + if filepath.find("master") != -1: + master_idx.append(commit_ind) + elif filepath.find("Latest_Commit") != -1: + latest_idx.append(commit_ind) + commit_ind += 1 + except Exception as e: + print(e) + continue # need arrays of size [ num benchmarks x num commits ] # one for mean one for stddev # number of benchmark cases -num_benchmarks = len(data[list(data.keys())[0]]["benchmarks"]) -num_commits = len(list(data.keys())) +num_benchmarks = 0 +# sum number of benchmarks splitted into different jobs +for split in master_idx: + num_benchmarks += len(data[split]) +num_commits = 2 + times = np.zeros([num_benchmarks, num_commits]) stddevs = np.zeros([num_benchmarks, num_commits]) commit_ids = [] test_names = [None] * num_benchmarks -for id_num, commit_id in enumerate(data.keys()): - commit_ids.append(commit_id) - for i, test in enumerate(data[commit_id]["benchmarks"]): +id_num = 0 +for i in master_idx: + for test in data[i]: t_mean = test["stats"]["median"] t_stddev = test["stats"]["iqr"] - times[i, id_num] = t_mean - stddevs[i, id_num] = t_stddev - test_names[i] = test["name"] - + times[id_num, 0] = t_mean + stddevs[id_num, 0] = t_stddev + test_names[id_num] = test["name"] + id_num = id_num + 1 + +id_num = 0 +for i in latest_idx: + for test in data[i]: + t_mean = test["stats"]["median"] + t_stddev = test["stats"]["iqr"] + times[id_num, 1] = t_mean + stddevs[id_num, 1] = t_stddev + test_names[id_num] = test["name"] + id_num = id_num + 1 # we say a slowdown/speedup has occurred if the mean time difference is greater than # n_sigma * (stdev of time delta) significance = 3 # n_sigmas of normal distribution, ie z score of 3 colors = [" "] * num_benchmarks # g if faster, w if similar, r if slower -delta_times_ms = times[:, latest_idx] - times[:, master_idx] -delta_stds_ms = np.sqrt(stddevs[:, latest_idx] ** 2 + stddevs[:, master_idx] ** 2) -delta_times_pct = delta_times_ms / times[:, master_idx] * 100 -delta_stds_pct = delta_stds_ms / times[:, master_idx] * 100 +delta_times_ms = times[:, 1] - times[:, 0] +delta_stds_ms = np.sqrt(stddevs[:, 1] ** 2 + stddevs[:, 0] ** 2) +delta_times_pct = delta_times_ms / times[:, 0] * 100 +delta_stds_pct = delta_stds_ms / times[:, 0] * 100 for i, (pct, spct) in enumerate(zip(delta_times_pct, delta_stds_pct)): if pct > 0 and pct > significance * spct: colors[i] = "-" # this will make the line red @@ -72,8 +99,6 @@ # now make the commit message, save as a txt file # benchmark_name dt(%) dt(s) t_new(s) t_old(s) -print(latest_idx) -print(master_idx) commit_msg_lines = [ "```diff", f"| {'benchmark_name':^38} | {'dt(%)':^22} | {'dt(s)':^22} |" @@ -88,8 +113,8 @@ line = f"{colors[i]:>1}{test_names[i]:<39} |" line += f" {f'{dpct:+6.2f} +/- {sdpct:4.2f}':^22} |" line += f" {f'{dt:+.2e} +/- {sdt:.2e}':^22} |" - line += f" {f'{times[i, latest_idx]:.2e} +/- {stddevs[i, latest_idx]:.1e}':^22} |" - line += f" {f'{times[i, master_idx]:.2e} +/- {stddevs[i, master_idx]:.1e}':^22} |" + line += f" {f'{times[i, 1]:.2e} +/- {stddevs[i, 1]:.1e}':^22} |" + line += f" {f'{times[i, 0]:.2e} +/- {stddevs[i, 0]:.1e}':^22} |" commit_msg_lines.append(line)