From 4c3d9931fce034fae70f59891a9391583ed81c9e Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 14:06:13 -0700 Subject: [PATCH 01/46] Add conda-env-tests workflow --- .github/workflows/conda-env-tests.yml | 95 +++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 .github/workflows/conda-env-tests.yml diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml new file mode 100644 index 0000000..32923f0 --- /dev/null +++ b/.github/workflows/conda-env-tests.yml @@ -0,0 +1,95 @@ +name: Unit Tests in NSLS-II Conda Environments + +on: [push, pull_request, workflow_dispatch] + +jobs: + test-in-conda-env: + + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: ["3.9", "3.10"] + + fail-fast: false + + steps: + + - name: Set env vars + shell: bash -l {0} + run: | + export REPOSITORY_NAME=${GITHUB_REPOSITORY#*/} # just the repo, as opposed to org/repo + echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV + + export ZENODO_ID=7651500 # 2023-1.2 with Python 3.9 and Python 3.10 + echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV + + if [ "${{ matrix.python-version }}" == "3.9" ]; then + export CONDA_ENV_NAME='2023-1.2-py39-tiled' + echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV + fi + + - name: Check out the code repo + uses: actions/checkout@v4 + + # - name: Set up Python ${{ matrix.python-version }} + # uses: actions/setup-python@v4 + # with: + # python-version: ${{ matrix.python-version }} + + - name: Set up Python ${{ matrix.python-version }} with conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }} + auto-update-conda: true + miniconda-version: "latest" + python-version: ${{ matrix.python-version }} + mamba-version: "*" + channels: conda-forge + + - name: Download and install the archived conda environment + shell: bash -l {0} + run: | + set -vxeuo pipefail + source $HOME/miniconda/etc/profile.d/conda.sh # this makes all conda vars available + url="https://zenodo.org/record/${ZENODO_ID}/files/${CONDA_ENV_NAME}.tar.gz?download=1" + wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz + status=$? + if [ $status -gt 0 ]; then + echo "Cannot download from ${url}. Exit code: ${status}" + exit $status + fi + echo "${MD5_CHECKSUM} ${CONDA_ENV_NAME}.tar.gz" > checksum.txt + md5sum --check checksum.txt + mkdir -p $HOME/miniconda/envs/${CONDA_ENV_NAME} + tar -xf ${CONDA_ENV_NAME}.tar.gz -C $HOME/miniconda/envs/${CONDA_ENV_NAME} + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + conda unpack + + conda info -a + conda env list + conda list + pip list + + - name: Install the package and its dependencies + shell: bash -l {0} + run: | + set -vxeuo pipefail + pip install --upgrade pip setuptools wheel numpy + pip install -r requirements-dev.txt + pip install -r requirements-extras.txt + pip install -e . + pip list + + - name: Test with pytest + shell: bash -l {0} + run: | + set -vxeuo pipefail + coverage run --source=csxtools run_tests.py + coverage xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + file: ./coverage.xml + flags: conda-env-unittests From d0dd2209d6663086bd64e008206a476d566f36d7 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 14:10:28 -0700 Subject: [PATCH 02/46] Activate incubator conda environment --- .github/workflows/conda-env-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 32923f0..6015edc 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -51,7 +51,8 @@ jobs: shell: bash -l {0} run: | set -vxeuo pipefail - source $HOME/miniconda/etc/profile.d/conda.sh # this makes all conda vars available + # source $HOME/miniconda/etc/profile.d/conda.sh # this makes all conda vars available + conda activate "${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }}" url="https://zenodo.org/record/${ZENODO_ID}/files/${CONDA_ENV_NAME}.tar.gz?download=1" wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz status=$? From b08833e007a06b137ff5fa870151758005f575b9 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 14:18:37 -0700 Subject: [PATCH 03/46] Include archived conda env checksums --- .github/workflows/conda-env-tests.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 6015edc..2652eb8 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -27,16 +27,22 @@ jobs: if [ "${{ matrix.python-version }}" == "3.9" ]; then export CONDA_ENV_NAME='2023-1.2-py39-tiled' echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV + + export MD5_CHECKSUM='31586643a0cdd9aee8349ccb93ccac87' + echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV + + elif [ "${{ matrix.python-version }}" == "3.10" ]; then + export CONDA_ENV_NAME='2023-1.2-py310-tiled' + echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV + + export MD5_CHECKSUM='ff8c0c475a007434c0011f5d8976d853' + echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV + fi - name: Check out the code repo uses: actions/checkout@v4 - # - name: Set up Python ${{ matrix.python-version }} - # uses: actions/setup-python@v4 - # with: - # python-version: ${{ matrix.python-version }} - - name: Set up Python ${{ matrix.python-version }} with conda uses: conda-incubator/setup-miniconda@v3 with: From 90410abf0cebd577abf2b186f5bdb1a12379a519 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 14:42:42 -0700 Subject: [PATCH 04/46] Cache the conda environment archive --- .github/workflows/conda-env-tests.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 2652eb8..05b38fc 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -53,7 +53,21 @@ jobs: mamba-version: "*" channels: conda-forge + - name: Cache the archived conda environment file + uses: actions/cache@v3 + with: + path: ${CONDA_ENV_NAME}.tar.gz + key: + ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}--${{ runner.arch }}-${{ + env.MD5_CHECKSUM }}-${{ + env.CACHE_NUMBER }} + env: + # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed + CACHE_NUMBER: 0 + id: cache-conda-archive + - name: Download and install the archived conda environment + if: steps.cache-conda-archive.outputs.cache-hit != 'true' shell: bash -l {0} run: | set -vxeuo pipefail @@ -70,6 +84,11 @@ jobs: md5sum --check checksum.txt mkdir -p $HOME/miniconda/envs/${CONDA_ENV_NAME} tar -xf ${CONDA_ENV_NAME}.tar.gz -C $HOME/miniconda/envs/${CONDA_ENV_NAME} + + - name: Activate and inspect the archived conda environment + shell: bash -l {0} + run: | + set -vxeuo pipefail conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} conda unpack From a0a24aafe05d7b467f0c8fb51e00710f7e37b127 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 14:49:20 -0700 Subject: [PATCH 05/46] Record the conda environment used for installation and test --- .github/workflows/conda-env-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 05b38fc..92fefdc 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -71,7 +71,6 @@ jobs: shell: bash -l {0} run: | set -vxeuo pipefail - # source $HOME/miniconda/etc/profile.d/conda.sh # this makes all conda vars available conda activate "${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }}" url="https://zenodo.org/record/${ZENODO_ID}/files/${CONDA_ENV_NAME}.tar.gz?download=1" wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz @@ -101,6 +100,7 @@ jobs: shell: bash -l {0} run: | set -vxeuo pipefail + echo "Using conda environment at ${CONDA_PREFIX}" pip install --upgrade pip setuptools wheel numpy pip install -r requirements-dev.txt pip install -r requirements-extras.txt @@ -111,6 +111,7 @@ jobs: shell: bash -l {0} run: | set -vxeuo pipefail + echo "Using conda environment at ${CONDA_PREFIX}" coverage run --source=csxtools run_tests.py coverage xml From d1c0cb2b1ed79306473bb17a2bd2731d1b115886 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 14:53:14 -0700 Subject: [PATCH 06/46] Fix cache path for archived conda environment file --- .github/workflows/conda-env-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 92fefdc..86c897a 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -56,7 +56,7 @@ jobs: - name: Cache the archived conda environment file uses: actions/cache@v3 with: - path: ${CONDA_ENV_NAME}.tar.gz + path: ${{ env.CONDA_ENV_NAME }}.tar.gz key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}--${{ runner.arch }}-${{ env.MD5_CHECKSUM }}-${{ From 771d2b8494171b484b3d3ab244ecf1847b8048c0 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 15:01:40 -0700 Subject: [PATCH 07/46] Activate the NSLS-II conda environment for installation and test --- .github/workflows/conda-env-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 86c897a..4341be3 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -100,6 +100,7 @@ jobs: shell: bash -l {0} run: | set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} echo "Using conda environment at ${CONDA_PREFIX}" pip install --upgrade pip setuptools wheel numpy pip install -r requirements-dev.txt @@ -111,6 +112,7 @@ jobs: shell: bash -l {0} run: | set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} echo "Using conda environment at ${CONDA_PREFIX}" coverage run --source=csxtools run_tests.py coverage xml From c7d3e31f68a38cdea15a0e887456b7ca644f9c7c Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 15:13:49 -0700 Subject: [PATCH 08/46] Cache the archive conda env --- .github/workflows/conda-env-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 4341be3..be834f1 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -53,10 +53,10 @@ jobs: mamba-version: "*" channels: conda-forge - - name: Cache the archived conda environment file + - name: Cache the archived conda environment uses: actions/cache@v3 with: - path: ${{ env.CONDA_ENV_NAME }}.tar.gz + path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} key: ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}--${{ runner.arch }}-${{ env.MD5_CHECKSUM }}-${{ From 9eb6a0099e860b67203da8f2e11865907055d0f5 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 15:27:23 -0700 Subject: [PATCH 09/46] Do not upgrade numpy for conda env tests --- .github/workflows/conda-env-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index be834f1..ed7dca3 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -102,7 +102,8 @@ jobs: set -vxeuo pipefail conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} echo "Using conda environment at ${CONDA_PREFIX}" - pip install --upgrade pip setuptools wheel numpy + # pip install --upgrade pip setuptools wheel numpy + pip install --upgrade pip setuptools wheel pip install -r requirements-dev.txt pip install -r requirements-extras.txt pip install -e . From 800c801ab262c73de17bfd73e1a984c6a2c9a83a Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 15:51:08 -0700 Subject: [PATCH 10/46] Move conda env details to GHA matrix strategy --- .github/workflows/conda-env-tests.yml | 33 +++++++++++++-------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index ed7dca3..9078668 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -9,7 +9,16 @@ jobs: strategy: matrix: - python-version: ["3.9", "3.10"] + include: + - conda_env_name: '2023-1.2-py39-tiled' + zenodo_id: '7651500' + md5_checksum: '31586643a0cdd9aee8349ccb93ccac87' + python-version: '3.9' + + - conda_env_name: '2023-1.2-py310-tiled' + zenodo_id: '7651500' + md5_checksum: 'ff8c0c475a007434c0011f5d8976d853' + python-version: '3.10' fail-fast: false @@ -21,24 +30,14 @@ jobs: export REPOSITORY_NAME=${GITHUB_REPOSITORY#*/} # just the repo, as opposed to org/repo echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV - export ZENODO_ID=7651500 # 2023-1.2 with Python 3.9 and Python 3.10 - echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV - - if [ "${{ matrix.python-version }}" == "3.9" ]; then - export CONDA_ENV_NAME='2023-1.2-py39-tiled' - echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV - - export MD5_CHECKSUM='31586643a0cdd9aee8349ccb93ccac87' - echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV + export CONDA_ENV_NAME=${{ matrix.conda_env_name }} + echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV - elif [ "${{ matrix.python-version }}" == "3.10" ]; then - export CONDA_ENV_NAME='2023-1.2-py310-tiled' - echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV - - export MD5_CHECKSUM='ff8c0c475a007434c0011f5d8976d853' - echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV + export ZENODO_ID=${{ matrix.zenodo_id }} + echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV - fi + export MD5_CHECKSUM='${{ matrix.md5_checksum }} + echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV - name: Check out the code repo uses: actions/checkout@v4 From e2fad9e6d3e4c63932b48482e2bd76036a2ff4f9 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 15:53:31 -0700 Subject: [PATCH 11/46] Fix quotes in GHA env vars --- .github/workflows/conda-env-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 9078668..f128435 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -27,16 +27,16 @@ jobs: - name: Set env vars shell: bash -l {0} run: | - export REPOSITORY_NAME=${GITHUB_REPOSITORY#*/} # just the repo, as opposed to org/repo + export REPOSITORY_NAME="${GITHUB_REPOSITORY#*/}" # just the repo, as opposed to org/repo echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV - export CONDA_ENV_NAME=${{ matrix.conda_env_name }} + export CONDA_ENV_NAME='${{ matrix.conda_env_name }}' echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV - export ZENODO_ID=${{ matrix.zenodo_id }} + export ZENODO_ID='${{ matrix.zenodo_id }}' echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV - export MD5_CHECKSUM='${{ matrix.md5_checksum }} + export MD5_CHECKSUM='${{ matrix.md5_checksum }}' echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV - name: Check out the code repo From 240573e5a4b1cd49c67e7554ba10d8308729df26 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 15:57:26 -0700 Subject: [PATCH 12/46] Tests with non-tiled conda envs --- .github/workflows/conda-env-tests.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index f128435..83386c5 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -10,16 +10,26 @@ jobs: strategy: matrix: include: - - conda_env_name: '2023-1.2-py39-tiled' + - conda_env_name: '2023-1.2-py39' zenodo_id: '7651500' - md5_checksum: '31586643a0cdd9aee8349ccb93ccac87' + md5_checksum: '14d0629f468df311918cfea2e6413bfa' python-version: '3.9' - - conda_env_name: '2023-1.2-py310-tiled' + - conda_env_name: '2023-1.2-py310' zenodo_id: '7651500' - md5_checksum: 'ff8c0c475a007434c0011f5d8976d853' + md5_checksum: '2dc85c4d2392f751c76ed7d01f363814' python-version: '3.10' + # - conda_env_name: '2023-1.2-py39-tiled' + # zenodo_id: '7651500' + # md5_checksum: '31586643a0cdd9aee8349ccb93ccac87' + # python-version: '3.9' + + # - conda_env_name: '2023-1.2-py310-tiled' + # zenodo_id: '7651500' + # md5_checksum: 'ff8c0c475a007434c0011f5d8976d853' + # python-version: '3.10' + fail-fast: false steps: From b0b99406f500d7fe7965aaa4d12d5e2f30c3a836 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 16:20:15 -0700 Subject: [PATCH 13/46] Add all 2023 and 2024 conda envs --- .github/workflows/conda-env-tests.yml | 183 ++++++++++++++++++++++++-- 1 file changed, 175 insertions(+), 8 deletions(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 83386c5..59573bf 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -10,25 +10,192 @@ jobs: strategy: matrix: include: + # 2023-1.0 + - conda_env_name: '2023-1.0-py39' + zenodo_id: '7470912' + md5_checksum: '651be0e26cdb230dd8c809b4eed046e9' + python-version: '3.9' + + - conda_env_name: '2023-1.0-py39-tiled' + zenodo_id: '7470912' + md5_checksum: 'c092b113d19510ac9ecb68eda66d3e79' + python-version: '3.9' + + - conda_env_name: '2023-1.0-py310' + zenodo_id: '7470912' + md5_checksum: '2c93e892a55d8c684acea4ccb2680ac5' + python-version: '3.10' + + - conda_env_name: '2023-1.0-py310-tiled' + zenodo_id: '7470912' + md5_checksum: '1de3a83f0ead58d3cb60d721478748a2' + python-version: '3.10' + + # 2023-1.1 + - conda_env_name: '2023-1.1-py39' + zenodo_id: '7470912' + md5_checksum: '531a712033d907b781d24030182357f4' + python-version: '3.9' + + - conda_env_name: '2023-1.1-py39-tiled' + zenodo_id: '7470912' + md5_checksum: '31dad55d53436605c47dbfc510e876a2' + python-version: '3.9' + + - conda_env_name: '2023-1.1-py310' + zenodo_id: '7470912' + md5_checksum: 'd60ceb752a798590334cbec3d0155fb6' + python-version: '3.10' + + - conda_env_name: '2023-1.1-py310-tiled' + zenodo_id: '7470912' + md5_checksum: 'f501314939d5237edb58244b0fcc6857' + python-version: '3.10' + + # 2023-1.2 - conda_env_name: '2023-1.2-py39' zenodo_id: '7651500' md5_checksum: '14d0629f468df311918cfea2e6413bfa' python-version: '3.9' + - conda_env_name: '2023-1.2-py39-tiled' + zenodo_id: '7651500' + md5_checksum: '31586643a0cdd9aee8349ccb93ccac87' + python-version: '3.9' + - conda_env_name: '2023-1.2-py310' zenodo_id: '7651500' md5_checksum: '2dc85c4d2392f751c76ed7d01f363814' python-version: '3.10' - # - conda_env_name: '2023-1.2-py39-tiled' - # zenodo_id: '7651500' - # md5_checksum: '31586643a0cdd9aee8349ccb93ccac87' - # python-version: '3.9' + # 2023-1.3 + - conda_env_name: '2023-1.3-py39' + zenodo_id: '7791542' + md5_checksum: '9cc18f3e09dfa30c4e8f165a932ca394' + python-version: '3.9' + + - conda_env_name: '2023-1.3-py39-tiled' + zenodo_id: '7791542' + md5_checksum: '1f40f5786fc0c06baad3fd3b456a6abc' + python-version: '3.9' + + - conda_env_name: '2023-1.3-py310' + zenodo_id: '7791542' + md5_checksum: '3304bc674d14b2820d60b311b58661fd' + python-version: '3.10' + + - conda_env_name: '2023-1.3-py310-tiled' + zenodo_id: '7791542' + md5_checksum: 'b7869b9fdb1c076bfba1f7eb2c63c6f0' + python-version: '3.10' + + # 2023-2.0 + - conda_env_name: '2023-2.0-py310' + zenodo_id: '7876026' + md5_checksum: '2f81225554a16339e2a49432cd8a013b' + python-version: '3.10' + + - conda_env_name: '2023-2.0-py310-tiled' + zenodo_id: '7876026' + md5_checksum: 'ef85af9ff76bd0afa527bdfc391b0442' + python-version: '3.10' + + # 2023-2.1 + - conda_env_name: '2023-2.1-py310' + zenodo_id: '8098505' + md5_checksum: '31add8231d47d92821b5f4178cd9a58a' + python-version: '3.10' + + - conda_env_name: '2023-2.1-py310-tiled' + zenodo_id: '8098505' + md5_checksum: 'd75299b2a6ffc2ed5aca7d01150c97f0' + python-version: '3.10' + + # 2023-3.0 + - conda_env_name: '2023-3.0-py310' + zenodo_id: '8342731' + md5_checksum: '49ad184de893ab1a27db3c4d3e91a045' + python-version: '3.10' + + - conda_env_name: '2023-3.0-py310-tiled' + zenodo_id: '8342731' + md5_checksum: 'b727a21d791192bc3981fbe56deb44bd' + python-version: '3.10' + + # 2023-3.1 + - conda_env_name: '2023-3.1-py310' + zenodo_id: '8342775' + md5_checksum: '06c16140f909b11ac6cb712c96c849af' + python-version: '3.10' + + - conda_env_name: '2023-3.1-py310-tiled' + zenodo_id: '8342775' + md5_checksum: '1551a5591bfde30ddbb34d0798e4c94c' + python-version: '3.10' + + # 2023-3.2 + - conda_env_name: '2023-3.2-py310' + zenodo_id: '10056340' + md5_checksum: '71d187467850f10289ce0d8691aa80fd' + python-version: '3.10' + + - conda_env_name: '2023-3.2-py310-tiled' + zenodo_id: '10056340' + md5_checksum: '8c3740791930b2f230e64b0f9399b7b8' + python-version: '3.10' + + # 2023-3.3 + - conda_env_name: '2023-3.3-py310' + zenodo_id: '10148425' + md5_checksum: 'bb77ea6f7a5bb63980d5181f7666ecc2' + python-version: '3.10' + + - conda_env_name: '2023-3.3-py310-tiled' + zenodo_id: '10148425' + md5_checksum: '0a47934380db013b36f3e089afdff6aa' + python-version: '3.10' + + - conda_env_name: '2023-3.3-py311' + zenodo_id: '10148425' + md5_checksum: 'dd6c06a02e43b286cdf84908d8339992' + python-version: '3.11' + + - conda_env_name: '2023-3.3-py311-tiled' + zenodo_id: '10148425' + md5_checksum: '48d2a73ea743d34152400cd642fd7472' + python-version: '3.11' + + # 2024-1.0 + - conda_env_name: '2024-1.0-py310' + zenodo_id: '10548109' + md5_checksum: 'b6107d565f0ff9d7315eab96d28894e9' + python-version: '3.10' + + - conda_env_name: '2024-1.0-py310-tiled' + zenodo_id: '10548109' + md5_checksum: '216c4df93c6893a7ebe6456b678ac89e' + python-version: '3.10' + + - conda_env_name: '2024-1.0-py311' + zenodo_id: '10548109' + md5_checksum: '2592ed2002823c423b4a664183d384da' + python-version: '3.11' + + - conda_env_name: '2024-1.0-py311-tiled' + zenodo_id: '10548109' + md5_checksum: 'a51aac76916f19f5982650c1c6f21503' + python-version: '3.11' + + # 2024-2.0 + - conda_env_name: '2024-2.0-py310-tiled' + zenodo_id: '11122851' + md5_checksum: '91955234f1617df158e7301f05a89b64' + python-version: '3.10' - # - conda_env_name: '2023-1.2-py310-tiled' - # zenodo_id: '7651500' - # md5_checksum: 'ff8c0c475a007434c0011f5d8976d853' - # python-version: '3.10' + - conda_env_name: '2024-2.0-py311-tiled' + zenodo_id: '11122851' + md5_checksum: 'd30ca6db3d93de99655a85a481c966c2' + python-version: '3.11' fail-fast: false From 69df7be288d759e0dbffc0b158924b565af21921 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 16:21:57 -0700 Subject: [PATCH 14/46] Limit number of concurrent GHA jobs --- .github/workflows/conda-env-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 59573bf..82ce6d1 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -8,6 +8,7 @@ jobs: runs-on: ubuntu-latest strategy: + max-parallel: 4 matrix: include: # 2023-1.0 From 2c677b537995e697f78b991894dbca080ffd1ce5 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 16:39:32 -0700 Subject: [PATCH 15/46] Fix zenodo id for 2023-1.1 --- .github/workflows/conda-env-tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 82ce6d1..745a1b2 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -34,22 +34,22 @@ jobs: # 2023-1.1 - conda_env_name: '2023-1.1-py39' - zenodo_id: '7470912' + zenodo_id: '7603440' md5_checksum: '531a712033d907b781d24030182357f4' python-version: '3.9' - conda_env_name: '2023-1.1-py39-tiled' - zenodo_id: '7470912' + zenodo_id: '7603440' md5_checksum: '31dad55d53436605c47dbfc510e876a2' python-version: '3.9' - conda_env_name: '2023-1.1-py310' - zenodo_id: '7470912' + zenodo_id: '7603440' md5_checksum: 'd60ceb752a798590334cbec3d0155fb6' python-version: '3.10' - conda_env_name: '2023-1.1-py310-tiled' - zenodo_id: '7470912' + zenodo_id: '7603440' md5_checksum: 'f501314939d5237edb58244b0fcc6857' python-version: '3.10' From ba97b04c2fbd59f506d1f8c0bc9e755509971533 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 17:10:14 -0700 Subject: [PATCH 16/46] Workaround for unset GDAL_DATA variable --- .github/workflows/conda-env-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 745a1b2..f1358fd 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -217,6 +217,9 @@ jobs: export MD5_CHECKSUM='${{ matrix.md5_checksum }}' echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV + # Workaround for unset GDAL_DATA env variable used by newer conda envs + export GDAL_DATA="${GDAL_DATA:-}" + - name: Check out the code repo uses: actions/checkout@v4 From 97f0e4cd8bfdcf51d9fc95e5dd34659f856c3974 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 17:33:54 -0700 Subject: [PATCH 17/46] Workaround #2 for unset GDAL_DATA variable --- .github/workflows/conda-env-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index f1358fd..4a147e6 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -218,7 +218,8 @@ jobs: echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV # Workaround for unset GDAL_DATA env variable used by newer conda envs - export GDAL_DATA="${GDAL_DATA:-}" + export GDAL_DATA="${GDAL_DATA-''}" + echo "GDAL_DATA=${GDAL_DATA}" >> $GITHUB_ENV - name: Check out the code repo uses: actions/checkout@v4 From db625abba786562840a6b37b9cf86f362ca413c0 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 17:51:27 -0700 Subject: [PATCH 18/46] Workaround for unset GDAL_DRIVER_PATH variable --- .github/workflows/conda-env-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 4a147e6..2505a1e 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -217,9 +217,11 @@ jobs: export MD5_CHECKSUM='${{ matrix.md5_checksum }}' echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV - # Workaround for unset GDAL_DATA env variable used by newer conda envs + # Workaround for unset GDAL env variables used by newer conda envs export GDAL_DATA="${GDAL_DATA-''}" echo "GDAL_DATA=${GDAL_DATA}" >> $GITHUB_ENV + export GDAL_DRIVER_PATH="${GDAL_DRIVER_PATH-''}" + echo "GDAL_DRIVER_PATH=${GDAL_DRIVER_PATH}" >> $GITHUB_ENV - name: Check out the code repo uses: actions/checkout@v4 From f5101eff466576a1e22dd39d66aa06ac18b01f7a Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 17:51:48 -0700 Subject: [PATCH 19/46] Skip passing tests for now --- .github/workflows/conda-env-tests.yml | 242 +++++++++++++------------- 1 file changed, 121 insertions(+), 121 deletions(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 2505a1e..bb01ae8 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -12,127 +12,127 @@ jobs: matrix: include: # 2023-1.0 - - conda_env_name: '2023-1.0-py39' - zenodo_id: '7470912' - md5_checksum: '651be0e26cdb230dd8c809b4eed046e9' - python-version: '3.9' - - - conda_env_name: '2023-1.0-py39-tiled' - zenodo_id: '7470912' - md5_checksum: 'c092b113d19510ac9ecb68eda66d3e79' - python-version: '3.9' - - - conda_env_name: '2023-1.0-py310' - zenodo_id: '7470912' - md5_checksum: '2c93e892a55d8c684acea4ccb2680ac5' - python-version: '3.10' - - - conda_env_name: '2023-1.0-py310-tiled' - zenodo_id: '7470912' - md5_checksum: '1de3a83f0ead58d3cb60d721478748a2' - python-version: '3.10' - - # 2023-1.1 - - conda_env_name: '2023-1.1-py39' - zenodo_id: '7603440' - md5_checksum: '531a712033d907b781d24030182357f4' - python-version: '3.9' - - - conda_env_name: '2023-1.1-py39-tiled' - zenodo_id: '7603440' - md5_checksum: '31dad55d53436605c47dbfc510e876a2' - python-version: '3.9' - - - conda_env_name: '2023-1.1-py310' - zenodo_id: '7603440' - md5_checksum: 'd60ceb752a798590334cbec3d0155fb6' - python-version: '3.10' - - - conda_env_name: '2023-1.1-py310-tiled' - zenodo_id: '7603440' - md5_checksum: 'f501314939d5237edb58244b0fcc6857' - python-version: '3.10' - - # 2023-1.2 - - conda_env_name: '2023-1.2-py39' - zenodo_id: '7651500' - md5_checksum: '14d0629f468df311918cfea2e6413bfa' - python-version: '3.9' - - - conda_env_name: '2023-1.2-py39-tiled' - zenodo_id: '7651500' - md5_checksum: '31586643a0cdd9aee8349ccb93ccac87' - python-version: '3.9' - - - conda_env_name: '2023-1.2-py310' - zenodo_id: '7651500' - md5_checksum: '2dc85c4d2392f751c76ed7d01f363814' - python-version: '3.10' - - # 2023-1.3 - - conda_env_name: '2023-1.3-py39' - zenodo_id: '7791542' - md5_checksum: '9cc18f3e09dfa30c4e8f165a932ca394' - python-version: '3.9' - - - conda_env_name: '2023-1.3-py39-tiled' - zenodo_id: '7791542' - md5_checksum: '1f40f5786fc0c06baad3fd3b456a6abc' - python-version: '3.9' - - - conda_env_name: '2023-1.3-py310' - zenodo_id: '7791542' - md5_checksum: '3304bc674d14b2820d60b311b58661fd' - python-version: '3.10' - - - conda_env_name: '2023-1.3-py310-tiled' - zenodo_id: '7791542' - md5_checksum: 'b7869b9fdb1c076bfba1f7eb2c63c6f0' - python-version: '3.10' - - # 2023-2.0 - - conda_env_name: '2023-2.0-py310' - zenodo_id: '7876026' - md5_checksum: '2f81225554a16339e2a49432cd8a013b' - python-version: '3.10' - - - conda_env_name: '2023-2.0-py310-tiled' - zenodo_id: '7876026' - md5_checksum: 'ef85af9ff76bd0afa527bdfc391b0442' - python-version: '3.10' - - # 2023-2.1 - - conda_env_name: '2023-2.1-py310' - zenodo_id: '8098505' - md5_checksum: '31add8231d47d92821b5f4178cd9a58a' - python-version: '3.10' - - - conda_env_name: '2023-2.1-py310-tiled' - zenodo_id: '8098505' - md5_checksum: 'd75299b2a6ffc2ed5aca7d01150c97f0' - python-version: '3.10' - - # 2023-3.0 - - conda_env_name: '2023-3.0-py310' - zenodo_id: '8342731' - md5_checksum: '49ad184de893ab1a27db3c4d3e91a045' - python-version: '3.10' - - - conda_env_name: '2023-3.0-py310-tiled' - zenodo_id: '8342731' - md5_checksum: 'b727a21d791192bc3981fbe56deb44bd' - python-version: '3.10' - - # 2023-3.1 - - conda_env_name: '2023-3.1-py310' - zenodo_id: '8342775' - md5_checksum: '06c16140f909b11ac6cb712c96c849af' - python-version: '3.10' - - - conda_env_name: '2023-3.1-py310-tiled' - zenodo_id: '8342775' - md5_checksum: '1551a5591bfde30ddbb34d0798e4c94c' - python-version: '3.10' + # - conda_env_name: '2023-1.0-py39' + # zenodo_id: '7470912' + # md5_checksum: '651be0e26cdb230dd8c809b4eed046e9' + # python-version: '3.9' + + # - conda_env_name: '2023-1.0-py39-tiled' + # zenodo_id: '7470912' + # md5_checksum: 'c092b113d19510ac9ecb68eda66d3e79' + # python-version: '3.9' + + # - conda_env_name: '2023-1.0-py310' + # zenodo_id: '7470912' + # md5_checksum: '2c93e892a55d8c684acea4ccb2680ac5' + # python-version: '3.10' + + # - conda_env_name: '2023-1.0-py310-tiled' + # zenodo_id: '7470912' + # md5_checksum: '1de3a83f0ead58d3cb60d721478748a2' + # python-version: '3.10' + + # # 2023-1.1 + # - conda_env_name: '2023-1.1-py39' + # zenodo_id: '7603440' + # md5_checksum: '531a712033d907b781d24030182357f4' + # python-version: '3.9' + + # - conda_env_name: '2023-1.1-py39-tiled' + # zenodo_id: '7603440' + # md5_checksum: '31dad55d53436605c47dbfc510e876a2' + # python-version: '3.9' + + # - conda_env_name: '2023-1.1-py310' + # zenodo_id: '7603440' + # md5_checksum: 'd60ceb752a798590334cbec3d0155fb6' + # python-version: '3.10' + + # - conda_env_name: '2023-1.1-py310-tiled' + # zenodo_id: '7603440' + # md5_checksum: 'f501314939d5237edb58244b0fcc6857' + # python-version: '3.10' + + # # 2023-1.2 + # - conda_env_name: '2023-1.2-py39' + # zenodo_id: '7651500' + # md5_checksum: '14d0629f468df311918cfea2e6413bfa' + # python-version: '3.9' + + # - conda_env_name: '2023-1.2-py39-tiled' + # zenodo_id: '7651500' + # md5_checksum: '31586643a0cdd9aee8349ccb93ccac87' + # python-version: '3.9' + + # - conda_env_name: '2023-1.2-py310' + # zenodo_id: '7651500' + # md5_checksum: '2dc85c4d2392f751c76ed7d01f363814' + # python-version: '3.10' + + # # 2023-1.3 + # - conda_env_name: '2023-1.3-py39' + # zenodo_id: '7791542' + # md5_checksum: '9cc18f3e09dfa30c4e8f165a932ca394' + # python-version: '3.9' + + # - conda_env_name: '2023-1.3-py39-tiled' + # zenodo_id: '7791542' + # md5_checksum: '1f40f5786fc0c06baad3fd3b456a6abc' + # python-version: '3.9' + + # - conda_env_name: '2023-1.3-py310' + # zenodo_id: '7791542' + # md5_checksum: '3304bc674d14b2820d60b311b58661fd' + # python-version: '3.10' + + # - conda_env_name: '2023-1.3-py310-tiled' + # zenodo_id: '7791542' + # md5_checksum: 'b7869b9fdb1c076bfba1f7eb2c63c6f0' + # python-version: '3.10' + + # # 2023-2.0 + # - conda_env_name: '2023-2.0-py310' + # zenodo_id: '7876026' + # md5_checksum: '2f81225554a16339e2a49432cd8a013b' + # python-version: '3.10' + + # - conda_env_name: '2023-2.0-py310-tiled' + # zenodo_id: '7876026' + # md5_checksum: 'ef85af9ff76bd0afa527bdfc391b0442' + # python-version: '3.10' + + # # 2023-2.1 + # - conda_env_name: '2023-2.1-py310' + # zenodo_id: '8098505' + # md5_checksum: '31add8231d47d92821b5f4178cd9a58a' + # python-version: '3.10' + + # - conda_env_name: '2023-2.1-py310-tiled' + # zenodo_id: '8098505' + # md5_checksum: 'd75299b2a6ffc2ed5aca7d01150c97f0' + # python-version: '3.10' + + # # 2023-3.0 + # - conda_env_name: '2023-3.0-py310' + # zenodo_id: '8342731' + # md5_checksum: '49ad184de893ab1a27db3c4d3e91a045' + # python-version: '3.10' + + # - conda_env_name: '2023-3.0-py310-tiled' + # zenodo_id: '8342731' + # md5_checksum: 'b727a21d791192bc3981fbe56deb44bd' + # python-version: '3.10' + + # # 2023-3.1 + # - conda_env_name: '2023-3.1-py310' + # zenodo_id: '8342775' + # md5_checksum: '06c16140f909b11ac6cb712c96c849af' + # python-version: '3.10' + + # - conda_env_name: '2023-3.1-py310-tiled' + # zenodo_id: '8342775' + # md5_checksum: '1551a5591bfde30ddbb34d0798e4c94c' + # python-version: '3.10' # 2023-3.2 - conda_env_name: '2023-3.2-py310' From ac1aedd37b64157c446a6de991d508da800fa018 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 17:56:33 -0700 Subject: [PATCH 20/46] Workaround for unset GEOTIFF_CSV variable --- .github/workflows/conda-env-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index bb01ae8..0539417 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -222,6 +222,8 @@ jobs: echo "GDAL_DATA=${GDAL_DATA}" >> $GITHUB_ENV export GDAL_DRIVER_PATH="${GDAL_DRIVER_PATH-''}" echo "GDAL_DRIVER_PATH=${GDAL_DRIVER_PATH}" >> $GITHUB_ENV + export GEOTIFF_CSV="${GEOTIFF_CSV-''}" + echo "GEOTIFF_CSV=${GEOTIFF_CSV}" >> $GITHUB_ENV - name: Check out the code repo uses: actions/checkout@v4 From 9f53424fb96692d06cbede577db4188bfc22a351 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 18:03:54 -0700 Subject: [PATCH 21/46] Workaround for unset MKL_INTERFACE_LAYER variable --- .github/workflows/conda-env-tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 0539417..df76b48 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -225,6 +225,10 @@ jobs: export GEOTIFF_CSV="${GEOTIFF_CSV-''}" echo "GEOTIFF_CSV=${GEOTIFF_CSV}" >> $GITHUB_ENV + # Workaround for unset MKL env variables used by newer conda envs + export MKL_INTERFACE_LAYER="${MKL_INTERFACE_LAYER-''}" + echo "MKL_INTERFACE_LAYER=${MKL_INTERFACE_LAYER}" >> $GITHUB_ENV + - name: Check out the code repo uses: actions/checkout@v4 From 47cb4d16813755545ae9c7e653465076beb7d915 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 18:27:13 -0700 Subject: [PATCH 22/46] Tidy up conda-envs-test config --- .github/workflows/conda-env-tests.yml | 252 +++++++++++++------------- 1 file changed, 125 insertions(+), 127 deletions(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index df76b48..274bc77 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -7,132 +7,136 @@ jobs: runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + strategy: max-parallel: 4 matrix: include: # 2023-1.0 - # - conda_env_name: '2023-1.0-py39' - # zenodo_id: '7470912' - # md5_checksum: '651be0e26cdb230dd8c809b4eed046e9' - # python-version: '3.9' - - # - conda_env_name: '2023-1.0-py39-tiled' - # zenodo_id: '7470912' - # md5_checksum: 'c092b113d19510ac9ecb68eda66d3e79' - # python-version: '3.9' - - # - conda_env_name: '2023-1.0-py310' - # zenodo_id: '7470912' - # md5_checksum: '2c93e892a55d8c684acea4ccb2680ac5' - # python-version: '3.10' - - # - conda_env_name: '2023-1.0-py310-tiled' - # zenodo_id: '7470912' - # md5_checksum: '1de3a83f0ead58d3cb60d721478748a2' - # python-version: '3.10' - - # # 2023-1.1 - # - conda_env_name: '2023-1.1-py39' - # zenodo_id: '7603440' - # md5_checksum: '531a712033d907b781d24030182357f4' - # python-version: '3.9' - - # - conda_env_name: '2023-1.1-py39-tiled' - # zenodo_id: '7603440' - # md5_checksum: '31dad55d53436605c47dbfc510e876a2' - # python-version: '3.9' - - # - conda_env_name: '2023-1.1-py310' - # zenodo_id: '7603440' - # md5_checksum: 'd60ceb752a798590334cbec3d0155fb6' - # python-version: '3.10' - - # - conda_env_name: '2023-1.1-py310-tiled' - # zenodo_id: '7603440' - # md5_checksum: 'f501314939d5237edb58244b0fcc6857' - # python-version: '3.10' - - # # 2023-1.2 - # - conda_env_name: '2023-1.2-py39' - # zenodo_id: '7651500' - # md5_checksum: '14d0629f468df311918cfea2e6413bfa' - # python-version: '3.9' - - # - conda_env_name: '2023-1.2-py39-tiled' - # zenodo_id: '7651500' - # md5_checksum: '31586643a0cdd9aee8349ccb93ccac87' - # python-version: '3.9' - - # - conda_env_name: '2023-1.2-py310' - # zenodo_id: '7651500' - # md5_checksum: '2dc85c4d2392f751c76ed7d01f363814' - # python-version: '3.10' - - # # 2023-1.3 - # - conda_env_name: '2023-1.3-py39' - # zenodo_id: '7791542' - # md5_checksum: '9cc18f3e09dfa30c4e8f165a932ca394' - # python-version: '3.9' - - # - conda_env_name: '2023-1.3-py39-tiled' - # zenodo_id: '7791542' - # md5_checksum: '1f40f5786fc0c06baad3fd3b456a6abc' - # python-version: '3.9' - - # - conda_env_name: '2023-1.3-py310' - # zenodo_id: '7791542' - # md5_checksum: '3304bc674d14b2820d60b311b58661fd' - # python-version: '3.10' - - # - conda_env_name: '2023-1.3-py310-tiled' - # zenodo_id: '7791542' - # md5_checksum: 'b7869b9fdb1c076bfba1f7eb2c63c6f0' - # python-version: '3.10' - - # # 2023-2.0 - # - conda_env_name: '2023-2.0-py310' - # zenodo_id: '7876026' - # md5_checksum: '2f81225554a16339e2a49432cd8a013b' - # python-version: '3.10' - - # - conda_env_name: '2023-2.0-py310-tiled' - # zenodo_id: '7876026' - # md5_checksum: 'ef85af9ff76bd0afa527bdfc391b0442' - # python-version: '3.10' - - # # 2023-2.1 - # - conda_env_name: '2023-2.1-py310' - # zenodo_id: '8098505' - # md5_checksum: '31add8231d47d92821b5f4178cd9a58a' - # python-version: '3.10' - - # - conda_env_name: '2023-2.1-py310-tiled' - # zenodo_id: '8098505' - # md5_checksum: 'd75299b2a6ffc2ed5aca7d01150c97f0' - # python-version: '3.10' - - # # 2023-3.0 - # - conda_env_name: '2023-3.0-py310' - # zenodo_id: '8342731' - # md5_checksum: '49ad184de893ab1a27db3c4d3e91a045' - # python-version: '3.10' - - # - conda_env_name: '2023-3.0-py310-tiled' - # zenodo_id: '8342731' - # md5_checksum: 'b727a21d791192bc3981fbe56deb44bd' - # python-version: '3.10' - - # # 2023-3.1 - # - conda_env_name: '2023-3.1-py310' - # zenodo_id: '8342775' - # md5_checksum: '06c16140f909b11ac6cb712c96c849af' - # python-version: '3.10' - - # - conda_env_name: '2023-3.1-py310-tiled' - # zenodo_id: '8342775' - # md5_checksum: '1551a5591bfde30ddbb34d0798e4c94c' - # python-version: '3.10' + - conda_env_name: '2023-1.0-py39' + zenodo_id: '7470912' + md5_checksum: '651be0e26cdb230dd8c809b4eed046e9' + python-version: '3.9' + + - conda_env_name: '2023-1.0-py39-tiled' + zenodo_id: '7470912' + md5_checksum: 'c092b113d19510ac9ecb68eda66d3e79' + python-version: '3.9' + + - conda_env_name: '2023-1.0-py310' + zenodo_id: '7470912' + md5_checksum: '2c93e892a55d8c684acea4ccb2680ac5' + python-version: '3.10' + + - conda_env_name: '2023-1.0-py310-tiled' + zenodo_id: '7470912' + md5_checksum: '1de3a83f0ead58d3cb60d721478748a2' + python-version: '3.10' + + # 2023-1.1 + - conda_env_name: '2023-1.1-py39' + zenodo_id: '7603440' + md5_checksum: '531a712033d907b781d24030182357f4' + python-version: '3.9' + + - conda_env_name: '2023-1.1-py39-tiled' + zenodo_id: '7603440' + md5_checksum: '31dad55d53436605c47dbfc510e876a2' + python-version: '3.9' + + - conda_env_name: '2023-1.1-py310' + zenodo_id: '7603440' + md5_checksum: 'd60ceb752a798590334cbec3d0155fb6' + python-version: '3.10' + + - conda_env_name: '2023-1.1-py310-tiled' + zenodo_id: '7603440' + md5_checksum: 'f501314939d5237edb58244b0fcc6857' + python-version: '3.10' + + # 2023-1.2 + - conda_env_name: '2023-1.2-py39' + zenodo_id: '7651500' + md5_checksum: '14d0629f468df311918cfea2e6413bfa' + python-version: '3.9' + + - conda_env_name: '2023-1.2-py39-tiled' + zenodo_id: '7651500' + md5_checksum: '31586643a0cdd9aee8349ccb93ccac87' + python-version: '3.9' + + - conda_env_name: '2023-1.2-py310' + zenodo_id: '7651500' + md5_checksum: '2dc85c4d2392f751c76ed7d01f363814' + python-version: '3.10' + + # 2023-1.3 + - conda_env_name: '2023-1.3-py39' + zenodo_id: '7791542' + md5_checksum: '9cc18f3e09dfa30c4e8f165a932ca394' + python-version: '3.9' + + - conda_env_name: '2023-1.3-py39-tiled' + zenodo_id: '7791542' + md5_checksum: '1f40f5786fc0c06baad3fd3b456a6abc' + python-version: '3.9' + + - conda_env_name: '2023-1.3-py310' + zenodo_id: '7791542' + md5_checksum: '3304bc674d14b2820d60b311b58661fd' + python-version: '3.10' + + - conda_env_name: '2023-1.3-py310-tiled' + zenodo_id: '7791542' + md5_checksum: 'b7869b9fdb1c076bfba1f7eb2c63c6f0' + python-version: '3.10' + + # 2023-2.0 + - conda_env_name: '2023-2.0-py310' + zenodo_id: '7876026' + md5_checksum: '2f81225554a16339e2a49432cd8a013b' + python-version: '3.10' + + - conda_env_name: '2023-2.0-py310-tiled' + zenodo_id: '7876026' + md5_checksum: 'ef85af9ff76bd0afa527bdfc391b0442' + python-version: '3.10' + + # 2023-2.1 + - conda_env_name: '2023-2.1-py310' + zenodo_id: '8098505' + md5_checksum: '31add8231d47d92821b5f4178cd9a58a' + python-version: '3.10' + + - conda_env_name: '2023-2.1-py310-tiled' + zenodo_id: '8098505' + md5_checksum: 'd75299b2a6ffc2ed5aca7d01150c97f0' + python-version: '3.10' + + # 2023-3.0 + - conda_env_name: '2023-3.0-py310' + zenodo_id: '8342731' + md5_checksum: '49ad184de893ab1a27db3c4d3e91a045' + python-version: '3.10' + + - conda_env_name: '2023-3.0-py310-tiled' + zenodo_id: '8342731' + md5_checksum: 'b727a21d791192bc3981fbe56deb44bd' + python-version: '3.10' + + # 2023-3.1 + - conda_env_name: '2023-3.1-py310' + zenodo_id: '8342775' + md5_checksum: '06c16140f909b11ac6cb712c96c849af' + python-version: '3.10' + + - conda_env_name: '2023-3.1-py310-tiled' + zenodo_id: '8342775' + md5_checksum: '1551a5591bfde30ddbb34d0798e4c94c' + python-version: '3.10' # 2023-3.2 - conda_env_name: '2023-3.2-py310' @@ -203,7 +207,6 @@ jobs: steps: - name: Set env vars - shell: bash -l {0} run: | export REPOSITORY_NAME="${GITHUB_REPOSITORY#*/}" # just the repo, as opposed to org/repo echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV @@ -257,7 +260,6 @@ jobs: - name: Download and install the archived conda environment if: steps.cache-conda-archive.outputs.cache-hit != 'true' - shell: bash -l {0} run: | set -vxeuo pipefail conda activate "${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }}" @@ -274,7 +276,6 @@ jobs: tar -xf ${CONDA_ENV_NAME}.tar.gz -C $HOME/miniconda/envs/${CONDA_ENV_NAME} - name: Activate and inspect the archived conda environment - shell: bash -l {0} run: | set -vxeuo pipefail conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} @@ -286,12 +287,10 @@ jobs: pip list - name: Install the package and its dependencies - shell: bash -l {0} run: | set -vxeuo pipefail conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} echo "Using conda environment at ${CONDA_PREFIX}" - # pip install --upgrade pip setuptools wheel numpy pip install --upgrade pip setuptools wheel pip install -r requirements-dev.txt pip install -r requirements-extras.txt @@ -299,7 +298,6 @@ jobs: pip list - name: Test with pytest - shell: bash -l {0} run: | set -vxeuo pipefail conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} From 73c10db4f2ccccd5b06e8bfb9f74524a370d1265 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Tue, 21 May 2024 18:33:38 -0700 Subject: [PATCH 23/46] Add missing conda env 2023-1.2-py310-tiled --- .github/workflows/conda-env-tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 274bc77..e0f94e0 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -73,6 +73,11 @@ jobs: md5_checksum: '2dc85c4d2392f751c76ed7d01f363814' python-version: '3.10' + - conda_env_name: '2023-1.2-py310-tiled' + zenodo_id: '7651500' + md5_checksum: 'ff8c0c475a007434c0011f5d8976d853' + python-version: '3.10' + # 2023-1.3 - conda_env_name: '2023-1.3-py39' zenodo_id: '7791542' From 1cc9f9b37dc2b48b56cd491bca8c9e49ef1917d5 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 06:11:37 -0700 Subject: [PATCH 24/46] Build conda-env-tests matrix from YAML input --- .github/conda-env-tests/legacy-envs.yml | 140 ++++++++++++++++ .github/conda-env-tests/recent-envs.yml | 54 ++++++ .github/workflows/conda-env-tests.yml | 210 ++---------------------- 3 files changed, 212 insertions(+), 192 deletions(-) create mode 100644 .github/conda-env-tests/legacy-envs.yml create mode 100644 .github/conda-env-tests/recent-envs.yml diff --git a/.github/conda-env-tests/legacy-envs.yml b/.github/conda-env-tests/legacy-envs.yml new file mode 100644 index 0000000..f86d6c8 --- /dev/null +++ b/.github/conda-env-tests/legacy-envs.yml @@ -0,0 +1,140 @@ +--- + +# 2023-1.0 +- conda_env_name: '2023-1.0-py39' + zenodo_id: '7470912' + md5_checksum: '651be0e26cdb230dd8c809b4eed046e9' + python-version: '3.9' + +- conda_env_name: '2023-1.0-py39-tiled' + zenodo_id: '7470912' + md5_checksum: 'c092b113d19510ac9ecb68eda66d3e79' + python-version: '3.9' + +- conda_env_name: '2023-1.0-py310' + zenodo_id: '7470912' + md5_checksum: '2c93e892a55d8c684acea4ccb2680ac5' + python-version: '3.10' + +- conda_env_name: '2023-1.0-py310-tiled' + zenodo_id: '7470912' + md5_checksum: '1de3a83f0ead58d3cb60d721478748a2' + python-version: '3.10' + +# 2023-1.1 +- conda_env_name: '2023-1.1-py39' + zenodo_id: '7603440' + md5_checksum: '531a712033d907b781d24030182357f4' + python-version: '3.9' + +- conda_env_name: '2023-1.1-py39-tiled' + zenodo_id: '7603440' + md5_checksum: '31dad55d53436605c47dbfc510e876a2' + python-version: '3.9' + +- conda_env_name: '2023-1.1-py310' + zenodo_id: '7603440' + md5_checksum: 'd60ceb752a798590334cbec3d0155fb6' + python-version: '3.10' + +- conda_env_name: '2023-1.1-py310-tiled' + zenodo_id: '7603440' + md5_checksum: 'f501314939d5237edb58244b0fcc6857' + python-version: '3.10' + +# 2023-1.2 +- conda_env_name: '2023-1.2-py39' + zenodo_id: '7651500' + md5_checksum: '14d0629f468df311918cfea2e6413bfa' + python-version: '3.9' + +- conda_env_name: '2023-1.2-py39-tiled' + zenodo_id: '7651500' + md5_checksum: '31586643a0cdd9aee8349ccb93ccac87' + python-version: '3.9' + +- conda_env_name: '2023-1.2-py310' + zenodo_id: '7651500' + md5_checksum: '2dc85c4d2392f751c76ed7d01f363814' + python-version: '3.10' + +- conda_env_name: '2023-1.2-py310-tiled' + zenodo_id: '7651500' + md5_checksum: 'ff8c0c475a007434c0011f5d8976d853' + python-version: '3.10' + +# 2023-1.3 +- conda_env_name: '2023-1.3-py39' + zenodo_id: '7791542' + md5_checksum: '9cc18f3e09dfa30c4e8f165a932ca394' + python-version: '3.9' + +- conda_env_name: '2023-1.3-py39-tiled' + zenodo_id: '7791542' + md5_checksum: '1f40f5786fc0c06baad3fd3b456a6abc' + python-version: '3.9' + +- conda_env_name: '2023-1.3-py310' + zenodo_id: '7791542' + md5_checksum: '3304bc674d14b2820d60b311b58661fd' + python-version: '3.10' + +- conda_env_name: '2023-1.3-py310-tiled' + zenodo_id: '7791542' + md5_checksum: 'b7869b9fdb1c076bfba1f7eb2c63c6f0' + python-version: '3.10' + +# 2023-2.0 +- conda_env_name: '2023-2.0-py310' + zenodo_id: '7876026' + md5_checksum: '2f81225554a16339e2a49432cd8a013b' + python-version: '3.10' + +- conda_env_name: '2023-2.0-py310-tiled' + zenodo_id: '7876026' + md5_checksum: 'ef85af9ff76bd0afa527bdfc391b0442' + python-version: '3.10' + +# 2023-2.1 +- conda_env_name: '2023-2.1-py310' + zenodo_id: '8098505' + md5_checksum: '31add8231d47d92821b5f4178cd9a58a' + python-version: '3.10' + +- conda_env_name: '2023-2.1-py310-tiled' + zenodo_id: '8098505' + md5_checksum: 'd75299b2a6ffc2ed5aca7d01150c97f0' + python-version: '3.10' + +# 2023-3.0 +- conda_env_name: '2023-3.0-py310' + zenodo_id: '8342731' + md5_checksum: '49ad184de893ab1a27db3c4d3e91a045' + python-version: '3.10' + +- conda_env_name: '2023-3.0-py310-tiled' + zenodo_id: '8342731' + md5_checksum: 'b727a21d791192bc3981fbe56deb44bd' + python-version: '3.10' + +# 2023-3.1 +- conda_env_name: '2023-3.1-py310' + zenodo_id: '8342775' + md5_checksum: '06c16140f909b11ac6cb712c96c849af' + python-version: '3.10' + +- conda_env_name: '2023-3.1-py310-tiled' + zenodo_id: '8342775' + md5_checksum: '1551a5591bfde30ddbb34d0798e4c94c' + python-version: '3.10' + +# 2023-3.2 +- conda_env_name: '2023-3.2-py310' + zenodo_id: '10056340' + md5_checksum: '71d187467850f10289ce0d8691aa80fd' + python-version: '3.10' + +- conda_env_name: '2023-3.2-py310-tiled' + zenodo_id: '10056340' + md5_checksum: '8c3740791930b2f230e64b0f9399b7b8' + python-version: '3.10' diff --git a/.github/conda-env-tests/recent-envs.yml b/.github/conda-env-tests/recent-envs.yml new file mode 100644 index 0000000..412029f --- /dev/null +++ b/.github/conda-env-tests/recent-envs.yml @@ -0,0 +1,54 @@ +--- + +# 2023-3.3 +- conda_env_name: '2023-3.3-py310' + zenodo_id: '10148425' + md5_checksum: 'bb77ea6f7a5bb63980d5181f7666ecc2' + python-version: '3.10' + +- conda_env_name: '2023-3.3-py310-tiled' + zenodo_id: '10148425' + md5_checksum: '0a47934380db013b36f3e089afdff6aa' + python-version: '3.10' + +- conda_env_name: '2023-3.3-py311' + zenodo_id: '10148425' + md5_checksum: 'dd6c06a02e43b286cdf84908d8339992' + python-version: '3.11' + +- conda_env_name: '2023-3.3-py311-tiled' + zenodo_id: '10148425' + md5_checksum: '48d2a73ea743d34152400cd642fd7472' + python-version: '3.11' + +# 2024-1.0 +- conda_env_name: '2024-1.0-py310' + zenodo_id: '10548109' + md5_checksum: 'b6107d565f0ff9d7315eab96d28894e9' + python-version: '3.10' + +- conda_env_name: '2024-1.0-py310-tiled' + zenodo_id: '10548109' + md5_checksum: '216c4df93c6893a7ebe6456b678ac89e' + python-version: '3.10' + +- conda_env_name: '2024-1.0-py311' + zenodo_id: '10548109' + md5_checksum: '2592ed2002823c423b4a664183d384da' + python-version: '3.11' + +- conda_env_name: '2024-1.0-py311-tiled' + zenodo_id: '10548109' + md5_checksum: 'a51aac76916f19f5982650c1c6f21503' + python-version: '3.11' + +# 2024-2.0 +- conda_env_name: '2024-2.0-py310-tiled' + zenodo_id: '11122851' + md5_checksum: '91955234f1617df158e7301f05a89b64' + python-version: '3.10' + +- conda_env_name: '2024-2.0-py311-tiled' + zenodo_id: '11122851' + md5_checksum: 'd30ca6db3d93de99655a85a481c966c2' + python-version: '3.11' diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index e0f94e0..b1078ce 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -3,6 +3,23 @@ name: Unit Tests in NSLS-II Conda Environments on: [push, pull_request, workflow_dispatch] jobs: + + matrix_prep: + + runs-on: ubuntu-latest + outputs: + matrix_include: ${{ steps.set-matrix.outputs.matrix_include }} + steps: + - name: Check out the code repo + uses: actions/checkout@v4 + - id: set-matrix + run: | + set -vxeuo pipefail + export MATRIX_INCLUDE_DIR=${{ github.workspace }}/.github/conda-env-tests + export MATRIX_INCLUDE_FILE=${MATRIX_INCLUDE_DIR}/recent-envs.yml + echo "Building matrix from file ${MATRIX_INCLUDE_FILE}" + echo "matrix_include=$(cat ${MATRIX_INCLUDE_FILE})" >> $GITHUB_OUTPUT + test-in-conda-env: runs-on: ubuntu-latest @@ -14,198 +31,7 @@ jobs: strategy: max-parallel: 4 matrix: - include: - # 2023-1.0 - - conda_env_name: '2023-1.0-py39' - zenodo_id: '7470912' - md5_checksum: '651be0e26cdb230dd8c809b4eed046e9' - python-version: '3.9' - - - conda_env_name: '2023-1.0-py39-tiled' - zenodo_id: '7470912' - md5_checksum: 'c092b113d19510ac9ecb68eda66d3e79' - python-version: '3.9' - - - conda_env_name: '2023-1.0-py310' - zenodo_id: '7470912' - md5_checksum: '2c93e892a55d8c684acea4ccb2680ac5' - python-version: '3.10' - - - conda_env_name: '2023-1.0-py310-tiled' - zenodo_id: '7470912' - md5_checksum: '1de3a83f0ead58d3cb60d721478748a2' - python-version: '3.10' - - # 2023-1.1 - - conda_env_name: '2023-1.1-py39' - zenodo_id: '7603440' - md5_checksum: '531a712033d907b781d24030182357f4' - python-version: '3.9' - - - conda_env_name: '2023-1.1-py39-tiled' - zenodo_id: '7603440' - md5_checksum: '31dad55d53436605c47dbfc510e876a2' - python-version: '3.9' - - - conda_env_name: '2023-1.1-py310' - zenodo_id: '7603440' - md5_checksum: 'd60ceb752a798590334cbec3d0155fb6' - python-version: '3.10' - - - conda_env_name: '2023-1.1-py310-tiled' - zenodo_id: '7603440' - md5_checksum: 'f501314939d5237edb58244b0fcc6857' - python-version: '3.10' - - # 2023-1.2 - - conda_env_name: '2023-1.2-py39' - zenodo_id: '7651500' - md5_checksum: '14d0629f468df311918cfea2e6413bfa' - python-version: '3.9' - - - conda_env_name: '2023-1.2-py39-tiled' - zenodo_id: '7651500' - md5_checksum: '31586643a0cdd9aee8349ccb93ccac87' - python-version: '3.9' - - - conda_env_name: '2023-1.2-py310' - zenodo_id: '7651500' - md5_checksum: '2dc85c4d2392f751c76ed7d01f363814' - python-version: '3.10' - - - conda_env_name: '2023-1.2-py310-tiled' - zenodo_id: '7651500' - md5_checksum: 'ff8c0c475a007434c0011f5d8976d853' - python-version: '3.10' - - # 2023-1.3 - - conda_env_name: '2023-1.3-py39' - zenodo_id: '7791542' - md5_checksum: '9cc18f3e09dfa30c4e8f165a932ca394' - python-version: '3.9' - - - conda_env_name: '2023-1.3-py39-tiled' - zenodo_id: '7791542' - md5_checksum: '1f40f5786fc0c06baad3fd3b456a6abc' - python-version: '3.9' - - - conda_env_name: '2023-1.3-py310' - zenodo_id: '7791542' - md5_checksum: '3304bc674d14b2820d60b311b58661fd' - python-version: '3.10' - - - conda_env_name: '2023-1.3-py310-tiled' - zenodo_id: '7791542' - md5_checksum: 'b7869b9fdb1c076bfba1f7eb2c63c6f0' - python-version: '3.10' - - # 2023-2.0 - - conda_env_name: '2023-2.0-py310' - zenodo_id: '7876026' - md5_checksum: '2f81225554a16339e2a49432cd8a013b' - python-version: '3.10' - - - conda_env_name: '2023-2.0-py310-tiled' - zenodo_id: '7876026' - md5_checksum: 'ef85af9ff76bd0afa527bdfc391b0442' - python-version: '3.10' - - # 2023-2.1 - - conda_env_name: '2023-2.1-py310' - zenodo_id: '8098505' - md5_checksum: '31add8231d47d92821b5f4178cd9a58a' - python-version: '3.10' - - - conda_env_name: '2023-2.1-py310-tiled' - zenodo_id: '8098505' - md5_checksum: 'd75299b2a6ffc2ed5aca7d01150c97f0' - python-version: '3.10' - - # 2023-3.0 - - conda_env_name: '2023-3.0-py310' - zenodo_id: '8342731' - md5_checksum: '49ad184de893ab1a27db3c4d3e91a045' - python-version: '3.10' - - - conda_env_name: '2023-3.0-py310-tiled' - zenodo_id: '8342731' - md5_checksum: 'b727a21d791192bc3981fbe56deb44bd' - python-version: '3.10' - - # 2023-3.1 - - conda_env_name: '2023-3.1-py310' - zenodo_id: '8342775' - md5_checksum: '06c16140f909b11ac6cb712c96c849af' - python-version: '3.10' - - - conda_env_name: '2023-3.1-py310-tiled' - zenodo_id: '8342775' - md5_checksum: '1551a5591bfde30ddbb34d0798e4c94c' - python-version: '3.10' - - # 2023-3.2 - - conda_env_name: '2023-3.2-py310' - zenodo_id: '10056340' - md5_checksum: '71d187467850f10289ce0d8691aa80fd' - python-version: '3.10' - - - conda_env_name: '2023-3.2-py310-tiled' - zenodo_id: '10056340' - md5_checksum: '8c3740791930b2f230e64b0f9399b7b8' - python-version: '3.10' - - # 2023-3.3 - - conda_env_name: '2023-3.3-py310' - zenodo_id: '10148425' - md5_checksum: 'bb77ea6f7a5bb63980d5181f7666ecc2' - python-version: '3.10' - - - conda_env_name: '2023-3.3-py310-tiled' - zenodo_id: '10148425' - md5_checksum: '0a47934380db013b36f3e089afdff6aa' - python-version: '3.10' - - - conda_env_name: '2023-3.3-py311' - zenodo_id: '10148425' - md5_checksum: 'dd6c06a02e43b286cdf84908d8339992' - python-version: '3.11' - - - conda_env_name: '2023-3.3-py311-tiled' - zenodo_id: '10148425' - md5_checksum: '48d2a73ea743d34152400cd642fd7472' - python-version: '3.11' - - # 2024-1.0 - - conda_env_name: '2024-1.0-py310' - zenodo_id: '10548109' - md5_checksum: 'b6107d565f0ff9d7315eab96d28894e9' - python-version: '3.10' - - - conda_env_name: '2024-1.0-py310-tiled' - zenodo_id: '10548109' - md5_checksum: '216c4df93c6893a7ebe6456b678ac89e' - python-version: '3.10' - - - conda_env_name: '2024-1.0-py311' - zenodo_id: '10548109' - md5_checksum: '2592ed2002823c423b4a664183d384da' - python-version: '3.11' - - - conda_env_name: '2024-1.0-py311-tiled' - zenodo_id: '10548109' - md5_checksum: 'a51aac76916f19f5982650c1c6f21503' - python-version: '3.11' - - # 2024-2.0 - - conda_env_name: '2024-2.0-py310-tiled' - zenodo_id: '11122851' - md5_checksum: '91955234f1617df158e7301f05a89b64' - python-version: '3.10' - - - conda_env_name: '2024-2.0-py311-tiled' - zenodo_id: '11122851' - md5_checksum: 'd30ca6db3d93de99655a85a481c966c2' - python-version: '3.11' + include: ${{ matrix_include }} fail-fast: false From 37993cf69d65d5f87abe322f27e9b16224fcd3de Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 06:14:32 -0700 Subject: [PATCH 25/46] Define job dependency in conda-env-tests --- .github/workflows/conda-env-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index b1078ce..fef1ea7 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -21,6 +21,8 @@ jobs: echo "matrix_include=$(cat ${MATRIX_INCLUDE_FILE})" >> $GITHUB_OUTPUT test-in-conda-env: + + needs: matrix_prep runs-on: ubuntu-latest From 0b449cd4223143358f0deffb02429de054994065 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 06:16:41 -0700 Subject: [PATCH 26/46] Use the job dependency in conda-env-tests --- .github/workflows/conda-env-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index fef1ea7..7f5fa83 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -33,7 +33,7 @@ jobs: strategy: max-parallel: 4 matrix: - include: ${{ matrix_include }} + include: ${{ needs.matrix_prep.matrix_include }} fail-fast: false From bd4856cfb1e1625a176130c6972f3209c10aa7a8 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 06:55:51 -0700 Subject: [PATCH 27/46] Use python script to import test matrix --- .github/conda-env-tests/matrix_include.py | 25 +++++++++++++++++++++++ .github/workflows/conda-env-tests.yml | 14 +++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) create mode 100644 .github/conda-env-tests/matrix_include.py diff --git a/.github/conda-env-tests/matrix_include.py b/.github/conda-env-tests/matrix_include.py new file mode 100644 index 0000000..51c55ac --- /dev/null +++ b/.github/conda-env-tests/matrix_include.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +"""matrix_include.py + + Read a YAML configuration file and export the matrix.include field that is + used by GitHub Actions +""" + +import os +import yaml +import sys + + +filename = os.getenv("MATRIX_INCLUDE_FILE", "") +if not filename: + print("You must set environment variable MATRIX_INCLUDE_FILE") + print("to the path of the YAML configuration file for matrix.include input.") + sys.exit(1) + +with open(filename, "r") as config_file: + matrix_include = yaml.safe_load(config_file) + +print(matrix_include) + +sys.exit(0) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 7f5fa83..e5cb41c 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -12,13 +12,23 @@ jobs: steps: - name: Check out the code repo uses: actions/checkout@v4 - - id: set-matrix + - name: Install python interpreter + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install dependencies + run: | + set -vxeuo pipefail + python -m pip install pyyaml + - name: Import matrix.include from config file + id: set-matrix run: | set -vxeuo pipefail export MATRIX_INCLUDE_DIR=${{ github.workspace }}/.github/conda-env-tests export MATRIX_INCLUDE_FILE=${MATRIX_INCLUDE_DIR}/recent-envs.yml echo "Building matrix from file ${MATRIX_INCLUDE_FILE}" - echo "matrix_include=$(cat ${MATRIX_INCLUDE_FILE})" >> $GITHUB_OUTPUT + export MATRIX_INCLUDE_SCRIPT=${MATRIX_INCLUDE_DIR}/matrix_include.py + echo "matrix_include=$(python ${MATRIX_INCLUDE_SCRIPT})" >> $GITHUB_OUTPUT test-in-conda-env: From 7a86ea1af70c864e1c76d786054b46c8a9890a67 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 07:00:42 -0700 Subject: [PATCH 28/46] Reference output ot matrix_prep job --- .github/workflows/conda-env-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index e5cb41c..7548be2 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -43,7 +43,7 @@ jobs: strategy: max-parallel: 4 matrix: - include: ${{ needs.matrix_prep.matrix_include }} + include: ${{ needs.matrix_prep.outputs.matrix_include }} fail-fast: false From fb4bf8fa8222ca348b62b3c201c00d1c7ab6d6c9 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 07:07:12 -0700 Subject: [PATCH 29/46] Parse JSON input for test strategy matrix --- .github/workflows/conda-env-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 7548be2..8e93dd0 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -43,7 +43,7 @@ jobs: strategy: max-parallel: 4 matrix: - include: ${{ needs.matrix_prep.outputs.matrix_include }} + include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} fail-fast: false From 48ac597e9a61cb6f7baae5ee9221f1c7703ebc24 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 16:51:15 -0700 Subject: [PATCH 30/46] Refactor test-in-conda-env as reusable workflow --- .github/conda-env-tests/test-in-conda-env.yml | 133 ++++++++++++++++++ .github/workflows/conda-env-tests.yml | 123 +--------------- 2 files changed, 136 insertions(+), 120 deletions(-) create mode 100644 .github/conda-env-tests/test-in-conda-env.yml diff --git a/.github/conda-env-tests/test-in-conda-env.yml b/.github/conda-env-tests/test-in-conda-env.yml new file mode 100644 index 0000000..cbfe1bb --- /dev/null +++ b/.github/conda-env-tests/test-in-conda-env.yml @@ -0,0 +1,133 @@ +name: Unit Test in NSLS-II Conda Environment + +on: + workflow_call: + inputs: + matrix_include: + description: 'JSON array of strategy.matrix.include entries' + required: true + type: string + +jobs: + + test-in-conda-env: + + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l {0} + + strategy: + max-parallel: 4 + matrix: + include: ${{ fromJson(inputs.matrix_include) }} + + fail-fast: false + + steps: + + - name: Set env vars + run: | + export REPOSITORY_NAME="${GITHUB_REPOSITORY#*/}" # just the repo, as opposed to org/repo + echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV + + export CONDA_ENV_NAME='${{ matrix.conda_env_name }}' + echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV + + export ZENODO_ID='${{ matrix.zenodo_id }}' + echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV + + export MD5_CHECKSUM='${{ matrix.md5_checksum }}' + echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV + + # Workaround for unset GDAL env variables used by newer conda envs + export GDAL_DATA="${GDAL_DATA-''}" + echo "GDAL_DATA=${GDAL_DATA}" >> $GITHUB_ENV + export GDAL_DRIVER_PATH="${GDAL_DRIVER_PATH-''}" + echo "GDAL_DRIVER_PATH=${GDAL_DRIVER_PATH}" >> $GITHUB_ENV + export GEOTIFF_CSV="${GEOTIFF_CSV-''}" + echo "GEOTIFF_CSV=${GEOTIFF_CSV}" >> $GITHUB_ENV + + # Workaround for unset MKL env variables used by newer conda envs + export MKL_INTERFACE_LAYER="${MKL_INTERFACE_LAYER-''}" + echo "MKL_INTERFACE_LAYER=${MKL_INTERFACE_LAYER}" >> $GITHUB_ENV + + - name: Check out the code repo + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} with conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }} + auto-update-conda: true + miniconda-version: "latest" + python-version: ${{ matrix.python-version }} + mamba-version: "*" + channels: conda-forge + + - name: Cache the archived conda environment + uses: actions/cache@v3 + with: + path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} + key: + ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}--${{ runner.arch }}-${{ + env.MD5_CHECKSUM }}-${{ + env.CACHE_NUMBER }} + env: + # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed + CACHE_NUMBER: 0 + id: cache-conda-archive + + - name: Download and install the archived conda environment + if: steps.cache-conda-archive.outputs.cache-hit != 'true' + run: | + set -vxeuo pipefail + conda activate "${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }}" + url="https://zenodo.org/record/${ZENODO_ID}/files/${CONDA_ENV_NAME}.tar.gz?download=1" + wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz + status=$? + if [ $status -gt 0 ]; then + echo "Cannot download from ${url}. Exit code: ${status}" + exit $status + fi + echo "${MD5_CHECKSUM} ${CONDA_ENV_NAME}.tar.gz" > checksum.txt + md5sum --check checksum.txt + mkdir -p $HOME/miniconda/envs/${CONDA_ENV_NAME} + tar -xf ${CONDA_ENV_NAME}.tar.gz -C $HOME/miniconda/envs/${CONDA_ENV_NAME} + + - name: Activate and inspect the archived conda environment + run: | + set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + conda unpack + + conda info -a + conda env list + conda list + pip list + + - name: Install the package and its dependencies + run: | + set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + echo "Using conda environment at ${CONDA_PREFIX}" + pip install --upgrade pip setuptools wheel + pip install -r requirements-dev.txt + pip install -r requirements-extras.txt + pip install -e . + pip list + + - name: Test with pytest + run: | + set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + echo "Using conda environment at ${CONDA_PREFIX}" + coverage run --source=csxtools run_tests.py + coverage xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + file: ./coverage.xml + flags: conda-env-unittests diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 8e93dd0..6ae7974 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -33,123 +33,6 @@ jobs: test-in-conda-env: needs: matrix_prep - - runs-on: ubuntu-latest - - defaults: - run: - shell: bash -l {0} - - strategy: - max-parallel: 4 - matrix: - include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} - - fail-fast: false - - steps: - - - name: Set env vars - run: | - export REPOSITORY_NAME="${GITHUB_REPOSITORY#*/}" # just the repo, as opposed to org/repo - echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV - - export CONDA_ENV_NAME='${{ matrix.conda_env_name }}' - echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV - - export ZENODO_ID='${{ matrix.zenodo_id }}' - echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV - - export MD5_CHECKSUM='${{ matrix.md5_checksum }}' - echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV - - # Workaround for unset GDAL env variables used by newer conda envs - export GDAL_DATA="${GDAL_DATA-''}" - echo "GDAL_DATA=${GDAL_DATA}" >> $GITHUB_ENV - export GDAL_DRIVER_PATH="${GDAL_DRIVER_PATH-''}" - echo "GDAL_DRIVER_PATH=${GDAL_DRIVER_PATH}" >> $GITHUB_ENV - export GEOTIFF_CSV="${GEOTIFF_CSV-''}" - echo "GEOTIFF_CSV=${GEOTIFF_CSV}" >> $GITHUB_ENV - - # Workaround for unset MKL env variables used by newer conda envs - export MKL_INTERFACE_LAYER="${MKL_INTERFACE_LAYER-''}" - echo "MKL_INTERFACE_LAYER=${MKL_INTERFACE_LAYER}" >> $GITHUB_ENV - - - name: Check out the code repo - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} with conda - uses: conda-incubator/setup-miniconda@v3 - with: - activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }} - auto-update-conda: true - miniconda-version: "latest" - python-version: ${{ matrix.python-version }} - mamba-version: "*" - channels: conda-forge - - - name: Cache the archived conda environment - uses: actions/cache@v3 - with: - path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} - key: - ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}--${{ runner.arch }}-${{ - env.MD5_CHECKSUM }}-${{ - env.CACHE_NUMBER }} - env: - # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed - CACHE_NUMBER: 0 - id: cache-conda-archive - - - name: Download and install the archived conda environment - if: steps.cache-conda-archive.outputs.cache-hit != 'true' - run: | - set -vxeuo pipefail - conda activate "${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }}" - url="https://zenodo.org/record/${ZENODO_ID}/files/${CONDA_ENV_NAME}.tar.gz?download=1" - wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz - status=$? - if [ $status -gt 0 ]; then - echo "Cannot download from ${url}. Exit code: ${status}" - exit $status - fi - echo "${MD5_CHECKSUM} ${CONDA_ENV_NAME}.tar.gz" > checksum.txt - md5sum --check checksum.txt - mkdir -p $HOME/miniconda/envs/${CONDA_ENV_NAME} - tar -xf ${CONDA_ENV_NAME}.tar.gz -C $HOME/miniconda/envs/${CONDA_ENV_NAME} - - - name: Activate and inspect the archived conda environment - run: | - set -vxeuo pipefail - conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} - conda unpack - - conda info -a - conda env list - conda list - pip list - - - name: Install the package and its dependencies - run: | - set -vxeuo pipefail - conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} - echo "Using conda environment at ${CONDA_PREFIX}" - pip install --upgrade pip setuptools wheel - pip install -r requirements-dev.txt - pip install -r requirements-extras.txt - pip install -e . - pip list - - - name: Test with pytest - run: | - set -vxeuo pipefail - conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} - echo "Using conda environment at ${CONDA_PREFIX}" - coverage run --source=csxtools run_tests.py - coverage xml - - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v2 - with: - file: ./coverage.xml - flags: conda-env-unittests + uses: ./.github/conda-env-tests/test-in-conda-env.yml + with: + matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} From 6ad70f78ee0547ab27f7b6dae21a1364787f8c1e Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 16:56:08 -0700 Subject: [PATCH 31/46] Use branch name to version the workflow action --- .github/workflows/conda-env-tests.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 6ae7974..e90b045 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -2,6 +2,9 @@ name: Unit Tests in NSLS-II Conda Environments on: [push, pull_request, workflow_dispatch] +env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + jobs: matrix_prep: @@ -33,6 +36,6 @@ jobs: test-in-conda-env: needs: matrix_prep - uses: ./.github/conda-env-tests/test-in-conda-env.yml + uses: ./.github/conda-env-tests/test-in-conda-env.yml@${{ BRANCH_NAME }} with: matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} From c57777acb0e6be62ce7852991bc2e5961a713331 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 16:57:52 -0700 Subject: [PATCH 32/46] Fix env usage --- .github/workflows/conda-env-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index e90b045..ad3bfd0 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -36,6 +36,6 @@ jobs: test-in-conda-env: needs: matrix_prep - uses: ./.github/conda-env-tests/test-in-conda-env.yml@${{ BRANCH_NAME }} + uses: ./.github/conda-env-tests/test-in-conda-env.yml@${{ env.BRANCH_NAME }} with: matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} From 8d6844085414c8f9ea6c8253aebc23bca7e3edd6 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 17:00:29 -0700 Subject: [PATCH 33/46] Remove spaces from variable expansion --- .github/workflows/conda-env-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index ad3bfd0..22fa0a5 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -36,6 +36,6 @@ jobs: test-in-conda-env: needs: matrix_prep - uses: ./.github/conda-env-tests/test-in-conda-env.yml@${{ env.BRANCH_NAME }} + uses: ./.github/conda-env-tests/test-in-conda-env.yml@${{env.BRANCH_NAME}} with: matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} From bee124ef0e8db0b0a6e6a7b524348e0ee2b445f6 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 17:02:00 -0700 Subject: [PATCH 34/46] Move reusable workflows into workflows folder --- .github/workflows/conda-env-tests.yml | 2 +- .../reusable}/test-in-conda-env.yml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename .github/{conda-env-tests => workflows/reusable}/test-in-conda-env.yml (100%) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 22fa0a5..0437786 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -36,6 +36,6 @@ jobs: test-in-conda-env: needs: matrix_prep - uses: ./.github/conda-env-tests/test-in-conda-env.yml@${{env.BRANCH_NAME}} + uses: ./.github/workflows/reusable/test-in-conda-env.yml@${{env.BRANCH_NAME}} with: matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} diff --git a/.github/conda-env-tests/test-in-conda-env.yml b/.github/workflows/reusable/test-in-conda-env.yml similarity index 100% rename from .github/conda-env-tests/test-in-conda-env.yml rename to .github/workflows/reusable/test-in-conda-env.yml From 7293beb286863b3bc288a7d3bb48d946dadd11fd Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 17:14:47 -0700 Subject: [PATCH 35/46] Try actions rather than workflows --- .../reusable => actions}/test-in-conda-env.yml | 0 .github/workflows/conda-env-tests.yml | 11 ++++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) rename .github/{workflows/reusable => actions}/test-in-conda-env.yml (100%) diff --git a/.github/workflows/reusable/test-in-conda-env.yml b/.github/actions/test-in-conda-env.yml similarity index 100% rename from .github/workflows/reusable/test-in-conda-env.yml rename to .github/actions/test-in-conda-env.yml diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 0437786..674e99e 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -36,6 +36,11 @@ jobs: test-in-conda-env: needs: matrix_prep - uses: ./.github/workflows/reusable/test-in-conda-env.yml@${{env.BRANCH_NAME}} - with: - matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} + runs-on: ubuntu-latest + steps: + - name: Check out the code repo + uses: actions/checkout@v4 + - name: Dispatch the test + uses: ./.github/actions/test-in-conda-env.yml@${{env.BRANCH_NAME}} + with: + matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} From 53d21568a1e833931a8b2eb467d5145be72b0f88 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 17:17:16 -0700 Subject: [PATCH 36/46] Debug env var --- .github/workflows/conda-env-tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 674e99e..daa2d05 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -38,6 +38,9 @@ jobs: needs: matrix_prep runs-on: ubuntu-latest steps: + - name: Debug + run: | + echo ${{ env.BRANCH_NAME }} - name: Check out the code repo uses: actions/checkout@v4 - name: Dispatch the test From 0499b410272a4b3793ef67a5db57ee23a1666a50 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 17:18:21 -0700 Subject: [PATCH 37/46] Debug env var isolated --- .github/workflows/conda-env-tests.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index daa2d05..aba4eca 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -41,9 +41,9 @@ jobs: - name: Debug run: | echo ${{ env.BRANCH_NAME }} - - name: Check out the code repo - uses: actions/checkout@v4 - - name: Dispatch the test - uses: ./.github/actions/test-in-conda-env.yml@${{env.BRANCH_NAME}} - with: - matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} + # - name: Check out the code repo + # uses: actions/checkout@v4 + # - name: Dispatch the test + # uses: ./.github/actions/test-in-conda-env.yml@${{env.BRANCH_NAME}} + # with: + # matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} From 1299368de9ed1f01c7799699a69a5e43b05c40aa Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 17:20:58 -0700 Subject: [PATCH 38/46] Revert to workflows for dispatched workflow --- .../_test-in-conda-env.yml} | 0 .github/workflows/conda-env-tests.yml | 13 ++++--------- 2 files changed, 4 insertions(+), 9 deletions(-) rename .github/{actions/test-in-conda-env.yml => workflows/_test-in-conda-env.yml} (100%) diff --git a/.github/actions/test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml similarity index 100% rename from .github/actions/test-in-conda-env.yml rename to .github/workflows/_test-in-conda-env.yml diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index aba4eca..4777e7f 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -38,12 +38,7 @@ jobs: needs: matrix_prep runs-on: ubuntu-latest steps: - - name: Debug - run: | - echo ${{ env.BRANCH_NAME }} - # - name: Check out the code repo - # uses: actions/checkout@v4 - # - name: Dispatch the test - # uses: ./.github/actions/test-in-conda-env.yml@${{env.BRANCH_NAME}} - # with: - # matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} + - name: Dispatch the test + uses: ./.github/workflows/_test-in-conda-env.yml + with: + matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} From 7b9bd87dd2bcb07ded1d547138f684ffa0d059fc Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 17:22:54 -0700 Subject: [PATCH 39/46] Checkout code for local action --- .github/workflows/conda-env-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 4777e7f..c9378c3 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -38,6 +38,8 @@ jobs: needs: matrix_prep runs-on: ubuntu-latest steps: + - name: Check out the code repo + uses: actions/checkout@v4 - name: Dispatch the test uses: ./.github/workflows/_test-in-conda-env.yml with: From 7618e48bcc9d5d0efccb60fadb646e7639e940aa Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 17:24:28 -0700 Subject: [PATCH 40/46] Revert to single workflow step --- .github/workflows/conda-env-tests.yml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index c9378c3..ba2be37 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -36,11 +36,6 @@ jobs: test-in-conda-env: needs: matrix_prep - runs-on: ubuntu-latest - steps: - - name: Check out the code repo - uses: actions/checkout@v4 - - name: Dispatch the test - uses: ./.github/workflows/_test-in-conda-env.yml - with: - matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} + uses: ./.github/workflows/_test-in-conda-env.yml + with: + matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} From e941a6ae9266a5e9faee5e8a9988364e60ebeb32 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 17:26:41 -0700 Subject: [PATCH 41/46] Pass JSON string, not sequence to workflow --- .github/workflows/conda-env-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index ba2be37..dcbb939 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -38,4 +38,4 @@ jobs: needs: matrix_prep uses: ./.github/workflows/_test-in-conda-env.yml with: - matrix_include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} + matrix_include: ${{ needs.matrix_prep.outputs.matrix_include }} From b2688ca9b69368ab1a0d7939c23ab5e3f428c2e2 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 17:46:31 -0700 Subject: [PATCH 42/46] Extract matrix strategy from inner to outer workflow --- .github/workflows/_test-in-conda-env.yml | 37 ++++++++++++++---------- .github/workflows/conda-env-tests.yml | 10 ++++++- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/.github/workflows/_test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml index cbfe1bb..5ebfbcf 100644 --- a/.github/workflows/_test-in-conda-env.yml +++ b/.github/workflows/_test-in-conda-env.yml @@ -3,8 +3,20 @@ name: Unit Test in NSLS-II Conda Environment on: workflow_call: inputs: - matrix_include: - description: 'JSON array of strategy.matrix.include entries' + conda_env_name: + description: 'Name of the conda environment to use for running the test' + required: true + type: string + zenodo_id: + description: 'Zenodo ID for the archived conda environment' + required: true + type: string + md5_checksum: + description: 'MD5 checksum for the archived conda environment' + required: true + type: string + python-version: + description: 'Python version ID to use for running the test' required: true type: string @@ -18,13 +30,6 @@ jobs: run: shell: bash -l {0} - strategy: - max-parallel: 4 - matrix: - include: ${{ fromJson(inputs.matrix_include) }} - - fail-fast: false - steps: - name: Set env vars @@ -32,13 +37,13 @@ jobs: export REPOSITORY_NAME="${GITHUB_REPOSITORY#*/}" # just the repo, as opposed to org/repo echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV - export CONDA_ENV_NAME='${{ matrix.conda_env_name }}' + export CONDA_ENV_NAME='${{ inputs.conda_env_name }}' echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV - export ZENODO_ID='${{ matrix.zenodo_id }}' + export ZENODO_ID='${{ inputs.zenodo_id }}' echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV - export MD5_CHECKSUM='${{ matrix.md5_checksum }}' + export MD5_CHECKSUM='${{ inputs.md5_checksum }}' echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV # Workaround for unset GDAL env variables used by newer conda envs @@ -56,13 +61,13 @@ jobs: - name: Check out the code repo uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} with conda + - name: Set up Python ${{ inputs.python-version }} with conda uses: conda-incubator/setup-miniconda@v3 with: - activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }} + activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ inputs.python-version }} auto-update-conda: true miniconda-version: "latest" - python-version: ${{ matrix.python-version }} + python-version: ${{ inputs.python-version }} mamba-version: "*" channels: conda-forge @@ -83,7 +88,7 @@ jobs: if: steps.cache-conda-archive.outputs.cache-hit != 'true' run: | set -vxeuo pipefail - conda activate "${{ env.REPOSITORY_NAME }}-py${{ matrix.python-version }}" + conda activate "${{ env.REPOSITORY_NAME }}-py${{ inputs.python-version }}" url="https://zenodo.org/record/${ZENODO_ID}/files/${CONDA_ENV_NAME}.tar.gz?download=1" wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz status=$? diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index dcbb939..0ac5b8f 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -38,4 +38,12 @@ jobs: needs: matrix_prep uses: ./.github/workflows/_test-in-conda-env.yml with: - matrix_include: ${{ needs.matrix_prep.outputs.matrix_include }} + conda_env_name: ${{ matrix.conda_env_name }} + zenodo_id: ${{ matrix.zenodo_id }} + md5_checksum: ${{ matrix.md5_checksum }} + python-version: ${{ matrix.python-version }} + strategy: + max-parallel: 4 + matrix: + include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} + fail-fast: false From b7c38cacd5caddd78e9b3e5943db4a4c5b0f1f8c Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 18:41:50 -0700 Subject: [PATCH 43/46] Extract matrix_prep to a reusable workflow --- .github/workflows/_matrix_prep.yml | 52 +++++++++++++++++++++++++++ .github/workflows/conda-env-tests.yml | 28 +++------------ 2 files changed, 56 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/_matrix_prep.yml diff --git a/.github/workflows/_matrix_prep.yml b/.github/workflows/_matrix_prep.yml new file mode 100644 index 0000000..c0a58d9 --- /dev/null +++ b/.github/workflows/_matrix_prep.yml @@ -0,0 +1,52 @@ +name: Prepare a matrix of NSLS-II Conda Environments + +on: + workflow_call: + inputs: + config_prefix: + description: > + 'YAML file prefix {value}-env.yaml for a configuration file that' + 'conains the strategy.matrix.includes for conda environments' + required: true + type: string + default: recent + + outputs: + matrix_include: + description: 'JSON array of strategy.matrix.include entries' + value: ${{ jobs.matrix_prep.outputs.matrix_include }} + type: string + +jobs: + + matrix_prep: + + runs-on: ubuntu-latest + + outputs: + matrix_include: ${{ steps.set-matrix.outputs.matrix_include }} + + steps: + + - name: Check out the code repo + uses: actions/checkout@v4 + + - name: Install python interpreter + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + set -vxeuo pipefail + python -m pip install pyyaml + + - name: Import matrix.include from config file + id: set-matrix + run: | + set -vxeuo pipefail + export MATRIX_INCLUDE_DIR=${{ github.workspace }}/.github/conda-env-tests + export MATRIX_INCLUDE_FILE=${MATRIX_INCLUDE_DIR}/${{ inputs.config_prefix }}-envs.yml + echo "Building matrix from file ${MATRIX_INCLUDE_FILE}" + export MATRIX_INCLUDE_SCRIPT=${MATRIX_INCLUDE_DIR}/matrix_include.py + echo "matrix_include=$(python ${MATRIX_INCLUDE_SCRIPT})" >> $GITHUB_OUTPUT diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml index 0ac5b8f..aa0673a 100644 --- a/.github/workflows/conda-env-tests.yml +++ b/.github/workflows/conda-env-tests.yml @@ -8,30 +8,10 @@ env: jobs: matrix_prep: - - runs-on: ubuntu-latest - outputs: - matrix_include: ${{ steps.set-matrix.outputs.matrix_include }} - steps: - - name: Check out the code repo - uses: actions/checkout@v4 - - name: Install python interpreter - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - name: Install dependencies - run: | - set -vxeuo pipefail - python -m pip install pyyaml - - name: Import matrix.include from config file - id: set-matrix - run: | - set -vxeuo pipefail - export MATRIX_INCLUDE_DIR=${{ github.workspace }}/.github/conda-env-tests - export MATRIX_INCLUDE_FILE=${MATRIX_INCLUDE_DIR}/recent-envs.yml - echo "Building matrix from file ${MATRIX_INCLUDE_FILE}" - export MATRIX_INCLUDE_SCRIPT=${MATRIX_INCLUDE_DIR}/matrix_include.py - echo "matrix_include=$(python ${MATRIX_INCLUDE_SCRIPT})" >> $GITHUB_OUTPUT + + uses: ./.github/workflows/_matrix_prep.yml + with: + config_prefix: recent test-in-conda-env: From 6ccf7518e424953557eb9a34a9f55ec938863d9d Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 18:43:24 -0700 Subject: [PATCH 44/46] Do not define type of GHA workflow output --- .github/workflows/_matrix_prep.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/_matrix_prep.yml b/.github/workflows/_matrix_prep.yml index c0a58d9..2e59e43 100644 --- a/.github/workflows/_matrix_prep.yml +++ b/.github/workflows/_matrix_prep.yml @@ -13,9 +13,8 @@ on: outputs: matrix_include: - description: 'JSON array of strategy.matrix.include entries' + description: 'JSON array of strategy.matrix.include entries, as string' value: ${{ jobs.matrix_prep.outputs.matrix_include }} - type: string jobs: From e929218885622bf1dacb74301db4bdbecd0dd3ad Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 18:46:38 -0700 Subject: [PATCH 45/46] Split long-running legacy tests into separate GHA --- .github/workflows/_matrix_prep.yml | 1 + .github/workflows/legacy-conda-env-tests.yml | 29 ++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 .github/workflows/legacy-conda-env-tests.yml diff --git a/.github/workflows/_matrix_prep.yml b/.github/workflows/_matrix_prep.yml index 2e59e43..f95e378 100644 --- a/.github/workflows/_matrix_prep.yml +++ b/.github/workflows/_matrix_prep.yml @@ -2,6 +2,7 @@ name: Prepare a matrix of NSLS-II Conda Environments on: workflow_call: + inputs: config_prefix: description: > diff --git a/.github/workflows/legacy-conda-env-tests.yml b/.github/workflows/legacy-conda-env-tests.yml new file mode 100644 index 0000000..0fe178e --- /dev/null +++ b/.github/workflows/legacy-conda-env-tests.yml @@ -0,0 +1,29 @@ +name: Unit Tests in NSLS-II Conda Environments + +on: [workflow_dispatch] + +env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + +jobs: + + matrix_prep: + + uses: ./.github/workflows/_matrix_prep.yml + with: + config_prefix: legacy + + test-in-conda-env: + + needs: matrix_prep + uses: ./.github/workflows/_test-in-conda-env.yml + with: + conda_env_name: ${{ matrix.conda_env_name }} + zenodo_id: ${{ matrix.zenodo_id }} + md5_checksum: ${{ matrix.md5_checksum }} + python-version: ${{ matrix.python-version }} + strategy: + max-parallel: 4 + matrix: + include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} + fail-fast: false From c2aaa070c169b6f56aa8b04cbd7f8ea41e8e4cc9 Mon Sep 17 00:00:00 2001 From: Padraic Shafer Date: Wed, 22 May 2024 18:47:34 -0700 Subject: [PATCH 46/46] Rename long-running legacy tests --- .github/workflows/legacy-conda-env-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/legacy-conda-env-tests.yml b/.github/workflows/legacy-conda-env-tests.yml index 0fe178e..7831f1d 100644 --- a/.github/workflows/legacy-conda-env-tests.yml +++ b/.github/workflows/legacy-conda-env-tests.yml @@ -1,4 +1,4 @@ -name: Unit Tests in NSLS-II Conda Environments +name: Legacy Unit Tests in NSLS-II Conda Environments on: [workflow_dispatch]