diff --git a/.github/conda-env-tests/legacy-envs.yml b/.github/conda-env-tests/legacy-envs.yml new file mode 100644 index 0000000..f86d6c8 --- /dev/null +++ b/.github/conda-env-tests/legacy-envs.yml @@ -0,0 +1,140 @@ +--- + +# 2023-1.0 +- conda_env_name: '2023-1.0-py39' + zenodo_id: '7470912' + md5_checksum: '651be0e26cdb230dd8c809b4eed046e9' + python-version: '3.9' + +- conda_env_name: '2023-1.0-py39-tiled' + zenodo_id: '7470912' + md5_checksum: 'c092b113d19510ac9ecb68eda66d3e79' + python-version: '3.9' + +- conda_env_name: '2023-1.0-py310' + zenodo_id: '7470912' + md5_checksum: '2c93e892a55d8c684acea4ccb2680ac5' + python-version: '3.10' + +- conda_env_name: '2023-1.0-py310-tiled' + zenodo_id: '7470912' + md5_checksum: '1de3a83f0ead58d3cb60d721478748a2' + python-version: '3.10' + +# 2023-1.1 +- conda_env_name: '2023-1.1-py39' + zenodo_id: '7603440' + md5_checksum: '531a712033d907b781d24030182357f4' + python-version: '3.9' + +- conda_env_name: '2023-1.1-py39-tiled' + zenodo_id: '7603440' + md5_checksum: '31dad55d53436605c47dbfc510e876a2' + python-version: '3.9' + +- conda_env_name: '2023-1.1-py310' + zenodo_id: '7603440' + md5_checksum: 'd60ceb752a798590334cbec3d0155fb6' + python-version: '3.10' + +- conda_env_name: '2023-1.1-py310-tiled' + zenodo_id: '7603440' + md5_checksum: 'f501314939d5237edb58244b0fcc6857' + python-version: '3.10' + +# 2023-1.2 +- conda_env_name: '2023-1.2-py39' + zenodo_id: '7651500' + md5_checksum: '14d0629f468df311918cfea2e6413bfa' + python-version: '3.9' + +- conda_env_name: '2023-1.2-py39-tiled' + zenodo_id: '7651500' + md5_checksum: '31586643a0cdd9aee8349ccb93ccac87' + python-version: '3.9' + +- conda_env_name: '2023-1.2-py310' + zenodo_id: '7651500' + md5_checksum: '2dc85c4d2392f751c76ed7d01f363814' + python-version: '3.10' + +- conda_env_name: '2023-1.2-py310-tiled' + zenodo_id: '7651500' + md5_checksum: 'ff8c0c475a007434c0011f5d8976d853' + python-version: '3.10' + +# 2023-1.3 +- conda_env_name: '2023-1.3-py39' + zenodo_id: '7791542' + md5_checksum: '9cc18f3e09dfa30c4e8f165a932ca394' + python-version: '3.9' + +- conda_env_name: '2023-1.3-py39-tiled' + zenodo_id: '7791542' + md5_checksum: '1f40f5786fc0c06baad3fd3b456a6abc' + python-version: '3.9' + +- conda_env_name: '2023-1.3-py310' + zenodo_id: '7791542' + md5_checksum: '3304bc674d14b2820d60b311b58661fd' + python-version: '3.10' + +- conda_env_name: '2023-1.3-py310-tiled' + zenodo_id: '7791542' + md5_checksum: 'b7869b9fdb1c076bfba1f7eb2c63c6f0' + python-version: '3.10' + +# 2023-2.0 +- conda_env_name: '2023-2.0-py310' + zenodo_id: '7876026' + md5_checksum: '2f81225554a16339e2a49432cd8a013b' + python-version: '3.10' + +- conda_env_name: '2023-2.0-py310-tiled' + zenodo_id: '7876026' + md5_checksum: 'ef85af9ff76bd0afa527bdfc391b0442' + python-version: '3.10' + +# 2023-2.1 +- conda_env_name: '2023-2.1-py310' + zenodo_id: '8098505' + md5_checksum: '31add8231d47d92821b5f4178cd9a58a' + python-version: '3.10' + +- conda_env_name: '2023-2.1-py310-tiled' + zenodo_id: '8098505' + md5_checksum: 'd75299b2a6ffc2ed5aca7d01150c97f0' + python-version: '3.10' + +# 2023-3.0 +- conda_env_name: '2023-3.0-py310' + zenodo_id: '8342731' + md5_checksum: '49ad184de893ab1a27db3c4d3e91a045' + python-version: '3.10' + +- conda_env_name: '2023-3.0-py310-tiled' + zenodo_id: '8342731' + md5_checksum: 'b727a21d791192bc3981fbe56deb44bd' + python-version: '3.10' + +# 2023-3.1 +- conda_env_name: '2023-3.1-py310' + zenodo_id: '8342775' + md5_checksum: '06c16140f909b11ac6cb712c96c849af' + python-version: '3.10' + +- conda_env_name: '2023-3.1-py310-tiled' + zenodo_id: '8342775' + md5_checksum: '1551a5591bfde30ddbb34d0798e4c94c' + python-version: '3.10' + +# 2023-3.2 +- conda_env_name: '2023-3.2-py310' + zenodo_id: '10056340' + md5_checksum: '71d187467850f10289ce0d8691aa80fd' + python-version: '3.10' + +- conda_env_name: '2023-3.2-py310-tiled' + zenodo_id: '10056340' + md5_checksum: '8c3740791930b2f230e64b0f9399b7b8' + python-version: '3.10' diff --git a/.github/conda-env-tests/matrix_include.py b/.github/conda-env-tests/matrix_include.py new file mode 100644 index 0000000..51c55ac --- /dev/null +++ b/.github/conda-env-tests/matrix_include.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +"""matrix_include.py + + Read a YAML configuration file and export the matrix.include field that is + used by GitHub Actions +""" + +import os +import yaml +import sys + + +filename = os.getenv("MATRIX_INCLUDE_FILE", "") +if not filename: + print("You must set environment variable MATRIX_INCLUDE_FILE") + print("to the path of the YAML configuration file for matrix.include input.") + sys.exit(1) + +with open(filename, "r") as config_file: + matrix_include = yaml.safe_load(config_file) + +print(matrix_include) + +sys.exit(0) diff --git a/.github/conda-env-tests/recent-envs.yml b/.github/conda-env-tests/recent-envs.yml new file mode 100644 index 0000000..412029f --- /dev/null +++ b/.github/conda-env-tests/recent-envs.yml @@ -0,0 +1,54 @@ +--- + +# 2023-3.3 +- conda_env_name: '2023-3.3-py310' + zenodo_id: '10148425' + md5_checksum: 'bb77ea6f7a5bb63980d5181f7666ecc2' + python-version: '3.10' + +- conda_env_name: '2023-3.3-py310-tiled' + zenodo_id: '10148425' + md5_checksum: '0a47934380db013b36f3e089afdff6aa' + python-version: '3.10' + +- conda_env_name: '2023-3.3-py311' + zenodo_id: '10148425' + md5_checksum: 'dd6c06a02e43b286cdf84908d8339992' + python-version: '3.11' + +- conda_env_name: '2023-3.3-py311-tiled' + zenodo_id: '10148425' + md5_checksum: '48d2a73ea743d34152400cd642fd7472' + python-version: '3.11' + +# 2024-1.0 +- conda_env_name: '2024-1.0-py310' + zenodo_id: '10548109' + md5_checksum: 'b6107d565f0ff9d7315eab96d28894e9' + python-version: '3.10' + +- conda_env_name: '2024-1.0-py310-tiled' + zenodo_id: '10548109' + md5_checksum: '216c4df93c6893a7ebe6456b678ac89e' + python-version: '3.10' + +- conda_env_name: '2024-1.0-py311' + zenodo_id: '10548109' + md5_checksum: '2592ed2002823c423b4a664183d384da' + python-version: '3.11' + +- conda_env_name: '2024-1.0-py311-tiled' + zenodo_id: '10548109' + md5_checksum: 'a51aac76916f19f5982650c1c6f21503' + python-version: '3.11' + +# 2024-2.0 +- conda_env_name: '2024-2.0-py310-tiled' + zenodo_id: '11122851' + md5_checksum: '91955234f1617df158e7301f05a89b64' + python-version: '3.10' + +- conda_env_name: '2024-2.0-py311-tiled' + zenodo_id: '11122851' + md5_checksum: 'd30ca6db3d93de99655a85a481c966c2' + python-version: '3.11' diff --git a/.github/workflows/_matrix_prep.yml b/.github/workflows/_matrix_prep.yml new file mode 100644 index 0000000..f95e378 --- /dev/null +++ b/.github/workflows/_matrix_prep.yml @@ -0,0 +1,52 @@ +name: Prepare a matrix of NSLS-II Conda Environments + +on: + workflow_call: + + inputs: + config_prefix: + description: > + 'YAML file prefix {value}-env.yaml for a configuration file that' + 'conains the strategy.matrix.includes for conda environments' + required: true + type: string + default: recent + + outputs: + matrix_include: + description: 'JSON array of strategy.matrix.include entries, as string' + value: ${{ jobs.matrix_prep.outputs.matrix_include }} + +jobs: + + matrix_prep: + + runs-on: ubuntu-latest + + outputs: + matrix_include: ${{ steps.set-matrix.outputs.matrix_include }} + + steps: + + - name: Check out the code repo + uses: actions/checkout@v4 + + - name: Install python interpreter + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install dependencies + run: | + set -vxeuo pipefail + python -m pip install pyyaml + + - name: Import matrix.include from config file + id: set-matrix + run: | + set -vxeuo pipefail + export MATRIX_INCLUDE_DIR=${{ github.workspace }}/.github/conda-env-tests + export MATRIX_INCLUDE_FILE=${MATRIX_INCLUDE_DIR}/${{ inputs.config_prefix }}-envs.yml + echo "Building matrix from file ${MATRIX_INCLUDE_FILE}" + export MATRIX_INCLUDE_SCRIPT=${MATRIX_INCLUDE_DIR}/matrix_include.py + echo "matrix_include=$(python ${MATRIX_INCLUDE_SCRIPT})" >> $GITHUB_OUTPUT diff --git a/.github/workflows/_test-in-conda-env.yml b/.github/workflows/_test-in-conda-env.yml new file mode 100644 index 0000000..5ebfbcf --- /dev/null +++ b/.github/workflows/_test-in-conda-env.yml @@ -0,0 +1,138 @@ +name: Unit Test in NSLS-II Conda Environment + +on: + workflow_call: + inputs: + conda_env_name: + description: 'Name of the conda environment to use for running the test' + required: true + type: string + zenodo_id: + description: 'Zenodo ID for the archived conda environment' + required: true + type: string + md5_checksum: + description: 'MD5 checksum for the archived conda environment' + required: true + type: string + python-version: + description: 'Python version ID to use for running the test' + required: true + type: string + +jobs: + + test-in-conda-env: + + runs-on: ubuntu-latest + + defaults: + run: + shell: bash -l {0} + + steps: + + - name: Set env vars + run: | + export REPOSITORY_NAME="${GITHUB_REPOSITORY#*/}" # just the repo, as opposed to org/repo + echo "REPOSITORY_NAME=${REPOSITORY_NAME}" >> $GITHUB_ENV + + export CONDA_ENV_NAME='${{ inputs.conda_env_name }}' + echo "CONDA_ENV_NAME=${CONDA_ENV_NAME}" >> $GITHUB_ENV + + export ZENODO_ID='${{ inputs.zenodo_id }}' + echo "ZENODO_ID=${ZENODO_ID}" >> $GITHUB_ENV + + export MD5_CHECKSUM='${{ inputs.md5_checksum }}' + echo "MD5_CHECKSUM=${MD5_CHECKSUM}" >> $GITHUB_ENV + + # Workaround for unset GDAL env variables used by newer conda envs + export GDAL_DATA="${GDAL_DATA-''}" + echo "GDAL_DATA=${GDAL_DATA}" >> $GITHUB_ENV + export GDAL_DRIVER_PATH="${GDAL_DRIVER_PATH-''}" + echo "GDAL_DRIVER_PATH=${GDAL_DRIVER_PATH}" >> $GITHUB_ENV + export GEOTIFF_CSV="${GEOTIFF_CSV-''}" + echo "GEOTIFF_CSV=${GEOTIFF_CSV}" >> $GITHUB_ENV + + # Workaround for unset MKL env variables used by newer conda envs + export MKL_INTERFACE_LAYER="${MKL_INTERFACE_LAYER-''}" + echo "MKL_INTERFACE_LAYER=${MKL_INTERFACE_LAYER}" >> $GITHUB_ENV + + - name: Check out the code repo + uses: actions/checkout@v4 + + - name: Set up Python ${{ inputs.python-version }} with conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: ${{ env.REPOSITORY_NAME }}-py${{ inputs.python-version }} + auto-update-conda: true + miniconda-version: "latest" + python-version: ${{ inputs.python-version }} + mamba-version: "*" + channels: conda-forge + + - name: Cache the archived conda environment + uses: actions/cache@v3 + with: + path: ~/miniconda/envs/${{ env.CONDA_ENV_NAME }} + key: + ${{ env.CONDA_ENV_NAME }}-${{ runner.os }}--${{ runner.arch }}-${{ + env.MD5_CHECKSUM }}-${{ + env.CACHE_NUMBER }} + env: + # Increase this value to reset cache if ${CONDA_ENV_NAME}.tar.gz has not changed + CACHE_NUMBER: 0 + id: cache-conda-archive + + - name: Download and install the archived conda environment + if: steps.cache-conda-archive.outputs.cache-hit != 'true' + run: | + set -vxeuo pipefail + conda activate "${{ env.REPOSITORY_NAME }}-py${{ inputs.python-version }}" + url="https://zenodo.org/record/${ZENODO_ID}/files/${CONDA_ENV_NAME}.tar.gz?download=1" + wget --progress=dot:giga ${url} -O ${CONDA_ENV_NAME}.tar.gz + status=$? + if [ $status -gt 0 ]; then + echo "Cannot download from ${url}. Exit code: ${status}" + exit $status + fi + echo "${MD5_CHECKSUM} ${CONDA_ENV_NAME}.tar.gz" > checksum.txt + md5sum --check checksum.txt + mkdir -p $HOME/miniconda/envs/${CONDA_ENV_NAME} + tar -xf ${CONDA_ENV_NAME}.tar.gz -C $HOME/miniconda/envs/${CONDA_ENV_NAME} + + - name: Activate and inspect the archived conda environment + run: | + set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + conda unpack + + conda info -a + conda env list + conda list + pip list + + - name: Install the package and its dependencies + run: | + set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + echo "Using conda environment at ${CONDA_PREFIX}" + pip install --upgrade pip setuptools wheel + pip install -r requirements-dev.txt + pip install -r requirements-extras.txt + pip install -e . + pip list + + - name: Test with pytest + run: | + set -vxeuo pipefail + conda activate $HOME/miniconda/envs/${CONDA_ENV_NAME} + echo "Using conda environment at ${CONDA_PREFIX}" + coverage run --source=csxtools run_tests.py + coverage xml + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v2 + with: + file: ./coverage.xml + flags: conda-env-unittests diff --git a/.github/workflows/conda-env-tests.yml b/.github/workflows/conda-env-tests.yml new file mode 100644 index 0000000..aa0673a --- /dev/null +++ b/.github/workflows/conda-env-tests.yml @@ -0,0 +1,29 @@ +name: Unit Tests in NSLS-II Conda Environments + +on: [push, pull_request, workflow_dispatch] + +env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + +jobs: + + matrix_prep: + + uses: ./.github/workflows/_matrix_prep.yml + with: + config_prefix: recent + + test-in-conda-env: + + needs: matrix_prep + uses: ./.github/workflows/_test-in-conda-env.yml + with: + conda_env_name: ${{ matrix.conda_env_name }} + zenodo_id: ${{ matrix.zenodo_id }} + md5_checksum: ${{ matrix.md5_checksum }} + python-version: ${{ matrix.python-version }} + strategy: + max-parallel: 4 + matrix: + include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} + fail-fast: false diff --git a/.github/workflows/legacy-conda-env-tests.yml b/.github/workflows/legacy-conda-env-tests.yml new file mode 100644 index 0000000..7831f1d --- /dev/null +++ b/.github/workflows/legacy-conda-env-tests.yml @@ -0,0 +1,29 @@ +name: Legacy Unit Tests in NSLS-II Conda Environments + +on: [workflow_dispatch] + +env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + +jobs: + + matrix_prep: + + uses: ./.github/workflows/_matrix_prep.yml + with: + config_prefix: legacy + + test-in-conda-env: + + needs: matrix_prep + uses: ./.github/workflows/_test-in-conda-env.yml + with: + conda_env_name: ${{ matrix.conda_env_name }} + zenodo_id: ${{ matrix.zenodo_id }} + md5_checksum: ${{ matrix.md5_checksum }} + python-version: ${{ matrix.python-version }} + strategy: + max-parallel: 4 + matrix: + include: ${{ fromJson(needs.matrix_prep.outputs.matrix_include) }} + fail-fast: false