Move from Miniconda to Miniforge (#868)

* Move from Miniconda to Miniforge * Move from Miniconda to Miniforge
wtsi-npg · Nov 26, 2024 · 536cea3 · 536cea3
1 parent dbbb6e8
commit 536cea3
Show file tree

Hide file tree

Showing 2 changed files with 63 additions and 17 deletions.
diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -11,9 +11,10 @@ jobs:
         shell: bash -l -e -o pipefail {0}
 
     env:
-      PERL_CACHE: ~/perl5 # Perlbrew and CPAN modules installed here, cached
-      NPG_LIB: ~/perl5npg # NPG modules installed here, not cached
-      WSI_CONDA_CHANNEL: "https://dnap.cog.sanger.ac.uk/npg/conda/devel/generic"
+      PERL_CACHE: $HOME/perl5 # Perlbrew and CPAN modules installed here, cached
+      NPG_LIB: $HOME/perl5npg # NPG modules installed here, not cached
+      WSI_CONDA_CHANNEL: https://dnap.cog.sanger.ac.uk/npg/conda/devel/generic
+      CONDA_HOME: $HOME/conda
       CONDA_TEST_ENV: test-environment
       WTSI_NPG_GITHUB_URL: https://github.com/wtsi-npg
       WTSI_NPG_BUILD_BRANCH: ${{ github.base_ref || github.ref }}
@@ -24,6 +25,8 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: '0'
 
       - name: "Install OS dependencies"
         run: |
@@ -32,22 +35,32 @@ jobs:
           sudo apt-get remove -y nginx libgd3
           sudo apt-get install -y libgd-dev uuid-dev libgd-text-perl
 
-      - name: "Initialize Miniconda"
+      - name: "Install Miniforge"
         run: |
-          echo 'source $CONDA/etc/profile.d/conda.sh' >> "$HOME/.bash_profile"
+          CONDA_HOME=${{ env.CONDA_HOME }} ./scripts/install_miniforge.sh
+          echo 'source "${{ env.CONDA_HOME }}/etc/profile.d/conda.sh"' >> "$HOME/.bash_profile"
+
+      - name: "Cache Conda"
+        id: miniforgeCache
+        uses: actions/cache@v4
+        with:
+          path: |
+            ${{ env.CONDA_HOME }}/pkgs
+            ${{ env.CONDA_HOME }}/envs
+          key: ${{ runner.os }}-build-miniforge
 
       - name: "Install Conda packages"
         run: |
-          conda config --prepend pkgs_dirs ~/conda/pkgs
-          conda config --prepend envs_dirs ~/conda/envs
+          conda config --prepend pkgs_dirs ${{ env.CONDA_HOME }}/pkgs
+          conda config --prepend envs_dirs ${{ env.CONDA_HOME }}/envs
 
           conda config --set auto_update_conda False
-          conda config --prepend channels "$WSI_CONDA_CHANNEL"
+          conda config --prepend channels "${{ env.WSI_CONDA_CHANNEL }}"
           conda info
 
-          conda create  -y -n "$CONDA_TEST_ENV"
-          conda install -y -n "$CONDA_TEST_ENV" baton
-          conda install -y -n "$CONDA_TEST_ENV" samtools
+          conda create  -y -n "${{ env.CONDA_TEST_ENV }}"
+          conda install -y -n "${{ env.CONDA_TEST_ENV }}" baton
+          conda install -y -n "${{ env.CONDA_TEST_ENV }}" samtools
 
       - name: "Cache Perl"
         id: cache-perl
@@ -77,14 +90,14 @@ jobs:
       - name: "Install Perl dependencies"
         run: |
           cpanm --local-lib=${{ env.PERL_CACHE }} local::lib
-          eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib="$NPG_LIB")
+          eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib="${{ env.NPG_LIB }}")
           eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib)
           
           cpanm --quiet --notest Module::Build
           cpanm --quiet --notest Net::SSLeay
           cpanm --quiet --notest https://github.com/chapmanb/vcftools-cpan/archive/v0.953.tar.gz
           
-          ./scripts/install_wsi_dependencies.sh "$NPG_LIB" \
+          ./scripts/install_wsi_dependencies.sh "${{ env.NPG_LIB }}" \
              perl-dnap-utilities \
              perl-irods-wrap \
              ml_warehouse \
@@ -98,23 +111,23 @@ jobs:
       - name: "Log install failure"
         if: ${{ failure() }}
         run: |
-          find ~/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20  {} \;
+          find $HOME/.cpanm/work -cmin -1 -name '*.log' -exec tail -n20  {} \;
 
       - name: "Archive CPAN logs on failure"
         if: ${{ failure() }}
         uses: actions/upload-artifact@v4
         with:
           name: cpan_log
-          path: ~/.cpanm/work/*/build.log
+          path: $HOME/.cpanm/work/*/build.log
           retention-days: 5
 
       - name: "Run tests"
         run: |
-          conda activate "$CONDA_TEST_ENV"
+          conda activate "${{ env.CONDA_TEST_ENV }}"
           conda info --envs
           
           eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib)
-          eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib="$NPG_LIB")
+          eval $(perl -I ${{ env.PERL_CACHE }}/lib/perl5/ -Mlocal::lib="${{ env.NPG_LIB }}")
           
           export TEST_AUTHOR=1
           perl Build.PL

diff --git a/scripts/install_miniforge.sh b/scripts/install_miniforge.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Once installed, the Conda environment can be activated by running:
+#
+# source $CONDA_HOME/etc/profile.d/conda.sh
+#
+# conda activate
+
+set -ex
+
+MINIFORGE_VERSION="24.9.0-0"
+MINIFORGE_SHA256="77fb505f6266ffa1b5d59604cf6ba25948627e908928cbff148813957b1c28af"
+
+CONDA_HOME=${CONDA_HOME:="$HOME/conda"}
+export CONDA_HOME
+
+CONDARC="$HOME/.condarc"
+export CONDARC
+
+cat <<EOF > "$CONDARC"
+auto_update_conda: false
+always_yes: true
+ssl_verify: true
+show_channel_urls: true
+
+channels:
+  - conda-forge
+EOF
+
+curl -sSL "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Mambaforge-${MINIFORGE_VERSION}-Linux-x86_64.sh" -o ./miniforge.sh
+sha256sum ./miniforge.sh | grep "$MINIFORGE_SHA256"
+/bin/bash ./miniforge.sh -b -p "$CONDA_HOME"
+rm ./miniforge.sh