Merge upstream

.github/workflows/scripts/test_init_scripts.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+EESSI_VERSION="2023.06"
+export LMOD_PAGER=cat
+
+# initialize assert framework
+if [ ! -d assert.sh ]; then
+	echo "assert.sh not cloned."
+	echo ""
+	echo "run \`git clone https://github.com/lehmannro/assert.sh.git\`"
+	exit 1
+fi
+. assert.sh/assert.sh
+
+TEST_SHELLS=("bash" "zsh" "fish" "ksh")
+SHELLS=$@
+
+for shell in ${SHELLS[@]}; do
+	echo = | awk 'NF += (OFS = $_) + 100'
+	echo  RUNNING TESTS FOR SHELL: $shell
+	echo = | awk 'NF += (OFS = $_) + 100'
+  if [[ ! " ${TEST_SHELLS[*]} " =~ [[:space:]]${shell}[[:space:]] ]]; then
+		### EXCEPTION FOR CSH ###
+		echo -e "\033[33mWe don't now how to test the shell '$shell', PRs are Welcome.\033[0m" 
+  else
+		# TEST 1: Source Script and check Module Output
+		assert "$shell -c 'source init/lmod/$shell' 2>&1 " "EESSI/$EESSI_VERSION loaded successfully"
+		# TEST 2: Check if module overviews first section is the loaded EESSI module
+		MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'"))
+		PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all"
+		assert_raises 'echo "${MODULE_SECTIONS[1]}" | grep -E "$PATTERN"'
+		# TEST 3: Check if module overviews second section is the EESSI init module
+		assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules"
+		# Test 4: Load Python module and check version
+		command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; python --version'"
+		expected="Python 3.10.8"
+		assert "$command" "$expected"
+		# Test 5: Load Python module and check path
+		PYTHON_PATH=$($shell -c "source init/lmod/$shell 2>/dev/null; module load Python/3.10.8-GCCcore-12.2.0; which python")
+		PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/software/Python/3\.10\.8-GCCcore-12\.2\.0/bin/python"
+		echo "$PYTHON_PATH" | grep -E "$PATTERN"
+		assert_raises 'echo "$PYTHON_PATH" | grep -E "$PATTERN"'
+
+		#End Test Suite
+		assert_end "source_eessi_$shell"
+	fi
+done
+
+
+# RESET PAGER
+export LMOD_PAGER=

.github/workflows/tests_eessi_module.yml

@@ -0,0 +1,85 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests for eessi_module_functionality in software.eessi.io
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        EESSI_VERSION:
+          - 2023.06        
+    steps:
+      - name: Check out software-layer repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Mount EESSI CernVM-FS pilot repository
+        uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
+        with:
+          cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
+          cvmfs_http_proxy: DIRECT
+          cvmfs_repositories: software.eessi.io
+
+      - name: Test for making sure spider cache is being used and not being rebuilt
+        run: |
+          . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash  # Initialise Lmod
+          export MODULEPATH=init/modules
+          configfile="configfile.txt"
+          module -T load EESSI/${{matrix.EESSI_VERSION}}
+          module --config > "${configfile}" 2>&1
+          grep cache "${configfile}" | grep software | grep -v compat
+          if timeout 10s bash -c "LMOD_PAGER=none module --terse avail" && grep cache "${configfile}" | grep software | grep -v compat; then
+              echo "EESSI spider cache is being used"
+          else
+              echo "EESSI spider cache is being rebuilt" >&2
+              exit 1
+          fi
+          env | grep LMOD
+          module purge
+          unset MODULEPATH
+    
+      - name: Test for archdetect_cpu functionality with invalid path
+        run: |
+          . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash  # Initialise Lmod
+          export MODULEPATH=init/modules
+          set +e  # Do not exit immediately if a command exits with a non-zero status          
+          export EESSI_ARCHDETECT_OPTIONS="dummy/cpu"
+          outfile="outfile.txt"
+          module load EESSI/${{matrix.EESSI_VERSION}} > "${outfile}" 2>&1
+          cat "${outfile}"
+          if grep -q  "Software directory check" "${outfile}"; then
+              echo "Test for picking up invalid path on \${archdetect_cpu} PASSED"
+          else
+              echo "Test for picking up invalid path on \${archdetect_cpu} FAILED" >&2
+              exit 1
+          fi
+          unset EESSI_ARCHDETECT_OPTIONS
+          set -e  # Re-enable exit on non-zero status
+              
+      - name: Test for expected variables while adding dummy cpu archs and loading EESSI module 
+        run: |
+          . /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/compat/linux/$(uname -m)/usr/share/Lmod/init/bash  # Initialise Lmod
+          export MODULEPATH=init/modules
+          CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath)
+          export EESSI_ARCHDETECT_OPTIONS="dummy/cpu:${CPU_ARCH}:dummy1/cpu1"          
+          moduleoutfile="moduleout.txt"
+          sourceoutfile="sourceout.txt"
+          module load EESSI/${{matrix.EESSI_VERSION}}
+          env | grep -E '^(EESSI_S|EESSI_C)' | sort > "${moduleoutfile}"
+          module unload EESSI/${{matrix.EESSI_VERSION}}
+          source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
+          env | grep -E '^(EESSI_S|EESSI_C)' | sort > "${sourceoutfile}"
+          cat "${moduleoutfile}"
+          cat "${sourceoutfile}"
+          if (diff "${moduleoutfile}" "${sourceoutfile}" > /dev/null); then
+              echo "Test for checking env variables PASSED"
+          else 
+              echo "Test for checking env variables FAILED"  >&2
+              exit 1
+          fi
+

.github/workflows/tests_init_module.yml

@@ -0,0 +1,43 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Check for EESSI init shell scripts to load eessi software module in software.eessi.io
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
+  workflow_dispatch:
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        EESSI_VERSION:
+          - 2023.06
+        EESSI_SOFTWARE_SUBDIR_OVERRIDE:
+          - x86_64/intel/haswell
+    steps:
+      - name: Check out software-layer repository
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Mount EESSI CernVM-FS pilot repository
+        uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
+        with:
+          cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
+          cvmfs_http_proxy: DIRECT
+          cvmfs_repositories: software.eessi.io
+
+      - name: Clone assert.sh script
+        run: git clone https://github.com/lehmannro/assert.sh.git
+
+      - name: Install missing shells
+        run: |
+          sudo apt update
+          sudo apt install zsh ksh fish
+          echo "# INIT ZSH" > ~/.zshrc
+
+      - name: Run tests for available shells
+        run: |
+          .github/workflows/scripts/test_init_scripts.sh "bash" "zsh" "ksh" "fish" "csh"
+

create_lmodsitepackage.py

@@ -28,7 +28,7 @@
     -- If EESSI_PREFIX wasn't defined, we cannot check if this module was from the EESSI environment
     -- In that case, we assume it isn't, otherwise EESSI_PREFIX would (probably) have been set
     if eessi_prefix == nil then
-        return False
+        return false
     else
         -- NOTE: exact paths for site so may need to be updated later.
         -- See https://github.com/EESSI/software-layer/pull/371

create_tarball.sh

@@ -35,6 +35,7 @@ if [ ! -d ${software_dir_overlay} ]; then
     exit 3
 fi
 
+current_workdir=${PWD}
 cd ${overlay_upper_dir}/versions/
 echo ">> Collecting list of files/directories to include in tarball via ${PWD}..."
 
@@ -88,6 +89,9 @@ echo "wrote file list to ${files_list}"
 echo "wrote module file list to ${module_files_list}"
 [ -r ${module_files_list} ] && cat ${module_files_list}
 
+# Copy the module files list to current workindg dir for later use in the test step
+cp ${module_files_list} ${current_workdir}/module_files.list.txt
+
 topdir=${cvmfs_repo}/versions/
 
 echo ">> Creating tarball ${target_tgz} from ${topdir}..."

easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml

@@ -4,3 +4,23 @@ easyconfigs:
   - SciPy-bundle-2023.07-gfbf-2023a.eb
   - ESPResSo-4.2.2-foss-2023a.eb
   - ParaView-5.11.2-foss-2023a.eb
+  - OpenFOAM-10-foss-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958
+        from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f
+  - OpenFOAM-11-foss-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958
+        from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f
+  - OpenFOAM-v2312-foss-2023a.eb:
+      options:
+        # https://github.com/easybuilders/easybuild-easyblocks/pull/3388
+        include-easyblocks-from-commit: c8256a36e7062bc09f5ce30552a9de9827054c9e
+        # https://github.com/easybuilders/easybuild-easyconfigs/pull/20841
+        from-commit: f0e91e6e430ebf902f7788ebb47f0203dee60649
+  - R-4.3.2-gfbf-2023a.eb
+  - Highway-1.0.4-GCCcore-12.3.0.eb
+  - Brunsli-0.1-GCCcore-12.3.0.eb:
+      options:
+        # https://github.com/easybuilders/easybuild-easyconfigs/pull/21366
+        from-commit: 1736a123b1685836452587a5c51793257570bb2d

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml

@@ -0,0 +1,11 @@
+easyconfigs:
+  - EasyBuild-4.9.3.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21412
+        from-commit: 1cdd81524c974a29825e37bcf8ef3ccc291f5227
+  - ReFrame-4.6.2.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21307
+        from-commit: 0c4bd5c5a80f571a8932fbc38880d72455406816
+        # see https://github.com/easybuilders/easybuild-easyblocks/pull/3431
+        include-easyblocks-from-commit: efddeb02abe1a679324ac01ef19601dedbe79cc0

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml

@@ -6,4 +6,4 @@ easyconfigs:
         from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc
   - gnuplot-5.4.6-GCCcore-12.2.0.eb
   - h5py-3.8.0-foss-2022b.eb
-  - MDAnalysis-2.4.2-foss-2022b.eb 
+  - MDAnalysis-2.4.2-foss-2022b.eb

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml

@@ -45,5 +45,30 @@ easyconfigs:
       options:
         # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20880
         from-commit: bc6e08f89759b8b70166de5bfcb5056b9db8ec90
+  - wradlib-2.0.3-foss-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21094
+        from-commit: 3a2e0b8e6ee45277d01fb7e2eb93027a28c9461a
+  - MBX-1.1.0-foss-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21155
+        from-commit: 6929a67401f2a2ec58f91fb306332a77497d73ff
+  - Transrate-1.0.3-GCC-12.3.0.eb:
+      options:
+        # https://github.com/easybuilders/easybuild-easyblocks/pull/3381
+        include-easyblocks-from-commit: bb86f05d4917b29e022023f152efdf0ca5c14ded
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20964
+        from-commit: 7d539a9e599d8bc5ac2bda6ee9587ef62351ee03
+  - Critic2-1.2-foss-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20833
+        from-commit: 78426c2383fc7e4b9b9e77d7a77f336e1bee3843
+  - LRBinner-0.1-foss-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21310
+        from-commit: 799d9101df2cf81aabe252f00cc82a7246363f53
+  - Redland-1.0.17-GCC-12.3.0.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21227
+        from-commit: 4c5e3455dec31e68e8383c7fd86d1f80c434676d
   - PyTorch-bundle-2.1.2-foss-2023a.eb
-

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml

@@ -15,3 +15,16 @@ easyconfigs:
       options:
         # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034
         from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2
+  - STAR-2.7.11b-GCC-13.2.0.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21200
+        from-commit: 765ba900daf5953e306c4dad896febe52fdd6c00
+  - HPL-2.3-foss-2023b.eb
+  - Brunsli-0.1-GCCcore-13.2.0.eb:
+      options:
+        # https://github.com/easybuilders/easybuild-easyconfigs/pull/21366
+        from-commit: 1736a123b1685836452587a5c51793257570bb2d
+  - R-bundle-CRAN-2024.06-foss-2023b.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21285
+        from-commit: 41a2cd83f9fb017b76f0693f6a264d8acb548317

easystacks/software.eessi.io/2023.06/rebuilds/20240814-eb-4.9.2-hatchling-1.18.0-updated-easyconfig.yml

@@ -0,0 +1,9 @@
+# 2024.08.14
+# hatchling-1.18.0 rebuild to account for easyconfig changed upstream 
+# see https://gitlab.com/eessi/support/-/issues/85 and
+# https://github.com/easybuilders/easybuild-easyconfigs/pull/20389
+easyconfigs:
+  - hatchling-1.18.0-GCCcore-12.3.0.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20389
+        from-commit: 9580c0d67d6dd97b160b768a839bfcba6d5b21b9

easystacks/software.eessi.io/2023.06/rebuilds/20240823-eb-4.9.2-GObject-Introspection-filter-envvars-a64fx.yml

@@ -0,0 +1,13 @@
+# 2024.08.23
+# GObject-Introspection sets $LD_LIBRARY_PATH (to many different paths, including $EPREFIX/lib)
+# when calling gcc, and this causes a lot of issues for, especially, scripts using /bin/bash.
+#
+# This rebuild ensures (by using a new EasyBuild hook) that GObject-Introspection will not set
+# environment variables that are configured to be filtered by EasyBuild.
+# This rebuild was not done initially for A64FX. This file is meant to do the same as the
+# previous rebuild of GObject-Introspection-1.76.1-GCCcore-12.3.0 in other architectures, 
+# but for A64FX.
+# 
+# See https://github.com/EESSI/software-layer/issues/196
+easyconfigs:
+  - GObject-Introspection-1.76.1-GCCcore-12.3.0.eb

easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-001-system.yml

@@ -0,0 +1,2 @@
+easyconfigs:
+    - ReFrame-4.3.3.eb

easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml

@@ -22,3 +22,4 @@ easyconfigs:
         # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21136
         from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc
   - MODFLOW-6.4.4-foss-2023a.eb
+  - ALL-0.9.2-foss-2023a.eb

eb_hooks.py

@@ -83,7 +83,7 @@ def post_ready_hook(self, *args, **kwargs):
     # 'parallel' easyconfig parameter is set via EasyBlock.set_parallel in ready step based on available cores.
     # here we reduce parallellism to only use half of that for selected software,
     # to avoid failing builds/tests due to out-of-memory problems;
-    memory_hungry_build = self.name in ['libxc', 'TensorFlow']
+    memory_hungry_build = self.name in ['libxc', 'MBX', 'TensorFlow']
     # on A64FX systems, (HBM) memory is typically scarce, so we need to use fewer cores for some builds
     cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
     memory_hungry_build_a64fx = cpu_target == CPU_TARGET_A64FX and self.name in ['Qt5']
@@ -333,7 +333,7 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg
     Solve issues with compiling or running the tests on both
     neoverse_n1 and neoverse_v1 with Highway 1.0.4 and GCC 12.3.0:
       - for neoverse_n1 we set optarch to GENERIC
-      - for neoverse_v1 we completely disable the tests
+      - for neoverse_v1 and a64fx we completely disable the tests
     cfr. https://github.com/EESSI/software-layer/issues/469
     """
     if self.name == 'Highway':
@@ -342,7 +342,7 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg
         # note: keep condition in sync with the one used in 
         # post_prepare_hook_highway_handle_test_compilation_issues
         if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0':
-            if cpu_target == CPU_TARGET_NEOVERSE_V1:
+            if cpu_target in [CPU_TARGET_A64FX, CPU_TARGET_NEOVERSE_V1]:
                 self.cfg.update('configopts', '-DHWY_ENABLE_TESTS=OFF')
             if cpu_target == CPU_TARGET_NEOVERSE_N1:
                 self.orig_optarch = build_option('optarch')