Merge branch '2023.06-software.eessi.io' of github-trz:EESSI/software…

…-layer into test_zen4_support_on_azure

.github/workflows/scorecards.yml

@@ -40,7 +40,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6
+        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
         with:
           results_file: results.sarif
           results_format: sarif

.github/workflows/tests_scripts.yml

@@ -3,9 +3,9 @@ name: Tests for scripts
 on:
   push:
     paths:
-      - build_container.sh
       - create_directory_tarballs.sh
       - create_lmodsitepackage.py
+      - eessi_container.sh
       - EESSI-install-software.sh
       - install_software_layer.sh
       - load_easybuild_module.sh
@@ -15,9 +15,9 @@ on:
 
   pull_request:
     paths:
-      - build_container.sh
       - create_directory_tarballs.sh
       - create_lmodsitepackage.py
+      - eessi_container.sh
       - EESSI-install-software.sh
       - install_software_layer.sh
       - load_easybuild_module.sh

create_lmodsitepackage.py

@@ -20,6 +20,28 @@
     return content
 end
 
+local function from_eessi_prefix(t)
+    -- eessi_prefix is the prefix with official EESSI modules
+    -- e.g. /cvmfs/software.eessi.io/versions/2023.06
+    local eessi_prefix = os.getenv("EESSI_PREFIX")
+
+    -- If EESSI_PREFIX wasn't defined, we cannot check if this module was from the EESSI environment
+    -- In that case, we assume it isn't, otherwise EESSI_PREFIX would (probably) have been set
+    if eessi_prefix == nil then
+        return False
+    else
+        -- NOTE: exact paths for site so may need to be updated later.
+        -- See https://github.com/EESSI/software-layer/pull/371
+
+        -- eessi_prefix_host_injections is the prefix with site-extensions (i.e. additional modules)
+        -- to the official EESSI modules, e.g. /cvmfs/software.eessi.io/host_injections/2023.06
+        local eessi_prefix_host_injections = string.gsub(eessi_prefix, 'versions', 'host_injections')
+        
+       -- Check if the full modulepath starts with the eessi_prefix_*
+        return string.find(t.fn, "^" .. eessi_prefix) ~= nil or string.find(t.fn, "^" .. eessi_prefix_host_injections) ~= nil
+    end
+end
+
 local function load_site_specific_hooks()
     -- This function will be run after the EESSI hooks are registered
     -- It will load a local SitePackage.lua that is architecture independent (if it exists) from e.g.
@@ -153,10 +175,13 @@
 -- Combine both functions into a single one, as we can only register one function as load hook in lmod
 -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed
 function eessi_load_hook(t)
-    eessi_cuda_enabled_load_hook(t)
+    -- Only apply CUDA hooks if the loaded module is in the EESSI prefix
+    -- This avoids getting an Lmod Error when trying to load a CUDA module from a local software stack
+    if from_eessi_prefix(t) then
+        eessi_cuda_enabled_load_hook(t)
+    end
 end
 
-
 hook.register("load", eessi_load_hook)
 
 -- Note that this needs to happen at the end, so that any EESSI specific hooks can be overwritten by the site

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml

@@ -35,7 +35,6 @@ easyconfigs:
         from-pr: 19996
   - dask-2023.9.2-foss-2023a.eb
   - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb
-  - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb
   - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb
   - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb:
       options:

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml

@@ -57,3 +57,4 @@ easyconfigs:
   - Qt5-5.15.13-GCCcore-13.2.0.eb:
       options:
         from-pr: 20201
+  - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml

@@ -0,0 +1,4 @@
+easyconfigs:
+  - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb:
+      options:
+        from-pr: 20379

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml

@@ -1,2 +1,6 @@
 easyconfigs:
   - ncdu-1.18-GCC-12.3.0.eb
+  - SAMtools-1.18-GCC-12.3.0.eb
+  - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb:
+      options:
+        from-pr: 20379

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml

@@ -0,0 +1,4 @@
+easyconfigs:
+  - GROMACS-2024.1-foss-2023b.eb:
+      options:
+        from-pr: 20439

easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml

@@ -0,0 +1,9 @@
+# 2024.05.06
+# Original matching of files we could ship was not done correctly. We were
+# matching the basename for files (e.g., libcudart.so from libcudart.so.12)
+# rather than the name stub (libcudart)
+# See https://github.com/EESSI/software-layer/pull/559
+easyconfigs:
+  - CUDA-12.1.1.eb:
+      options:
+        accept-eula-for: CUDA

easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml

@@ -0,0 +1,13 @@
+# 2024-04-19
+# Move setuptools_scm extension from hatchling to Python by rebuilding
+# all affected modules with EasyBuild 4.9.1.
+# This solves an issue with pyarrow, which is part of the Arrow installation.
+# https://github.com/easybuilders/easybuild-easyconfigs/pull/19777
+# https://github.com/easybuilders/easybuild-easyconfigs/issues/19849
+easyconfigs:
+  - hatchling-1.18.0-GCCcore-12.3.0.eb
+  - hatchling-1.18.0-GCCcore-13.2.0.eb
+  - Python-bundle-PyPI-2023.06-GCCcore-12.3.0.eb
+  - Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb
+  - Python-3.11.3-GCCcore-12.3.0.eb
+  - Python-3.11.5-GCCcore-13.2.0.eb

eb_hooks.py

@@ -335,6 +335,21 @@ def pre_configure_hook(self, *args, **kwargs):
         PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs)
 
 
+def pre_configure_hook_gromacs(self, *args, **kwargs):
+    """
+    Pre-configure hook for GROMACS:
+    - avoid building with SVE instructions on Neoverse V1 as workaround for failing tests,
+      see https://gitlab.com/gromacs/gromacs/-/issues/5057 + https://gitlab.com/eessi/support/-/issues/47
+    """
+    if self.name == 'GROMACS':
+        cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
+        if LooseVersion(self.version) <= LooseVersion('2024.1') and cpu_target == CPU_TARGET_NEOVERSE_V1:
+            self.cfg.update('configopts', '-DGMX_SIMD=ARM_NEON_ASIMD')
+            print_msg("Avoiding use of SVE instructions for GROMACS %s by using ARM_NEON_ASIMD as GMX_SIMD value", self.version)
+    else:
+        raise EasyBuildError("GROMACS-specific hook triggered for non-GROMACS easyconfig?!")
+
+
 def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs):
     """
     Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC
@@ -597,8 +612,8 @@ def post_sanitycheck_cuda(self, *args, **kwargs):
                 full_path = os.path.join(dir_path, filename)
                 # we only really care about real files, i.e. not symlinks
                 if not os.path.islink(full_path):
-                    # check if the current file is part of the allowlist
-                    basename = os.path.splitext(filename)[0]
+                    # check if the current file name stub is part of the allowlist
+                    basename = filename.split('.')[0]
                     if basename in allowlist:
                         self.log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path)
                     else:
@@ -665,6 +680,7 @@ def inject_gpu_property(ec):
 }
 
 PRE_CONFIGURE_HOOKS = {
+    'GROMACS': pre_configure_hook_gromacs,
     'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic,
     'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep,
     'OpenBLAS': pre_configure_hook_openblas_optarch_generic,

eessi-2023.06-known-issues.yml

@@ -19,6 +19,9 @@
   - FFTW.MPI-3.3.10-gompi-2023b:
       - issue: https://github.com/EESSI/software-layer/issues/325
       - info: "Flaky FFTW tests, random failures"
+  - GROMACS-2024.1-foss-2023b:
+      - issue: https://github.com/EESSI/software-layer/issues/557
+      - info: "SVE disabled due to known bug which causes test failures"
   - Highway-1.0.3-GCCcore-12.2.0.eb:
     - issue: https://github.com/EESSI/software-layer/issues/469
     - info: "failing SVE test due to wrong expected value"

init/arch_specs/eessi_arch_x86.spec

@@ -1,6 +1,6 @@
 # x86_64 CPU architecture specifications
 # Software path in EESSI 	| Vendor ID 	| List of defining CPU features
-"x86_64/intel/haswell"		"GenuineIntel"	"avx2 fma"						# Intel Haswell, Broadwell
+"x86_64/intel/haswell"		"GenuineIntel"	"avx2 fma" 						# Intel Haswell, Broadwell
 "x86_64/intel/skylake_avx512"	"GenuineIntel"	"avx2 fma avx512f avx512bw avx512cd avx512dq avx512vl"	# Intel Skylake, Cascade Lake
 "x86_64/amd/zen2"		"AuthenticAMD"	"avx2 fma"						# AMD Rome
 "x86_64/amd/zen3"		"AuthenticAMD"	"avx2 fma vaes"						# AMD Milan, Milan-X