Merge branch '2023.06-software.eessi.io' into espresso-lmodwarn

.github/workflows/scorecards.yml

@@ -40,7 +40,7 @@ jobs:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6
+        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
         with:
           results_file: results.sarif
           results_format: sarif

.github/workflows/tests_archdetect.yml

@@ -13,6 +13,8 @@ jobs:
           - x86_64/intel/skylake_avx512/archspec-linux-6132
           - x86_64/amd/zen2/Azure-CentOS7-7V12
           - x86_64/amd/zen3/Azure-CentOS7-7V73X
+          - x86_64/amd/zen4/Azure-Alma8-9V33X
+          - x86_64/amd/zen4/Shinx-RHEL8-9654
           - aarch64/neoverse_n1/Azure-Ubuntu20-Altra
           - aarch64/neoverse_n1/AWS-awslinux-graviton2
           - aarch64/neoverse_v1/AWS-awslinux-graviton3

EESSI-extend-2023.06-easybuild.eb

@@ -0,0 +1,167 @@
+easyblock = 'Bundle'
+
+name = 'EESSI-extend'
+version = '2023.06'
+# May have different ways to extend EESSI in future (manually, other tools,...)
+versionsuffix = '-easybuild'
+
+homepage = 'https://eessi.io/docs/'
+
+description = """
+ The goal of the European Environment for Scientific Software Installations
+ (EESSI, pronounced as "easy") is to build a common stack of scientific
+ software installations for HPC systems and beyond, including laptops,
+ personal workstations and cloud infrastructure.
+
+ This module allows you to extend EESSI using the same configuration for
+ EasyBuild as EESSI itself uses. A number of environment variables control the
+ behaviour of the module:
+ - EESSI_USER_INSTALL can be set to a location to install modules for use by
+   the user only. The location must already exist on the filesystem.
+ - EESSI_PROJECT_INSTALL can be set to a location to install modules for use by
+   a project. The location must already exist on the filesystem and you should
+   ensure that the location has the correct Linux group and the SGID permission
+   is set on that directory (`chmod g+s $EESSI_PROJECT_INSTALL`) so that all
+   members of the group have permission to read and write installations.
+ - EESSI_SITE_INSTALL is either defined or not and cannot be used with another
+   environment variable. A site installation is done in a defined location and
+   any installations there are (by default) world readable.
+ - EESSI_CVMFS_INSTALL is either defined or not and cannot be used with another
+   environment variable. A CVMFS installation targets a defined location which
+   will be ingested into CVMFS and is only useful for CVMFS administrators.
+ - If none of the environment variables above are defined, an EESSI_USER_INSTALL
+   is assumed with a value of $HOME/EESSI
+ If both EESSI_USER_INSTALL and EESSI_PROJECT_INSTALL are defined, both sets of
+ installations are exposed, but new installations are created as user
+ installations.
+"""
+
+toolchain = SYSTEM
+
+# All the dependencies we filter in EESSI
+local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,M4,makeinfo,ncurses,util-linux,XZ,zlib"
+local_arch_specific_deps_to_filter = {'aarch64': ',yasm', 'x86_64': ''}
+local_deps_to_filter += local_arch_specific_deps_to_filter[ARCH]
+
+# Set the universal EasyBuild variables
+modextravars = {
+    'EASYBUILD_FILTER_DEPS': local_deps_to_filter,
+    'EASYBUILD_IGNORE_OSDEPS': '1',
+    'EASYBUILD_DEBUG': '1',
+    'EASYBUILD_TRACE': '1',
+    'EASYBUILD_ZIP_LOGS': 'bzip2',
+    'EASYBUILD_RPATH': '1',
+    'EASYBUILD_FILTER_ENV_VARS': 'LD_LIBRARY_PATH',
+    'EASYBUILD_READ_ONLY_INSTALLDIR': '1',
+    'EASYBUILD_MODULE_EXTENSIONS': '1',
+    'EASYBUILD_EXPERIMENTAL': '1',
+}
+
+# Need a few other variables, but they are more dynamic
+# EASYBUILD_SYSROOT=${EPREFIX}
+# EASYBUILD_PREFIX=${WORKDIR}/easybuild
+# EASYBUILD_HOOKS=${EESSI_PREFIX}/init/easybuild/eb_hooks.py
+# EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR}
+# EASYBUILD_SOURCEPATH=${WORKDIR}/easybuild/sources:${EESSI_SOURCEPATH}
+#
+# And also some optional ones based on the kind of installation
+# EASYBUILD_SET_GID_BIT
+# EASYBUILD_GROUP_WRITABLE_INSTALLDIR
+# EASYBUILD_UMASK
+# EASYBUILD_STICKY_BIT
+modluafooter = """
+if (mode() == "load") then
+  -- Use a working directory for temporary build files
+  if (os.getenv("WORKING_DIR") == nil) then
+    LmodMessage("-- Using /tmp/$USER as a temporary working directory for installations, you can override this by setting the environment variable WORKING_DIR and reloading the module (e.g., /dev/shm is a common option)")
+  end
+end
+working_dir = os.getenv("WORKING_DIR") or pathJoin("/tmp", os.getenv("USER"))
+-- Gather the EPREFIX to use as a sysroot
+sysroot = os.getenv("EESSI_EPREFIX")
+-- Use an installation prefix that we _should_ have write access to
+if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then
+  -- Make sure no other EESSI install environment variables are set
+  if ((os.getenv("EESSI_SITE_INSTALL") ~= nil) or (os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then
+    LmodError("You cannot use EESSI_CVMFS_INSTALL in combination with any other EESSI_*_INSTALL environment variables")
+  end
+  eessi_cvmfs_install = true
+  easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH")
+elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then
+  -- Make sure no other EESSI install environment variables are set
+  if ((os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then
+    LmodError("You cannot use EESSI_SITE_INSTALL in combination with any other EESSI_*_INSTALL environment variables")
+  end
+  easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), 'versions', 'host_injections')
+else
+  -- Deal with user and project installs
+  project_install = os.getenv("EESSI_PROJECT_INSTALL")
+  project_modulepath = nil
+  if (project_install ~= nil) then
+    -- Check the folder exists
+    if not isDir(project_install) then
+      LmodError("The location of EESSI_PROJECT_INSTALL (" .. project_install .. ") does not exist or is not a folder")
+    end
+    if (mode() == "load") then
+      LmodMessage("Configuring for use of EESSI_PROJECT_INSTALL under " .. project_install)
+    end
+    easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), project_install)
+    project_modulepath = pathJoin(easybuild_installpath, 'modules', 'all')
+  end
+  user_install = os.getenv("EESSI_USER_INSTALL")
+  user_modulepath = nil
+  if (user_install ~= nil) then
+    -- Check the folder exists
+    if not isDir(user_install) then
+      LmodError("The location of EESSI_USER_INSTALL (" .. user_install .. ") does not exist or is not a folder")
+    end
+  elseif (user_install == nil) and (project_install == nil) then
+    -- No need to check for existence when we use a HOME subdir
+    user_install = pathJoin(os.getenv("HOME"), "eessi")
+  end
+  if (user_install ~= nil) then
+    if (mode() == "load") then
+      LmodMessage("Configuring for use of EESSI_USER_INSTALL under " .. user_install)
+    end
+    easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), user_install)
+    user_modulepath = pathJoin(easybuild_installpath, 'modules', 'all')
+  end
+end
+if (mode() == "load") then
+  LmodMessage("-- To create installations for EESSI, you _must_ have write permissions to " .. easybuild_installpath)
+  -- Advise them to reuse sources
+  if (os.getenv("EASYBUILD_SOURCEPATH") == nil) then
+    LmodMessage("-- You may wish to configure a sources directory for EasyBuild (for example, via setting the environment variable EASYBUILD_SOURCEPATH) to allow you to reuse existing sources for packages.")
+  end
+end
+-- Set the relevant universal environment variables for EasyBuild
+setenv ("EASYBUILD_SYSROOT", sysroot)
+setenv ("EASYBUILD_PREFIX", pathJoin(working_dir, "easybuild"))
+setenv ("EASYBUILD_INSTALLPATH", easybuild_installpath)
+setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuild", "eb_hooks.py"))
+setenv ("EASYBUILD_UMASK", "002")
+
+-- Set all related environment variables if we have project or user installations (including extending MODULEPATH)
+if (user_modulepath ~= nil) then
+  -- Use a more restrictive umask for this case
+  setenv ("EASYBUILD_UMASK", "022")
+  setenv ("EASYBUILD_STICKY_BIT", "1")
+  -- configure MODULEPATH
+  if (project_modulepath ~= nil) then
+    prepend_path("MODULEPATH", project_modulepath)
+  end
+  prepend_path("MODULEPATH", user_modulepath)
+elseif (project_modulepath ~= nil) then
+  setenv ("EASYBUILD_SET_GID_BIT", "1")
+  setenv ("EASYBUILD_GROUP_WRITABLE_INSTALLDIR", "1")
+  setenv ("EASYBUILD_STICKY_BIT", "0")
+  -- configure MODULEPATH
+  prepend_path("MODULEPATH", project_modulepath)
+end
+-- Make sure EasyBuild itself is loaded
+if not ( isloaded("EasyBuild") ) then
+    load("EasyBuild")
+end
+"""
+
+moduleclass = 'devel'

EESSI-install-software.sh

@@ -147,6 +147,24 @@ else
   mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
 fi
 
+# if we run the script for the first time, e.g., to start building for a new
+#   stack, we need to ensure certain files are present in
+#   ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+# - .lmod/lmodrc.lua
+# - .lmod/SitePackage.lua
+_eessi_software_path=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+_lmod_cfg_dir=${_eessi_software_path}/.lmod
+_lmod_rc_file=${_lmod_cfg_dir}/lmodrc.lua
+if [ ! -f ${_lmod_rc_file} ]; then
+    command -V python3
+    python3 ${TOPDIR}/create_lmodrc.py ${_eessi_software_path}
+fi
+_lmod_sitepackage_file=${_lmod_cfg_dir}/SitePackage.lua
+if [ ! -f ${_lmod_sitepackage_file} ]; then
+    command -V python3
+    python3 ${TOPDIR}/create_lmodsitepackage.py ${_eessi_software_path}
+fi
+
 # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
 # $EESSI_SILENT - don't print any messages
 # $EESSI_BASIC_ENV - give a basic set of environment variables
@@ -203,10 +221,21 @@ ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX}
 # Hardcode this for now, see if it works
 # TODO: We should make a nice yaml and loop over all CUDA versions in that yaml to figure out what to install
 # Allow skipping CUDA SDK install in e.g. CI environments
+# The install_cuda... script uses EasyBuild. So, we need to check if we have EB
+# or skip this step.
+module_avail_out=$TMPDIR/ml.out
+module avail 2>&1 | grep EasyBuild &> ${module_avail_out}
+if [[ $? -eq 0 ]]; then
+    echo_green ">> Found an EasyBuild module"
+else
+    echo_yellow ">> No EasyBuild module found: skipping step to install CUDA (see output in ${module_avail_out})"
+    export skip_cuda_install=True
+fi
+
 if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then
     ${EESSI_PREFIX}/scripts/gpu_support/nvidia/install_cuda_host_injections.sh -c 12.1.1 --accept-cuda-eula
 else
-    echo "Skipping installation of CUDA SDK in host_injections, since the --skip-cuda-install flag was passed"
+    echo "Skipping installation of CUDA SDK in host_injections, since the --skip-cuda-install flag was passed OR no EasyBuild module was found"
 fi
 
 # Install drivers in host_injections

create_lmodsitepackage.py

@@ -20,6 +20,28 @@
     return content
 end
 
+local function from_eessi_prefix(t)
+    -- eessi_prefix is the prefix with official EESSI modules
+    -- e.g. /cvmfs/software.eessi.io/versions/2023.06
+    local eessi_prefix = os.getenv("EESSI_PREFIX")
+
+    -- If EESSI_PREFIX wasn't defined, we cannot check if this module was from the EESSI environment
+    -- In that case, we assume it isn't, otherwise EESSI_PREFIX would (probably) have been set
+    if eessi_prefix == nil then
+        return False
+    else
+        -- NOTE: exact paths for site so may need to be updated later.
+        -- See https://github.com/EESSI/software-layer/pull/371
+
+        -- eessi_prefix_host_injections is the prefix with site-extensions (i.e. additional modules)
+        -- to the official EESSI modules, e.g. /cvmfs/software.eessi.io/host_injections/2023.06
+        local eessi_prefix_host_injections = string.gsub(eessi_prefix, 'versions', 'host_injections')
+        
+       -- Check if the full modulepath starts with the eessi_prefix_*
+        return string.find(t.fn, "^" .. eessi_prefix) ~= nil or string.find(t.fn, "^" .. eessi_prefix_host_injections) ~= nil
+    end
+end
+
 local function load_site_specific_hooks()
     -- This function will be run after the EESSI hooks are registered
     -- It will load a local SitePackage.lua that is architecture independent (if it exists) from e.g.
@@ -166,11 +188,14 @@
 -- Combine both functions into a single one, as we can only register one function as load hook in lmod
 -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed
 function eessi_load_hook(t)
-    eessi_cuda_enabled_load_hook(t)
     eessi_espresso_deprecated_warning(t)
+    -- Only apply CUDA hooks if the loaded module is in the EESSI prefix
+    -- This avoids getting an Lmod Error when trying to load a CUDA module from a local software stack
+    if from_eessi_prefix(t) then
+        eessi_cuda_enabled_load_hook(t)
+    end
 end
 
-
 hook.register("load", eessi_load_hook)
 
 -- Note that this needs to happen at the end, so that any EESSI specific hooks can be overwritten by the site

create_tarball.sh

@@ -28,7 +28,7 @@ fi
 
 overlay_upper_dir="${eessi_tmpdir}/overlay-upper"
 
-software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}"
+software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}"
 if [ ! -d ${software_dir_overlay} ]; then
     echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2
     exit 3

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml

@@ -35,7 +35,6 @@ easyconfigs:
         from-pr: 19996
   - dask-2023.9.2-foss-2023a.eb
   - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb
-  - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb
   - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb
   - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb:
       options:

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml

@@ -57,3 +57,4 @@ easyconfigs:
   - Qt5-5.15.13-GCCcore-13.2.0.eb:
       options:
         from-pr: 20201
+  - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml

@@ -2,3 +2,4 @@ easyconfigs:
   - EasyBuild-4.9.1.eb:
       options:
         from-pr: 20299
+  - EESSI-extend-2023.06-easybuild.eb

easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml

@@ -0,0 +1,9 @@
+# 2024.05.06
+# Original matching of files we could ship was not done correctly. We were
+# matching the basename for files (e.g., libcudart.so from libcudart.so.12)
+# rather than the name stub (libcudart)
+# See https://github.com/EESSI/software-layer/pull/559
+easyconfigs:
+  - CUDA-12.1.1.eb:
+      options:
+        accept-eula-for: CUDA

easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.1-001-system.yml

@@ -0,0 +1,4 @@
+easyconfigs:
+  - EasyBuild-4.9.1.eb:
+      options:
+        from-pr: 20299

eb_hooks.py

@@ -612,8 +612,8 @@ def post_sanitycheck_cuda(self, *args, **kwargs):
                 full_path = os.path.join(dir_path, filename)
                 # we only really care about real files, i.e. not symlinks
                 if not os.path.islink(full_path):
-                    # check if the current file is part of the allowlist
-                    basename = os.path.splitext(filename)[0]
+                    # check if the current file name stub is part of the allowlist
+                    basename = filename.split('.')[0]
                     if basename in allowlist:
                         self.log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path)
                     else:

init/Magic_Castle/bash

@@ -10,7 +10,7 @@ source $(dirname "$BASH_SOURCE")/../eessi_environment_variables
 
 # Provide a clean MODULEPATH
 export MODULEPATH_ROOT=$EESSI_MODULEPATH
-export MODULEPATH=$EESSI_MODULEPATH
+export MODULEPATH=$EESSI_SITE_MODULEPATH:$EESSI_MODULEPATH
 
 # Extensions are too many, let's not print them by default (requires Lmod 8.4.12)
 export LMOD_AVAIL_EXTENSIONS=no

init/arch_specs/eessi_arch_x86.spec

@@ -1,6 +1,7 @@
 # x86_64 CPU architecture specifications
 # Software path in EESSI 	| Vendor ID 	| List of defining CPU features
-"x86_64/intel/haswell"		"GenuineIntel"	"avx2 fma" 		# Intel Haswell, Broadwell
+"x86_64/intel/haswell"		"GenuineIntel"	"avx2 fma" 						# Intel Haswell, Broadwell
 "x86_64/intel/skylake_avx512"	"GenuineIntel"	"avx2 fma avx512f avx512bw avx512cd avx512dq avx512vl"	# Intel Skylake, Cascade Lake
-"x86_64/amd/zen2"		"AuthenticAMD"	"avx2 fma"		# AMD Rome
-"x86_64/amd/zen3"		"AuthenticAMD"	"avx2 fma vaes"		# AMD Milan, Milan-X
+"x86_64/amd/zen2"		"AuthenticAMD"	"avx2 fma"						# AMD Rome
+"x86_64/amd/zen3"		"AuthenticAMD"	"avx2 fma vaes"						# AMD Milan, Milan-X
+"x86_64/amd/zen4"		"AuthenticAMD"	"avx2 fma vaes avx512f avx512ifma"			# AMD Genoa, Genoa-X

init/bash

@@ -26,6 +26,8 @@ if [ $? -eq 0 ]; then
     # prepend location of modules for EESSI software stack to $MODULEPATH
     show_msg "Prepending $EESSI_MODULEPATH to \$MODULEPATH..."
     module use $EESSI_MODULEPATH
+    show_msg "Prepending site path $EESSI_SITE_MODULEPATH to \$MODULEPATH..."
+    module use $EESSI_SITE_MODULEPATH
 
     #show_msg ""
     #show_msg "*** Known problems in the ${EESSI_VERSION} software stack ***"

init/eessi_environment_variables

@@ -48,6 +48,14 @@ if [ -d $EESSI_PREFIX ]; then
     fi
     if [ ! -z $EESSI_SOFTWARE_SUBDIR ]; then
 
+        # use x86_64/amd/zen3 for now when AMD Genoa (Zen4) CPU is detected,
+        # since optimized software installations for Zen4 are a work-in-progress,
+        # see https://gitlab.com/eessi/support/-/issues/37
+        if [[ "${EESSI_SOFTWARE_SUBDIR}" == "x86_64/amd/zen4" ]]; then
+            export EESSI_SOFTWARE_SUBDIR="x86_64/amd/zen3"
+            echo -e "\e[33mSticking to ${EESSI_SOFTWARE_SUBDIR} for now, since optimized installations for AMD Genoa (Zen4) are a work in progress, see https://gitlab.com/eessi/support/-/issues/37 for more information\e[0m"
+        fi
+
         show_msg "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory."
         export EESSI_SOFTWARE_PATH=$EESSI_PREFIX/software/$EESSI_OS_TYPE/$EESSI_SOFTWARE_SUBDIR
 
@@ -89,6 +97,8 @@ if [ -d $EESSI_PREFIX ]; then
           if [ -d $EESSI_MODULEPATH ]; then
             export EESSI_MODULEPATH=$EESSI_MODULEPATH
             show_msg "Using ${EESSI_MODULEPATH} as the directory to be added to MODULEPATH."
+            export EESSI_SITE_MODULEPATH=${EESSI_MODULEPATH/versions/host_injections}
+            show_msg "Using ${EESSI_SITE_MODULEPATH} as the site extension directory to be added to MODULEPATH."
           else
             error "EESSI module path at $EESSI_MODULEPATH not found!"
             false

install_scripts.sh

@@ -113,3 +113,9 @@ nvidia_files=(
     install_cuda_host_injections.sh link_nvidia_host_libraries.sh
 )
 copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}"
+
+# Copy over EasyBuild hooks file used for installations
+hook_files=(
+    eb_hooks.py
+)
+copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/init/easybuild "${hook_files[@]}"

tests/archdetect/x86_64/amd/zen4/Azure-Alma8-9V33X.all.output

@@ -0,0 +1 @@
+x86_64/amd/zen4:x86_64/amd/zen3:x86_64/amd/zen2:x86_64/generic