From 6e699eba7b46695e5c25e8f1563a6a32f7126b54 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 8 May 2024 07:50:21 +0000 Subject: [PATCH 1/5] Easyconfig that can extend EESSI --- EESSI-extend-2023.06-easybuild.eb | 167 ++++++++++++++++++ create_tarball.sh | 2 +- .../eessi-2023.06-eb-4.9.1-001-system.yml | 1 + init/Magic_Castle/bash | 2 +- init/bash | 2 + init/eessi_environment_variables | 2 + install_scripts.sh | 6 + 7 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 EESSI-extend-2023.06-easybuild.eb diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb new file mode 100644 index 0000000000..76e96766f9 --- /dev/null +++ b/EESSI-extend-2023.06-easybuild.eb @@ -0,0 +1,167 @@ +easyblock = 'Bundle' + +name = 'EESSI-extend' +version = '2023.06' +# May have different ways to extend EESSI in future (manually, other tools,...) +versionsuffix = '-easybuild' + +homepage = 'https://eessi.io/docs/' + +description = """ + The goal of the European Environment for Scientific Software Installations + (EESSI, pronounced as "easy") is to build a common stack of scientific + software installations for HPC systems and beyond, including laptops, + personal workstations and cloud infrastructure. + + This module allows you to extend EESSI using the same configuration for + EasyBuild as EESSI itself uses. A number of environment variables control the + behaviour of the module: + - EESSI_USER_INSTALL can be set to a location to install modules for use by + the user only. The location must already exist on the filesystem. + - EESSI_PROJECT_INSTALL can be set to a location to install modules for use by + a project. The location must already exist on the filesystem and you should + ensure that the location has the correct Linux group and the SGID permission + is set on that directory (`chmod g+s $EESSI_PROJECT_INSTALL`) so that all + members of the group have permission to read and write installations. + - EESSI_SITE_INSTALL is either defined or not and cannot be used with another + environment variable. A site installation is done in a defined location and + any installations there are (by default) world readable. + - EESSI_CVMFS_INSTALL is either defined or not and cannot be used with another + environment variable. A CVMFS installation targets a defined location which + will be ingested into CVMFS and is only useful for CVMFS administrators. + - If none of the environment variables above are defined, an EESSI_USER_INSTALL + is assumed with a value of $HOME/EESSI + If both EESSI_USER_INSTALL and EESSI_PROJECT_INSTALL are defined, both sets of + installations are exposed, but new installations are created as user + installations. +""" + +toolchain = SYSTEM + +# All the dependencies we filter in EESSI +local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,M4,makeinfo,ncurses,util-linux,XZ,zlib" +local_arch_specific_deps_to_filter = {'aarch64': ',yasm', 'x86_64': ''} +local_deps_to_filter += local_arch_specific_deps_to_filter[ARCH] + +# Set the universal EasyBuild variables +modextravars = { + 'EASYBUILD_FILTER_DEPS': local_deps_to_filter, + 'EASYBUILD_IGNORE_OSDEPS': '1', + 'EASYBUILD_DEBUG': '1', + 'EASYBUILD_TRACE': '1', + 'EASYBUILD_ZIP_LOGS': 'bzip2', + 'EASYBUILD_RPATH': '1', + 'EASYBUILD_FILTER_ENV_VARS': 'LD_LIBRARY_PATH', + 'EASYBUILD_READ_ONLY_INSTALLDIR': '1', + 'EASYBUILD_MODULE_EXTENSIONS': '1', + 'EASYBUILD_EXPERIMENTAL': '1', +} + +# Need a few other variables, but they are more dynamic +# EASYBUILD_SYSROOT=${EPREFIX} +# EASYBUILD_PREFIX=${WORKDIR}/easybuild +# EASYBUILD_HOOKS=${EESSI_PREFIX}/init/easybuild/eb_hooks.py +# EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR} +# EASYBUILD_SOURCEPATH=${WORKDIR}/easybuild/sources:${EESSI_SOURCEPATH} +# +# And also some optional ones based on the kind of installation +# EASYBUILD_SET_GID_BIT +# EASYBUILD_GROUP_WRITABLE_INSTALLDIR +# EASYBUILD_UMASK +# EASYBUILD_STICKY_BIT +modluafooter = """ +if (mode() == "load") then + -- Use a working directory for temporary build files + if (os.getenv("WORKING_DIR") == nil) then + LmodMessage("-- Using /tmp/$USER as a temporary working directory for installations, you can override this by setting the environment variable WORKING_DIR and reloading the module (e.g., /dev/shm is a common option)") + end +end +working_dir = os.getenv("WORKING_DIR") or pathJoin("/tmp", os.getenv("USER")) +-- Gather the EPREFIX to use as a sysroot +sysroot = os.getenv("EESSI_EPREFIX") +-- Use an installation prefix that we _should_ have write access to +if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then + -- Make sure no other EESSI install environment variables are set + if ((os.getenv("EESSI_SITE_INSTALL") ~= nil) or (os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then + LmodError("You cannot use EESSI_CVMFS_INSTALL in combination with any other EESSI_*_INSTALL environment variables") + end + eessi_cvmfs_install = true + easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH") +elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then + -- Make sure no other EESSI install environment variables are set + if ((os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then + LmodError("You cannot use EESSI_SITE_INSTALL in combination with any other EESSI_*_INSTALL environment variables") + end + easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), 'versions', 'host_injections') +else + -- Deal with user and project installs + project_install = os.getenv("EESSI_PROJECT_INSTALL") + project_modulepath = nil + if (project_install ~= nil) then + -- Check the folder exists + if not isDir(project_install) then + LmodError("The location of EESSI_PROJECT_INSTALL (" .. project_install .. ") does not exist or is not a folder") + end + if (mode() == "load") then + LmodMessage("Configuring for use of EESSI_PROJECT_INSTALL under " .. project_install) + end + easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), project_install) + project_modulepath = pathJoin(easybuild_installpath, 'modules', 'all') + end + user_install = os.getenv("EESSI_USER_INSTALL") + user_modulepath = nil + if (user_install ~= nil) then + -- Check the folder exists + if not isDir(user_install) then + LmodError("The location of EESSI_USER_INSTALL (" .. user_install .. ") does not exist or is not a folder") + end + elseif (user_install == nil) and (project_install == nil) then + -- No need to check for existence when we use a HOME subdir + user_install = pathJoin(os.getenv("HOME"), "eessi") + end + if (user_install ~= nil) then + if (mode() == "load") then + LmodMessage("Configuring for use of EESSI_USER_INSTALL under " .. user_install) + end + easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), user_install) + user_modulepath = pathJoin(easybuild_installpath, 'modules', 'all') + end +end +if (mode() == "load") then + LmodMessage("-- To create installations for EESSI, you _must_ have write permissions to " .. easybuild_installpath) + -- Advise them to reuse sources + if (os.getenv("EASYBUILD_SOURCEPATH") == nil) then + LmodMessage("-- You may wish to configure a sources directory for EasyBuild (for example, via setting the environment variable EASYBUILD_SOURCEPATH) to allow you to reuse existing sources for packages.") + end +end +-- Set the relevant universal environment variables for EasyBuild +setenv ("EASYBUILD_SYSROOT", sysroot) +setenv ("EASYBUILD_PREFIX", pathJoin(working_dir, "easybuild")) +setenv ("EASYBUILD_INSTALLPATH", easybuild_installpath) +setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuild", "eb_hooks.py")) +setenv ("EASYBUILD_UMASK", "002") + +-- Set all related environment variables if we have project or user installations (including extending MODULEPATH) +if (user_modulepath ~= nil) then + -- Use a more restrictive umask for this case + setenv ("EASYBUILD_UMASK", "022") + setenv ("EASYBUILD_STICKY_BIT", "1") + -- configure MODULEPATH + if (project_modulepath ~= nil) then + prepend_path("MODULEPATH", project_modulepath) + end + prepend_path("MODULEPATH", user_modulepath) +elseif (project_modulepath ~= nil) then + setenv ("EASYBUILD_SET_GID_BIT", "1") + setenv ("EASYBUILD_GROUP_WRITABLE_INSTALLDIR", "1") + setenv ("EASYBUILD_STICKY_BIT", "0") + -- configure MODULEPATH + prepend_path("MODULEPATH", project_modulepath) +end +-- Make sure EasyBuild itself is loaded +if not ( isloaded("EasyBuild") ) then + load("EasyBuild") +end +""" + +moduleclass = 'devel' diff --git a/create_tarball.sh b/create_tarball.sh index 2d77acfc43..0a7669f73f 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -28,7 +28,7 @@ fi overlay_upper_dir="${eessi_tmpdir}/overlay-upper" -software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" +software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2 exit 3 diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml index c5a08b5209..46ac979719 100644 --- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml +++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-001-system.yml @@ -2,3 +2,4 @@ easyconfigs: - EasyBuild-4.9.1.eb: options: from-pr: 20299 + - EESSI-extend-2023.06-easybuild.eb diff --git a/init/Magic_Castle/bash b/init/Magic_Castle/bash index 0fb2f670a3..e2fded897d 100644 --- a/init/Magic_Castle/bash +++ b/init/Magic_Castle/bash @@ -10,7 +10,7 @@ source $(dirname "$BASH_SOURCE")/../eessi_environment_variables # Provide a clean MODULEPATH export MODULEPATH_ROOT=$EESSI_MODULEPATH -export MODULEPATH=$EESSI_MODULEPATH +export MODULEPATH=$EESSI_SITE_MODULEPATH:$EESSI_MODULEPATH # Extensions are too many, let's not print them by default (requires Lmod 8.4.12) export LMOD_AVAIL_EXTENSIONS=no diff --git a/init/bash b/init/bash index 2097f03617..d72df1f346 100644 --- a/init/bash +++ b/init/bash @@ -26,6 +26,8 @@ if [ $? -eq 0 ]; then # prepend location of modules for EESSI software stack to $MODULEPATH show_msg "Prepending $EESSI_MODULEPATH to \$MODULEPATH..." module use $EESSI_MODULEPATH + show_msg "Prepending site path $EESSI_SITE_MODULEPATH to \$MODULEPATH..." + module use $EESSI_SITE_MODULEPATH #show_msg "" #show_msg "*** Known problems in the ${EESSI_VERSION} software stack ***" diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 815b46d0e1..1a44b53c41 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -89,6 +89,8 @@ if [ -d $EESSI_PREFIX ]; then if [ -d $EESSI_MODULEPATH ]; then export EESSI_MODULEPATH=$EESSI_MODULEPATH show_msg "Using ${EESSI_MODULEPATH} as the directory to be added to MODULEPATH." + export EESSI_SITE_MODULEPATH=${EESSI_MODULEPATH/versions/host_injections} + show_msg "Using ${EESSI_SITE_MODULEPATH} as the site extension directory to be added to MODULEPATH." else error "NESSI module path at $EESSI_MODULEPATH not found!" false diff --git a/install_scripts.sh b/install_scripts.sh index 508735975c..17f0b81008 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -113,3 +113,9 @@ nvidia_files=( install_cuda_host_injections.sh link_nvidia_host_libraries.sh ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" + +# Copy over EasyBuild hooks file used for installations +hook_files=( + eb_hooks.py +) +copy_files_by_list ${TOPDIR} ${INSTALL_PREFIX}/init/easybuild "${hook_files[@]}" From da3f5f5d20dc5137a1d255e6bd1a734507e80ad6 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 8 May 2024 09:53:38 +0000 Subject: [PATCH 2/5] modified changes in EESSI-Extend easyconfig to meet NESSI requirements --- EESSI-extend-2023.06-easybuild.eb | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 76e96766f9..1d8362d956 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -1,6 +1,6 @@ easyblock = 'Bundle' -name = 'EESSI-extend' +name = 'NESSI-extend' version = '2023.06' # May have different ways to extend EESSI in future (manually, other tools,...) versionsuffix = '-easybuild' @@ -80,55 +80,55 @@ working_dir = os.getenv("WORKING_DIR") or pathJoin("/tmp", os.getenv("USER")) -- Gather the EPREFIX to use as a sysroot sysroot = os.getenv("EESSI_EPREFIX") -- Use an installation prefix that we _should_ have write access to -if (os.getenv("EESSI_CVMFS_INSTALL") ~= nil) then +if (os.getenv("NESSI_CVMFS_INSTALL") ~= nil) then -- Make sure no other EESSI install environment variables are set - if ((os.getenv("EESSI_SITE_INSTALL") ~= nil) or (os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then - LmodError("You cannot use EESSI_CVMFS_INSTALL in combination with any other EESSI_*_INSTALL environment variables") + if ((os.getenv("NESSI_SITE_INSTALL") ~= nil) or (os.getenv("NESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("NESSI_USER_INSTALL") ~= nil)) then + LmodError("You cannot use NESSI_CVMFS_INSTALL in combination with any other NESSI_*_INSTALL environment variables") end eessi_cvmfs_install = true easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH") -elseif (os.getenv("EESSI_SITE_INSTALL") ~= nil) then +elseif (os.getenv("NESSI_SITE_INSTALL") ~= nil) then -- Make sure no other EESSI install environment variables are set - if ((os.getenv("EESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("EESSI_USER_INSTALL") ~= nil)) then - LmodError("You cannot use EESSI_SITE_INSTALL in combination with any other EESSI_*_INSTALL environment variables") + if ((os.getenv("NESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("NESSI_USER_INSTALL") ~= nil)) then + LmodError("You cannot use NESSI_SITE_INSTALL in combination with any other NESSI_*_INSTALL environment variables") end easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), 'versions', 'host_injections') else -- Deal with user and project installs - project_install = os.getenv("EESSI_PROJECT_INSTALL") + project_install = os.getenv("NESSI_PROJECT_INSTALL") project_modulepath = nil if (project_install ~= nil) then -- Check the folder exists if not isDir(project_install) then - LmodError("The location of EESSI_PROJECT_INSTALL (" .. project_install .. ") does not exist or is not a folder") + LmodError("The location of NESSI_PROJECT_INSTALL (" .. project_install .. ") does not exist or is not a folder") end if (mode() == "load") then - LmodMessage("Configuring for use of EESSI_PROJECT_INSTALL under " .. project_install) + LmodMessage("Configuring for use of NESSI_PROJECT_INSTALL under " .. project_install) end easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), project_install) project_modulepath = pathJoin(easybuild_installpath, 'modules', 'all') end - user_install = os.getenv("EESSI_USER_INSTALL") + user_install = os.getenv("NESSI_USER_INSTALL") user_modulepath = nil if (user_install ~= nil) then -- Check the folder exists if not isDir(user_install) then - LmodError("The location of EESSI_USER_INSTALL (" .. user_install .. ") does not exist or is not a folder") + LmodError("The location of NESSI_USER_INSTALL (" .. user_install .. ") does not exist or is not a folder") end elseif (user_install == nil) and (project_install == nil) then -- No need to check for existence when we use a HOME subdir - user_install = pathJoin(os.getenv("HOME"), "eessi") + user_install = pathJoin(os.getenv("HOME"), "nessi") end if (user_install ~= nil) then if (mode() == "load") then - LmodMessage("Configuring for use of EESSI_USER_INSTALL under " .. user_install) + LmodMessage("Configuring for use of NESSI_USER_INSTALL under " .. user_install) end easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), user_install) user_modulepath = pathJoin(easybuild_installpath, 'modules', 'all') end end if (mode() == "load") then - LmodMessage("-- To create installations for EESSI, you _must_ have write permissions to " .. easybuild_installpath) + LmodMessage("-- To create installations for NESSI, you _must_ have write permissions to " .. easybuild_installpath) -- Advise them to reuse sources if (os.getenv("EASYBUILD_SOURCEPATH") == nil) then LmodMessage("-- You may wish to configure a sources directory for EasyBuild (for example, via setting the environment variable EASYBUILD_SOURCEPATH) to allow you to reuse existing sources for packages.") From 2eeb6e0f8b703e620230251a9fadc7a89a1dae31 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Wed, 8 May 2024 13:12:45 +0000 Subject: [PATCH 3/5] reverted changes to create_tarball as it will be done in a seperate PR --- create_tarball.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_tarball.sh b/create_tarball.sh index 0a7669f73f..2d77acfc43 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -28,7 +28,7 @@ fi overlay_upper_dir="${eessi_tmpdir}/overlay-upper" -software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" +software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}" if [ ! -d ${software_dir_overlay} ]; then echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2 exit 3 From 46ef3724b7bfdc9ad8d4b7f46bad7252c3c07aa5 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 8 May 2024 21:57:33 +0200 Subject: [PATCH 4/5] further adjust easyconfig to NESSI --- EESSI-extend-2023.06-easybuild.eb | 41 +++++++++++++++++-------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index 1d8362d956..ed71ee5b53 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -2,43 +2,46 @@ easyblock = 'Bundle' name = 'NESSI-extend' version = '2023.06' -# May have different ways to extend EESSI in future (manually, other tools,...) +# May have different ways to extend NESSI in the future (manually, other tools,...) versionsuffix = '-easybuild' -homepage = 'https://eessi.io/docs/' +homepage = 'https://documentation.sigma2.no/software/nessi_eessi.html' description = """ - The goal of the European Environment for Scientific Software Installations - (EESSI, pronounced as "easy") is to build a common stack of scientific - software installations for HPC systems and beyond, including laptops, - personal workstations and cloud infrastructure. + NESSI is an innovative service to make optimized scientific software + installations available on any machine anywhere in the world in near + real-time - without the need to build or install the software. NESSI + works similarly to popular streaming services for videos and music. - This module allows you to extend EESSI using the same configuration for - EasyBuild as EESSI itself uses. A number of environment variables control the + NESSI is a sibling of EESSI, the European Environment for Scientific + Software Installations (see https://eessi.io/docs). + + This module allows you to extend NESSI using the same configuration for + EasyBuild as NESSI itself uses. A number of environment variables control the behaviour of the module: - - EESSI_USER_INSTALL can be set to a location to install modules for use by + - NESSI_USER_INSTALL can be set to a location to install modules for use by the user only. The location must already exist on the filesystem. - - EESSI_PROJECT_INSTALL can be set to a location to install modules for use by + - NESSI_PROJECT_INSTALL can be set to a location to install modules for use by a project. The location must already exist on the filesystem and you should ensure that the location has the correct Linux group and the SGID permission - is set on that directory (`chmod g+s $EESSI_PROJECT_INSTALL`) so that all + is set on that directory (`chmod g+s $NESSI_PROJECT_INSTALL`) so that all members of the group have permission to read and write installations. - - EESSI_SITE_INSTALL is either defined or not and cannot be used with another + - NESSI_SITE_INSTALL is either defined or not and cannot be used with another environment variable. A site installation is done in a defined location and any installations there are (by default) world readable. - - EESSI_CVMFS_INSTALL is either defined or not and cannot be used with another + - NESSI_CVMFS_INSTALL is either defined or not and cannot be used with another environment variable. A CVMFS installation targets a defined location which will be ingested into CVMFS and is only useful for CVMFS administrators. - - If none of the environment variables above are defined, an EESSI_USER_INSTALL - is assumed with a value of $HOME/EESSI - If both EESSI_USER_INSTALL and EESSI_PROJECT_INSTALL are defined, both sets of + - If none of the environment variables above are defined, a NESSI_USER_INSTALL + is assumed with a value of $HOME/NESSI + If both NESSI_USER_INSTALL and NESSI_PROJECT_INSTALL are defined, both sets of installations are exposed, but new installations are created as user installations. """ toolchain = SYSTEM -# All the dependencies we filter in EESSI +# All the dependencies we filter in NESSI local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,M4,makeinfo,ncurses,util-linux,XZ,zlib" local_arch_specific_deps_to_filter = {'aarch64': ',yasm', 'x86_64': ''} local_deps_to_filter += local_arch_specific_deps_to_filter[ARCH] @@ -81,14 +84,14 @@ working_dir = os.getenv("WORKING_DIR") or pathJoin("/tmp", os.getenv("USER")) sysroot = os.getenv("EESSI_EPREFIX") -- Use an installation prefix that we _should_ have write access to if (os.getenv("NESSI_CVMFS_INSTALL") ~= nil) then - -- Make sure no other EESSI install environment variables are set + -- Make sure no other NESSI install environment variables are set if ((os.getenv("NESSI_SITE_INSTALL") ~= nil) or (os.getenv("NESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("NESSI_USER_INSTALL") ~= nil)) then LmodError("You cannot use NESSI_CVMFS_INSTALL in combination with any other NESSI_*_INSTALL environment variables") end eessi_cvmfs_install = true easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH") elseif (os.getenv("NESSI_SITE_INSTALL") ~= nil) then - -- Make sure no other EESSI install environment variables are set + -- Make sure no other NESSI install environment variables are set if ((os.getenv("NESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("NESSI_USER_INSTALL") ~= nil)) then LmodError("You cannot use NESSI_SITE_INSTALL in combination with any other NESSI_*_INSTALL environment variables") end From b5f3c95d872a5d7267dfc5d3725cccbe9bf3ec6e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 8 May 2024 21:58:10 +0200 Subject: [PATCH 5/5] adjust creation of SitePackage.lua to NESSI --- create_lmodsitepackage.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 28605beea5..f9053cdf9e 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -21,19 +21,19 @@ end local function from_eessi_prefix(t) - -- eessi_prefix is the prefix with official EESSI modules - -- e.g. /cvmfs/software.eessi.io/versions/2023.06 + -- eessi_prefix is the prefix with official NESSI modules + -- e.g. /cvmfs/pilot.nessi.no/versions/2023.06 local eessi_prefix = os.getenv("EESSI_PREFIX") - -- If EESSI_PREFIX wasn't defined, we cannot check if this module was from the EESSI environment + -- If EESSI_PREFIX wasn't defined, we cannot check if this module was from the NESSI environment -- In that case, we assume it isn't, otherwise EESSI_PREFIX would (probably) have been set if eessi_prefix == nil then return False else -- NOTE: exact paths for site so may need to be updated later. - -- See https://github.com/EESSI/software-layer/pull/371 + -- See https://github.com/NorESSI/software-layer/pull/358 -- eessi_prefix_host_injections is the prefix with site-extensions (i.e. additional modules) - -- to the official EESSI modules, e.g. /cvmfs/software.eessi.io/host_injections/2023.06 + -- to the official NESSI modules, e.g. /cvmfs/pilot.nessi.no/host_injections/2023.06 local eessi_prefix_host_injections = string.gsub(eessi_prefix, 'versions', 'host_injections') -- Check if the full modulepath starts with the eessi_prefix_* @@ -42,9 +42,9 @@ end local function load_site_specific_hooks() - -- This function will be run after the EESSI hooks are registered + -- This function will be run after the NESSI hooks are registered -- It will load a local SitePackage.lua that is architecture independent (if it exists) from e.g. - -- /cvmfs/software.eessi.io/host_injections/2023.06/.lmod/SitePackage.lua + -- /cvmfs/pilot.nessi.no/host_injections/2023.06/.lmod/SitePackage.lua -- That can define a new hook -- -- function site_specific_load_hook(t) @@ -58,7 +58,7 @@ -- site_specific_load_hook(t) -- end -- - -- Over overwrite the EESSI hook entirely: + -- Or overwrite the NESSI hook entirely: -- -- hook.register("load", final_load_hook) -- @@ -66,7 +66,7 @@ -- See https://github.com/TACC/Lmod/pull/696#issuecomment-1998765722 -- -- Subsequently, this function will look for an architecture-specific SitePackage.lua, e.g. from - -- /cvmfs/software.eessi.io/host_injections/2023.06/software/linux/x86_64/amd/zen2/.lmod/SitePackage.lua + -- /cvmfs/pilot.nessi.no/host_injections/2023.06/software/linux/x86_64/amd/zen2/.lmod/SitePackage.lua -- This can then register an additional hook, e.g. -- -- function arch_specific_load_hook(t) @@ -112,7 +112,7 @@ local simpleName = string.match(t.modFullName, "(.-)/") -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse - -- to load the CUDA module and print an informative message on how to set up GPU support for EESSI + -- to load the CUDA module and print an informative message on how to set up GPU support for NESSI local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/gpu/.\\n" if simpleName == 'CUDA' then -- get the full host_injections path @@ -121,26 +121,26 @@ local cudaEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild" local cudaDirExists = isDir(cudaEasyBuildDir) if not cudaDirExists then - local advice = "but while the module file exists, the actual software is not entirely shipped with EESSI " - advice = advice .. "due to licencing. You will need to install a full copy of the CUDA SDK where EESSI " + local advice = "but while the module file exists, the actual software is not entirely shipped with NESSI " + advice = advice .. "due to licencing. You will need to install a full copy of the CUDA SDK where NESSI " advice = advice .. "can find it.\\n" advice = advice .. refer_to_docs LmodError("\\nYou requested to load ", simpleName, " ", advice) end end - -- when loading CUDA enabled modules check if the necessary driver libraries are accessible to the EESSI linker, + -- when loading CUDA enabled modules check if the necessary driver libraries are accessible to the NESSI linker, -- otherwise, refuse to load the requested module and print error message local haveGpu = mt:haveProperty(simpleName,"arch","gpu") if haveGpu then local arch = os.getenv("EESSI_CPU_FAMILY") or "" - local cudaVersionFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" - local cudaDriverFile = "/cvmfs/software.eessi.io/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" + local cudaVersionFile = "/cvmfs/pilot.nessi.no/host_injections/nvidia/" .. arch .. "/latest/cuda_version.txt" + local cudaDriverFile = "/cvmfs/pilot.nessi.no/host_injections/nvidia/" .. arch .. "/latest/libcuda.so" local cudaDriverExists = isFile(cudaDriverFile) local singularityCudaExists = isFile("/.singularity.d/libs/libcuda.so") if not (cudaDriverExists or singularityCudaExists) then local advice = "which relies on the CUDA runtime environment and driver libraries. " advice = advice .. "In order to be able to use the module, you will need " - advice = advice .. "to make sure EESSI can find the GPU driver libraries on your host system.\\n" + advice = advice .. "to make sure NESSI can find the GPU driver libraries on your host system.\\n" advice = advice .. refer_to_docs LmodError("\\nYou requested to load ", simpleName, " ", advice) else @@ -162,7 +162,7 @@ if driver_libs_need_update == true then local advice = "but the module you want to load requires CUDA " .. cudaVersion_req .. ". " advice = advice .. "Please update your CUDA driver libraries and then " - advice = advice .. "let EESSI know about the update.\\n" + advice = advice .. "let NESSI know about the update.\\n" advice = advice .. refer_to_docs LmodError("\\nYour driver CUDA version is ", cudaVersion, " ", advice) end @@ -174,7 +174,7 @@ -- Combine both functions into a single one, as we can only register one function as load hook in lmod -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed function eessi_load_hook(t) - -- Only apply CUDA hooks if the loaded module is in the EESSI prefix + -- Only apply CUDA hooks if the loaded module is in the NESSI prefix -- This avoids getting an Lmod Error when trying to load a CUDA module from a local software stack if from_eessi_prefix(t) then eessi_cuda_enabled_load_hook(t) @@ -183,7 +183,7 @@ hook.register("load", eessi_load_hook) --- Note that this needs to happen at the end, so that any EESSI specific hooks can be overwritten by the site +-- Note that this needs to happen at the end, so that any NESSI specific hooks can be overwritten by the site load_site_specific_hooks() """