forked from EESSI/software-layer
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'nessi.no-2023.06' of github-trz:NorESSI/software-layer …
…into nessi-2023.06-cuDNN-12.1.1
- Loading branch information
Showing
16 changed files
with
436 additions
and
75 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,123 @@ | ||
| #!/bin/bash | ||
| # | ||
| # Script to determine which parts of the EESSI software stack (version set through init/eessi_defaults) | ||
| # have to be rebuilt | ||
|
|
||
| # see example parsing of command line arguments at | ||
| # https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop | ||
| # https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash | ||
|
|
||
| display_help() { | ||
| echo "usage: $0 [OPTIONS]" | ||
| echo " -g | --generic - instructs script to build for generic architecture target" | ||
| echo " -h | --help - display this usage information" | ||
| } | ||
|
|
||
| POSITIONAL_ARGS=() | ||
|
|
||
| while [[ $# -gt 0 ]]; do | ||
| case $1 in | ||
| -g|--generic) | ||
| DETECTION_PARAMETERS="--generic" | ||
| shift | ||
| ;; | ||
| -h|--help) | ||
| display_help # Call your function | ||
| # no shifting needed here, we're done. | ||
| exit 0 | ||
| ;; | ||
| -*|--*) | ||
| echo "Error: Unknown option: $1" >&2 | ||
| exit 1 | ||
| ;; | ||
| *) # No more options | ||
| POSITIONAL_ARGS+=("$1") # save positional arg | ||
| shift | ||
| ;; | ||
| esac | ||
| done | ||
|
|
||
| set -- "${POSITIONAL_ARGS[@]}" | ||
|
|
||
| TOPDIR=$(dirname $(realpath $0)) | ||
|
|
||
| export TMPDIR=$(mktemp -d /tmp/eessi-remove.XXXXXXXX) | ||
|
|
||
| source $TOPDIR/scripts/utils.sh | ||
|
|
||
| echo ">> Determining software subdirectory to use for current build host..." | ||
| if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then | ||
| export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) | ||
| echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" | ||
| else | ||
| echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" | ||
| fi | ||
|
|
||
| echo ">> Setting up environment..." | ||
|
|
||
| source $TOPDIR/init/bash | ||
|
|
||
| if [ -d $EESSI_CVMFS_REPO ]; then | ||
| echo_green "$EESSI_CVMFS_REPO available, OK!" | ||
| else | ||
| fatal_error "$EESSI_CVMFS_REPO is not available!" | ||
| fi | ||
|
|
||
| if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then | ||
| fatal_error "Failed to determine software subdirectory?!" | ||
| elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then | ||
| fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!" | ||
| else | ||
| echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!" | ||
| fi | ||
|
|
||
| echo ">> Configuring EasyBuild..." | ||
| EB="eb" | ||
| source $TOPDIR/configure_easybuild | ||
|
|
||
| echo ">> Setting up \$MODULEPATH..." | ||
| # make sure no modules are loaded | ||
| module --force purge | ||
| # ignore current $MODULEPATH entirely | ||
| module unuse $MODULEPATH | ||
| module use $EASYBUILD_INSTALLPATH/modules/all | ||
| if [[ -z ${MODULEPATH} ]]; then | ||
| fatal_error "Failed to set up \$MODULEPATH?!" | ||
| else | ||
| echo_green ">> MODULEPATH set up: ${MODULEPATH}" | ||
| fi | ||
|
|
||
| # assume there's only one diff file that corresponds to the PR patch file | ||
| pr_diff=$(ls [0-9]*.diff | head -1) | ||
|
|
||
| # if this script is run as root, use PR patch file to determine if software needs to be removed first | ||
| changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/") | ||
| if [ -z ${changed_easystacks_rebuilds} ]; then | ||
| echo "No software needs to be removed." | ||
| else | ||
| for easystack_file in ${changed_easystacks_rebuilds}; do | ||
| # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file | ||
| eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') | ||
|
|
||
| # load EasyBuild module (will be installed if it's not available yet) | ||
| source ${TOPDIR}/load_easybuild_module.sh ${eb_version} | ||
|
|
||
| if [ -f ${easystack_file} ]; then | ||
| echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..." | ||
| # we need to remove existing installation directories first, | ||
| # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) | ||
| # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) | ||
| rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') | ||
| for app in ${rebuild_apps}; do | ||
| app_dir=${EASYBUILD_INSTALLPATH}/software/${app} | ||
| app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua | ||
| echo_yellow "Removing ${app_dir} and ${app_module}..." | ||
| find ${app_dir} -type d | sed -e 's/^/REMOVE_DIRECTORY /' | ||
| find ${app_dir} -type f | sed -e 's/^/REMOVE_FILE /' | ||
| echo "REMOVE_MODULE ${app_module}" | ||
| done | ||
| else | ||
| fatal_error "Easystack file ${easystack_file} not found!" | ||
| fi | ||
| done | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,170 @@ | ||
| easyblock = 'Bundle' | ||
|
|
||
| name = 'NESSI-extend' | ||
| version = '2023.06' | ||
| # May have different ways to extend NESSI in the future (manually, other tools,...) | ||
| versionsuffix = '-easybuild' | ||
|
|
||
| homepage = 'https://documentation.sigma2.no/software/nessi_eessi.html' | ||
|
|
||
| description = """ | ||
| NESSI is an innovative service to make optimized scientific software | ||
| installations available on any machine anywhere in the world in near | ||
| real-time - without the need to build or install the software. NESSI | ||
| works similarly to popular streaming services for videos and music. | ||
| NESSI is a sibling of EESSI, the European Environment for Scientific | ||
| Software Installations (see https://eessi.io/docs). | ||
| This module allows you to extend NESSI using the same configuration for | ||
| EasyBuild as NESSI itself uses. A number of environment variables control the | ||
| behaviour of the module: | ||
| - NESSI_USER_INSTALL can be set to a location to install modules for use by | ||
| the user only. The location must already exist on the filesystem. | ||
| - NESSI_PROJECT_INSTALL can be set to a location to install modules for use by | ||
| a project. The location must already exist on the filesystem and you should | ||
| ensure that the location has the correct Linux group and the SGID permission | ||
| is set on that directory (`chmod g+s $NESSI_PROJECT_INSTALL`) so that all | ||
| members of the group have permission to read and write installations. | ||
| - NESSI_SITE_INSTALL is either defined or not and cannot be used with another | ||
| environment variable. A site installation is done in a defined location and | ||
| any installations there are (by default) world readable. | ||
| - NESSI_CVMFS_INSTALL is either defined or not and cannot be used with another | ||
| environment variable. A CVMFS installation targets a defined location which | ||
| will be ingested into CVMFS and is only useful for CVMFS administrators. | ||
| - If none of the environment variables above are defined, a NESSI_USER_INSTALL | ||
| is assumed with a value of $HOME/NESSI | ||
| If both NESSI_USER_INSTALL and NESSI_PROJECT_INSTALL are defined, both sets of | ||
| installations are exposed, but new installations are created as user | ||
| installations. | ||
| """ | ||
|
|
||
| toolchain = SYSTEM | ||
|
|
||
| # All the dependencies we filter in NESSI | ||
| local_deps_to_filter = "Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,M4,makeinfo,ncurses,util-linux,XZ,zlib" | ||
| local_arch_specific_deps_to_filter = {'aarch64': ',yasm', 'x86_64': ''} | ||
| local_deps_to_filter += local_arch_specific_deps_to_filter[ARCH] | ||
|
|
||
| # Set the universal EasyBuild variables | ||
| modextravars = { | ||
| 'EASYBUILD_FILTER_DEPS': local_deps_to_filter, | ||
| 'EASYBUILD_IGNORE_OSDEPS': '1', | ||
| 'EASYBUILD_DEBUG': '1', | ||
| 'EASYBUILD_TRACE': '1', | ||
| 'EASYBUILD_ZIP_LOGS': 'bzip2', | ||
| 'EASYBUILD_RPATH': '1', | ||
| 'EASYBUILD_FILTER_ENV_VARS': 'LD_LIBRARY_PATH', | ||
| 'EASYBUILD_READ_ONLY_INSTALLDIR': '1', | ||
| 'EASYBUILD_MODULE_EXTENSIONS': '1', | ||
| 'EASYBUILD_EXPERIMENTAL': '1', | ||
| } | ||
|
|
||
| # Need a few other variables, but they are more dynamic | ||
| # EASYBUILD_SYSROOT=${EPREFIX} | ||
| # EASYBUILD_PREFIX=${WORKDIR}/easybuild | ||
| # EASYBUILD_HOOKS=${EESSI_PREFIX}/init/easybuild/eb_hooks.py | ||
| # EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR} | ||
| # EASYBUILD_SOURCEPATH=${WORKDIR}/easybuild/sources:${EESSI_SOURCEPATH} | ||
| # | ||
| # And also some optional ones based on the kind of installation | ||
| # EASYBUILD_SET_GID_BIT | ||
| # EASYBUILD_GROUP_WRITABLE_INSTALLDIR | ||
| # EASYBUILD_UMASK | ||
| # EASYBUILD_STICKY_BIT | ||
| modluafooter = """ | ||
| if (mode() == "load") then | ||
| -- Use a working directory for temporary build files | ||
| if (os.getenv("WORKING_DIR") == nil) then | ||
| LmodMessage("-- Using /tmp/$USER as a temporary working directory for installations, you can override this by setting the environment variable WORKING_DIR and reloading the module (e.g., /dev/shm is a common option)") | ||
| end | ||
| end | ||
| working_dir = os.getenv("WORKING_DIR") or pathJoin("/tmp", os.getenv("USER")) | ||
| -- Gather the EPREFIX to use as a sysroot | ||
| sysroot = os.getenv("EESSI_EPREFIX") | ||
| -- Use an installation prefix that we _should_ have write access to | ||
| if (os.getenv("NESSI_CVMFS_INSTALL") ~= nil) then | ||
| -- Make sure no other NESSI install environment variables are set | ||
| if ((os.getenv("NESSI_SITE_INSTALL") ~= nil) or (os.getenv("NESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("NESSI_USER_INSTALL") ~= nil)) then | ||
| LmodError("You cannot use NESSI_CVMFS_INSTALL in combination with any other NESSI_*_INSTALL environment variables") | ||
| end | ||
| eessi_cvmfs_install = true | ||
| easybuild_installpath = os.getenv("EESSI_SOFTWARE_PATH") | ||
| elseif (os.getenv("NESSI_SITE_INSTALL") ~= nil) then | ||
| -- Make sure no other NESSI install environment variables are set | ||
| if ((os.getenv("NESSI_PROJECT_INSTALL") ~= nil) or (os.getenv("NESSI_USER_INSTALL") ~= nil)) then | ||
| LmodError("You cannot use NESSI_SITE_INSTALL in combination with any other NESSI_*_INSTALL environment variables") | ||
| end | ||
| easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), 'versions', 'host_injections') | ||
| else | ||
| -- Deal with user and project installs | ||
| project_install = os.getenv("NESSI_PROJECT_INSTALL") | ||
| project_modulepath = nil | ||
| if (project_install ~= nil) then | ||
| -- Check the folder exists | ||
| if not isDir(project_install) then | ||
| LmodError("The location of NESSI_PROJECT_INSTALL (" .. project_install .. ") does not exist or is not a folder") | ||
| end | ||
| if (mode() == "load") then | ||
| LmodMessage("Configuring for use of NESSI_PROJECT_INSTALL under " .. project_install) | ||
| end | ||
| easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), project_install) | ||
| project_modulepath = pathJoin(easybuild_installpath, 'modules', 'all') | ||
| end | ||
| user_install = os.getenv("NESSI_USER_INSTALL") | ||
| user_modulepath = nil | ||
| if (user_install ~= nil) then | ||
| -- Check the folder exists | ||
| if not isDir(user_install) then | ||
| LmodError("The location of NESSI_USER_INSTALL (" .. user_install .. ") does not exist or is not a folder") | ||
| end | ||
| elseif (user_install == nil) and (project_install == nil) then | ||
| -- No need to check for existence when we use a HOME subdir | ||
| user_install = pathJoin(os.getenv("HOME"), "nessi") | ||
| end | ||
| if (user_install ~= nil) then | ||
| if (mode() == "load") then | ||
| LmodMessage("Configuring for use of NESSI_USER_INSTALL under " .. user_install) | ||
| end | ||
| easybuild_installpath = string.gsub(os.getenv("EESSI_SOFTWARE_PATH"), os.getenv("EESSI_CVMFS_REPO"), user_install) | ||
| user_modulepath = pathJoin(easybuild_installpath, 'modules', 'all') | ||
| end | ||
| end | ||
| if (mode() == "load") then | ||
| LmodMessage("-- To create installations for NESSI, you _must_ have write permissions to " .. easybuild_installpath) | ||
| -- Advise them to reuse sources | ||
| if (os.getenv("EASYBUILD_SOURCEPATH") == nil) then | ||
| LmodMessage("-- You may wish to configure a sources directory for EasyBuild (for example, via setting the environment variable EASYBUILD_SOURCEPATH) to allow you to reuse existing sources for packages.") | ||
| end | ||
| end | ||
| -- Set the relevant universal environment variables for EasyBuild | ||
| setenv ("EASYBUILD_SYSROOT", sysroot) | ||
| setenv ("EASYBUILD_PREFIX", pathJoin(working_dir, "easybuild")) | ||
| setenv ("EASYBUILD_INSTALLPATH", easybuild_installpath) | ||
| setenv ("EASYBUILD_HOOKS", pathJoin(os.getenv("EESSI_PREFIX"), "init", "easybuild", "eb_hooks.py")) | ||
| setenv ("EASYBUILD_UMASK", "002") | ||
| -- Set all related environment variables if we have project or user installations (including extending MODULEPATH) | ||
| if (user_modulepath ~= nil) then | ||
| -- Use a more restrictive umask for this case | ||
| setenv ("EASYBUILD_UMASK", "022") | ||
| setenv ("EASYBUILD_STICKY_BIT", "1") | ||
| -- configure MODULEPATH | ||
| if (project_modulepath ~= nil) then | ||
| prepend_path("MODULEPATH", project_modulepath) | ||
| end | ||
| prepend_path("MODULEPATH", user_modulepath) | ||
| elseif (project_modulepath ~= nil) then | ||
| setenv ("EASYBUILD_SET_GID_BIT", "1") | ||
| setenv ("EASYBUILD_GROUP_WRITABLE_INSTALLDIR", "1") | ||
| setenv ("EASYBUILD_STICKY_BIT", "0") | ||
| -- configure MODULEPATH | ||
| prepend_path("MODULEPATH", project_modulepath) | ||
| end | ||
| -- Make sure EasyBuild itself is loaded | ||
| if not ( isloaded("EasyBuild") ) then | ||
| load("EasyBuild") | ||
| end | ||
| """ | ||
|
|
||
| moduleclass = 'devel' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.