Merge branch 'nessi.no-2023.06' of github-trz:NorESSI/software-layer …

…into sync_nessi_eessi_test_pr

bot/build.sh

@@ -53,6 +53,33 @@ LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
 echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'"
 # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
 
+# check if path to copy build logs to is specified, so we can copy build logs for failing builds there
+BUILD_LOGS_DIR=$(cfg_get_value "site_config" "build_logs_dir")
+echo "bot/build.sh: BUILD_LOGS_DIR='${BUILD_LOGS_DIR}'"
+# if $BUILD_LOGS_DIR is set, add it to $SINGULARITY_BIND so the path is available in the build container
+if [[ ! -z ${BUILD_LOGS_DIR} ]]; then
+    mkdir -p ${BUILD_LOGS_DIR}
+    if [[ -z ${SINGULARITY_BIND} ]]; then
+        export SINGULARITY_BIND="${BUILD_LOGS_DIR}"
+    else
+        export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}"
+    fi
+fi
+
+# check if path to directory on shared filesystem is specified,
+# and use it as location for source tarballs used by EasyBuild if so
+SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path")
+echo "bot/build.sh: SHARED_FS_PATH='${SHARED_FS_PATH}'"
+# if $SHARED_FS_PATH is set, add it to $SINGULARITY_BIND so the path is available in the build container
+if [[ ! -z ${SHARED_FS_PATH} ]]; then
+    mkdir -p ${SHARED_FS_PATH}
+    if [[ -z ${SINGULARITY_BIND} ]]; then
+        export SINGULARITY_BIND="${SHARED_FS_PATH}"
+    else
+        export SINGULARITY_BIND="${SINGULARITY_BIND},${SHARED_FS_PATH}"
+    fi
+fi
+
 SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
 echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
 if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
@@ -151,19 +178,21 @@ BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
 
 # prepare arguments to install_software_layer.sh (specific to build step)
-GENERIC_OPT=
+declare -a INSTALL_SCRIPT_ARGS=()
 if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
-    GENERIC_OPT="--generic"
+    INSTALL_SCRIPT_ARGS+=("--generic")
 fi
+[[ ! -z ${BUILD_LOGS_DIR} ]] && INSTALL_SCRIPT_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}")
+[[ ! -z ${SHARED_FS_PATH} ]] && INSTALL_SCRIPT_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
 
 # create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)
 
 echo "Executing command to build software:"
 echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}"
-echo "                     -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}"
+echo "                     -- ./install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}"
 ./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \
-                     -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr}
+                     -- ./install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr}
 
 # prepare directory to store tarball of tmp for tarball step
 TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step

init/arch_specs/eessi_arch_arm.spec

@@ -1,6 +1,6 @@
 # ARM CPU architecture specifications
 # Software path in EESSI 	| Vendor ID 	| List of defining CPU features
-"aarch64/arm/neoverse-n1"	"ARM"		"asimd"		# Ampere Altra
-"aarch64/arm/neoverse-n1"	""		"asimd"		# AWS Graviton2
-"aarch64/arm/neoverse-v1"	"ARM"		"asimd svei8mm" 
-"aarch64/arm/neoverse-v1"	""		"asimd svei8mm"	# AWS Graviton3
+"aarch64/neoverse-n1"	"ARM"		"asimd"		# Ampere Altra
+"aarch64/neoverse-n1"	""		"asimd"		# AWS Graviton2
+"aarch64/neoverse-v1"	"ARM"		"asimd svei8mm" 
+"aarch64/neoverse-v1"	""		"asimd svei8mm"	# AWS Graviton3

init/bash

@@ -1,11 +1,10 @@
-# Allow for a silent mode
-if [[ -v EESSI_SILENT ]]; then
-  # EESSI_SILENT set
-  output=/dev/null
-else
-  output=/dev/stdout
-fi
-
+function show_msg {
+  # only echo msg if EESSI_SILENT is unset
+  msg=$1
+  if [[ ! -v EESSI_SILENT ]]; then
+    echo "$msg"
+  fi
+}
 # The following method should be safe, but might break if file is a symlink
 # (could switch to $(dirname "$(readlink -f "$BASH_SOURCE")") in that case)
 source $(dirname "$BASH_SOURCE")/eessi_environment_variables
@@ -20,19 +19,19 @@ if [ $? -eq 0 ]; then
     export PATH=$EPREFIX/usr/bin:$EPREFIX/bin:$PATH
 
     # init Lmod
-    echo "Initializing Lmod..." >> $output
+    show_msg "Initializing Lmod..."
     source $EESSI_EPREFIX/usr/share/Lmod/init/bash
 
     # prepend location of modules for EESSI software stack to $MODULEPATH
     echo "Prepending $EESSI_MODULEPATH to \$MODULEPATH..." >> $output
     module use $EESSI_MODULEPATH
 
-    #echo >> $output
-    #echo "*** Known problems in the ${EESSI_VERSION} software stack ***" >> $output
-    #echo >> $output
-    #echo "1) ..." >> $output
-    #echo >> $output
-    #echo >> $output
+    #show_msg ""
+    #show_msg "*** Known problems in the ${EESSI_VERSION} software stack ***"
+    #show_msg ""
+    #show_msg "1) ..."
+    #show_msg ""
+    #show_msg ""
 
     echo "Environment set up to use NESSI (${EESSI_VERSION}), have fun!"
 

init/eessi_archdetect.sh

@@ -1,8 +1,10 @@
 #!/usr/bin/env bash
-VERSION="1.0.0"
+VERSION="1.1.0"
 
-# Logging
-LOG_LEVEL="INFO"
+# default log level: only emit warnings or errors
+LOG_LEVEL="WARN"
+# Default result type is a best match
+CPUPATH_RESULT="best"
 
 timestamp () {
     date "+%Y-%m-%d %H:%M:%S"
@@ -67,8 +69,8 @@ check_allinfirst(){
 
 cpupath(){
     # If EESSI_SOFTWARE_SUBDIR_OVERRIDE is set, use it
-    log "DEBUG" "cpupath: Override variable set as '$EESI_SOFTWARE_SUBDIR_OVERRIDE' "
-    [ $EESI_SOFTWARE_SUBDIR_OVERRIDE ] && echo ${EESI_SOFTWARE_SUBDIR_OVERRIDE} && exit
+    log "DEBUG" "cpupath: Override variable set as '$EESSI_SOFTWARE_SUBDIR_OVERRIDE' "
+    [ $EESI_SOFTWARE_SUBDIR_OVERRIDE ] && echo ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} && exit
 
     # Identify the best matching CPU architecture from a list of supported specifications for the host CPU
     # Return the path to the installation files in NESSI of the best matching architecture
@@ -105,7 +107,8 @@ cpupath(){
     log "DEBUG" "cpupath: CPU flags of host system: '$cpu_flags'"
 
     # Default to generic CPU
-    local best_arch_match="generic"
+    local best_arch_match="$machine_type/generic"
+    local all_arch_matches=$best_arch_match
 
     # Iterate over the supported CPU specifications to find the best match for host CPU
     # Order of the specifications matters, the last one to match will be selected
@@ -114,22 +117,29 @@ cpupath(){
         if [ "${cpu_vendor}x" == "${arch_spec[1]}x" ]; then
             # each flag in this CPU specification must be found in the list of flags of the host
             check_allinfirst "${cpu_flags[*]}" ${arch_spec[2]} && best_arch_match=${arch_spec[0]} && \
+                all_arch_matches="$best_arch_match:$all_arch_matches" && \
                 log "DEBUG" "cpupath: host CPU best match updated to $best_arch_match"
         fi
     done
 
-    log "INFO" "cpupath: best match for host CPU: $best_arch_match"
-    echo "$best_arch_match"
+    if [ "allx" == "${CPUPATH_RESULT}x" ]; then
+        log "INFO" "cpupath: all matches for host CPU: $all_arch_matches"
+        echo "$all_arch_matches"
+    else
+        log "INFO" "cpupath: best match for host CPU: $best_arch_match"
+        echo "$best_arch_match"
+    fi
 }
 
 # Parse command line arguments
-USAGE="Usage: eessi_archdetect.sh [-h][-d] <action>"
+USAGE="Usage: eessi_archdetect.sh [-h][-d][-a] <action>"
 
-while getopts 'hdv' OPTION; do
+while getopts 'hdva' OPTION; do
     case "$OPTION" in
         h) echo "$USAGE"; exit 0;;
         d) LOG_LEVEL="DEBUG";;
         v) echo "eessi_archdetect.sh v$VERSION"; exit 0;;
+        a) CPUPATH_RESULT="all";;
         ?) echo "$USAGE"; exit 1;;
     esac
 done
@@ -139,5 +149,5 @@ ARGUMENT=${1:-none}
 
 case "$ARGUMENT" in
     "cpupath") cpupath; exit;;
-    *) echo "$USAGE"; log "ERROR" "Missing <action> argument";;
+    *) echo "$USAGE"; log "ERROR" "Missing <action> argument (possible actions: 'cpupath')";;
 esac

init/eessi_environment_variables

@@ -2,13 +2,13 @@
 # $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028
 EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE))
 
-# Allow for a silent mode
-if [[ -v EESSI_SILENT ]]; then
-  # EESSI_SILENT set
-  output=/dev/null
-else
-  output=/dev/stdout
-fi
+function show_msg {
+  # only echo msg if EESSI_SILENT is unset
+  msg=$1
+  if [[ ! -v EESSI_SILENT ]]; then
+    echo "$msg"
+  fi
+}
 
 function error() {
     echo -e "\e[31mERROR: $1\e[0m" >&2
@@ -19,7 +19,7 @@ function error() {
 source $EESSI_INIT_DIR_PATH/minimal_eessi_env
 
 if [ -d $EESSI_PREFIX ]; then
-  echo "Found NESSI repo @ $EESSI_PREFIX!" >> $output
+  show_msg "Found NESSI repo @ $EESSI_PREFIX!"
 
   export EESSI_EPREFIX=$EPREFIX
   if [ -d $EESSI_EPREFIX ]; then
@@ -28,21 +28,21 @@ if [ -d $EESSI_PREFIX ]; then
     if [ "$EESSI_USE_ARCHDETECT" == "1" ]; then
       # if archdetect is enabled, use internal code
       export EESSI_SOFTWARE_SUBDIR=$(${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh cpupath)
-      echo "archdetect says ${EESSI_SOFTWARE_SUBDIR}" >> $output
+      show_msg "archdetect says ${EESSI_SOFTWARE_SUBDIR}"
     elif [ "$EESSI_USE_ARCHSPEC" == "1" ]; then
       # note: eessi_software_subdir_for_host.py will pick up value from $EESSI_SOFTWARE_SUBDIR_OVERRIDE if it's defined!
       export EESSI_EPREFIX_PYTHON=$EESSI_EPREFIX/usr/bin/python3
       export EESSI_SOFTWARE_SUBDIR=$($EESSI_EPREFIX_PYTHON ${EESSI_INIT_DIR_PATH}/eessi_software_subdir_for_host.py $EESSI_PREFIX)
-      echo "archspec says ${EESSI_SOFTWARE_SUBDIR}" >> $output
+      show_msg "archspec says ${EESSI_SOFTWARE_SUBDIR}"
     else
       error "Don't know how to detect host CPU, giving up!"
     fi
     if [ ! -z $EESSI_SOFTWARE_SUBDIR ]; then
 
-        echo "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory." >> $output
+        show_msg "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory."
         export EESSI_SOFTWARE_PATH=$EESSI_PREFIX/software/$EESSI_OS_TYPE/$EESSI_SOFTWARE_SUBDIR
         if [ ! -z $EESSI_BASIC_ENV ]; then
-          echo "Only setting up basic environment, so we're done" >> $output
+          show_msg "Only setting up basic environment, so we're done"
         elif [ -d $EESSI_SOFTWARE_PATH ]; then
           # Allow for the use of a custom MNS
           if [ -z ${EESSI_CUSTOM_MODULEPATH+x} ]; then
@@ -55,21 +55,21 @@ if [ -d $EESSI_PREFIX ]; then
             fi
             EESSI_MODULEPATH=$EESSI_SOFTWARE_PATH/$EESSI_MODULE_SUBDIR
           else
-            echo "Using defined environment variable \$EESSI_CUSTOM_MODULEPATH to set EESSI_MODULEPATH." >> $output
+            show_msg "Using defined environment variable \$EESSI_CUSTOM_MODULEPATH to set EESSI_MODULEPATH."
             EESSI_MODULEPATH=$EESSI_CUSTOM_MODULEPATH
           fi
 
           if [ -d $EESSI_MODULEPATH ]; then
             export EESSI_MODULEPATH=$EESSI_MODULEPATH
-            echo "Using ${EESSI_MODULEPATH} as the directory to be added to MODULEPATH." >> $output
+            show_msg "Using ${EESSI_MODULEPATH} as the directory to be added to MODULEPATH."
           else
             error "NESSI module path at $EESSI_MODULEPATH not found!"
             false
           fi
 
           export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua"
           if [ -f $LMOD_RC ]; then
-            echo "Found Lmod configuration file at $LMOD_RC" >> $output
+            show_msg "Found Lmod configuration file at $LMOD_RC"
           else
             error "Lmod configuration file not found at $LMOD_RC"
           fi