Merge pull request EESSI#217 from TopRichard/nessi.no-2023.06-Directo…

…ry-structure

Modifying directory structure + drop pilot terminology

.github/workflows/tests_readme.yml

@@ -19,10 +19,10 @@ jobs:
         - name: Check out software-layer repository
           uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
 
-        - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults
+        - name: verify if README.md is consistent with EESSI_VERSION from init/eessi_defaults
           run: |
             source init/eessi_defaults
-            grep "${EESSI_PILOT_VERSION}" README.md
+            grep "${EESSI_VERSION}" README.md
 
         - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults
           run: |

.github/workflows/tests_scripts.yml

@@ -79,20 +79,18 @@ jobs:
 
     - name: test install_software_layer.sh script
       run: |
-          # scripts need to be copied to /tmp,
-          # since install_software_layer.sh must be accessible from within build container
-          cp -a * /tmp/
-          cd /tmp
+          # bind current directory into container as /software-layer
+          export SINGULARITY_BIND="${PWD}:/software-layer"
           # force using x86_64/generic, to avoid triggering an installation from scratch
-          sed -i "s@./EESSI-pilot-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-pilot-install-software.sh\"@g" install_software_layer.sh
-          ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh
+          sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh
+          ./eessi_container.sh --mode run --verbose /software-layer/install_software_layer.sh
 
     - name: test create_directory_tarballs.sh script
       run: |
+          # bind current directory into container as /software-layer
+          export SINGULARITY_BIND="${PWD}:/software-layer"
           # scripts need to be copied to /tmp,
           # since create_directory_tarballs.sh must be accessible from within build container
-          cp -a * /tmp/
-          cd /tmp
-          ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2023.06
+          ./eessi_container.sh --mode run --verbose /software-layer/create_directory_tarballs.sh 2023.06
           # check if tarballs have been produced
           ls -l *.tar.gz

EESSI-pilot-install-software.sh

@@ -188,7 +188,7 @@ fi
 pr_diff=$(ls [0-9]*.diff | head -1)
 
 # use PR patch file to determine in which easystack files stuff was added
-for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$'); do
+for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing'); do
 
     echo -e "Processing easystack file ${easystack_file}...\n\n"
 

README.md

@@ -1,8 +1,8 @@
 # Software layer
 
-The software layer of the EESSI project uses [EasyBuild](https://easybuild.readthedocs.io), [Lmod](https://lmod.readthedocs.io) and [archspec](https://archspec.readthedocs.io).
+The software layer of the EESSI project uses [EasyBuild](https://docs.easybuild.io), [Lmod](https://lmod.readthedocs.io) and [archspec](https://archspec.readthedocs.io).
 
-See also https://eessi.github.io/docs/software_layer.
+See also https://www.eessi.io/docs/software_layer .
 
 ## Pilot software stack
 
@@ -12,7 +12,7 @@ You can set up your environment by sourcing the init script:
 $ source /cvmfs/pilot.nessi.no/versions/2023.06/init/bash
 Found EESSI pilot repo @ /cvmfs/pilot.nessi.no/versions/2023.06!
 Derived subdirectory for software layer: x86_64/intel/broadwell
-Using x86_64/intel/broadwell subdirectory for software layer (HARDCODED)
+Using x86_64/intel/haswell subdirectory for software layer
 Initializing Lmod...
 Prepending /cvmfs/pilot.nessi.no/versions/2023.06/software/x86_64/intel/broadwell/modules/all to $MODULEPATH...
 Environment set up to use EESSI pilot software stack, have fun!

init/Magic_Castle/bash

@@ -6,7 +6,7 @@ EESSI_SILENT=1
 source $(dirname "$BASH_SOURCE")/../eessi_environment_variables
 
 # Don't change the default prompt
-# export PS1="[EESSI pilot $EESSI_PILOT_VERSION] $ "
+# export PS1="[EESSI pilot $EESSI_VERSION] $ "
 
 # Provide a clean MODULEPATH
 export MODULEPATH_ROOT=$EESSI_MODULEPATH
@@ -36,4 +36,4 @@ else
    module reload
 fi
 
-echo "Environment set up to use EESSI pilot software stack (${EESSI_PILOT_VERSION}), have fun!"
+echo "Environment set up to use EESSI (${EESSI_VERSION}), have fun!"

init/Magic_Castle/eessi_pilot_python3 → init/Magic_Castle/eessi_python3

@@ -12,10 +12,10 @@ export EESSI_SILENT=1
 # for MacOS due to the use of `readlink`)
 source $(dirname "$(readlink -f "$BASH_SOURCE")")/../eessi_environment_variables
 
-eessi_pilot_python=$(ls ${EESSI_SOFTWARE_PATH}/software/Python/3*GCCcore*/bin/python | sed 1q)
-if [ -f "$eessi_pilot_python" ]; then
-  $eessi_pilot_python "$@"
+eessi_python=$(ls ${EESSI_SOFTWARE_PATH}/software/Python/3*GCCcore*/bin/python | sed 1q)
+if [ -f "$eessi_python" ]; then
+  $eessi_python "$@"
 else
-  echo "ERROR: No EESSI pilot python 3 available."
+  echo "ERROR: No EESSI Python 3 available."
   false
 fi

init/bash

@@ -13,7 +13,7 @@ source $(dirname "$BASH_SOURCE")/eessi_environment_variables
 # only continue if setting EESSI environment variables worked fine
 if [ $? -eq 0 ]; then
 
-    export PS1="[EESSI pilot $EESSI_PILOT_VERSION] $ "
+    export PS1="{EESSI $EESSI_VERSION} $PS1"
 
     # add location of commands provided by compat layer to $PATH;
     # see https://github.com/EESSI/software-layer/issues/52
@@ -28,12 +28,12 @@ if [ $? -eq 0 ]; then
     module use $EESSI_MODULEPATH
 
     #echo >> $output
-    #echo "*** Known problems in the ${EESSI_PILOT_VERSION} pilot software stack ***" >> $output
+    #echo "*** Known problems in the ${EESSI_VERSION} pilot software stack ***" >> $output
     #echo >> $output
     #echo "1) ..." >> $output
     #echo >> $output
     #echo >> $output
 
-    echo "Environment set up to use EESSI pilot software stack, have fun!" >> $output
+    echo "Environment set up to use EESSI (${EESSI_VERSION}), have fun!"
 
 fi

init/eessi_defaults

@@ -9,4 +9,7 @@
 #
 
 export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.nessi.no}"
-export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2023.06}"
+export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=2023.06}"
+# use archdetect by default, unless otherwise specified
+export EESSI_USE_ARCHDETECT="${EESSI_USE_ARCHDETECT:=1}"
+export EESSI_USE_ARCHSPEC="${EESSI_USE_ARCHSPEC:=0}"

init/eessi_environment_variables

@@ -15,11 +15,11 @@ function error() {
     false
 }
 
-# set up minimal environment: $EESSI_PREFIX, $EESSI_PILOT_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX
+# set up minimal environment: $EESSI_PREFIX, $EESSI_VERSION, $EESSI_OS_TYPE, $EESSI_CPU_FAMILY, $EPREFIX
 source $EESSI_INIT_DIR_PATH/minimal_eessi_env
 
 if [ -d $EESSI_PREFIX ]; then
-  echo "Found EESSI pilot repo @ $EESSI_PREFIX!" >> $output
+  echo "Found EESSI repo @ $EESSI_PREFIX!" >> $output
 
   export EESSI_EPREFIX=$EPREFIX
   if [ -d $EESSI_EPREFIX ]; then
@@ -29,11 +29,13 @@ if [ -d $EESSI_PREFIX ]; then
       # if archdetect is enabled, use internal code
       export EESSI_SOFTWARE_SUBDIR=$(${EESSI_INIT_DIR_PATH}/eessi_archdetect.sh cpupath)
       echo "archdetect says ${EESSI_SOFTWARE_SUBDIR}" >> $output
-    else
+    elif [ "$EESSI_USE_ARCHSPEC" == "1" ]; then
       # note: eessi_software_subdir_for_host.py will pick up value from $EESSI_SOFTWARE_SUBDIR_OVERRIDE if it's defined!
       export EESSI_EPREFIX_PYTHON=$EESSI_EPREFIX/usr/bin/python3
       export EESSI_SOFTWARE_SUBDIR=$($EESSI_EPREFIX_PYTHON ${EESSI_INIT_DIR_PATH}/eessi_software_subdir_for_host.py $EESSI_PREFIX)
       echo "archspec says ${EESSI_SOFTWARE_SUBDIR}" >> $output
+    else
+      error "Don't know how to detect host CPU, giving up!"
     fi
     if [ ! -z $EESSI_SOFTWARE_SUBDIR ]; then
 
@@ -82,5 +84,5 @@ if [ -d $EESSI_PREFIX ]; then
     error "Compatibility layer directory $EESSI_EPREFIX not found!"
   fi
 else
-  error "EESSI pilot repository at $EESSI_PREFIX not found!"
+  error "EESSI repository at $EESSI_PREFIX not found!"
 fi

init/minimal_eessi_env

@@ -4,11 +4,11 @@
 # $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028
 EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE))
 
-# set up defaults: EESSI_CVMFS_REPO, EESSI_PILOT_VERSION
+# set up defaults: EESSI_CVMFS_REPO, EESSI_VERSION
 #   script takes *_OVERRIDEs into account
 source ${EESSI_INIT_DIR_PATH}/eessi_defaults
 
-export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_PILOT_VERSION
+export EESSI_PREFIX=$EESSI_CVMFS_REPO/versions/$EESSI_VERSION
 
 if [[ $(uname -s) == 'Linux' ]]; then
     export EESSI_OS_TYPE='linux'

load_easybuild_module.sh

@@ -71,21 +71,28 @@ else
     check_exit_code $? "${ok_msg}" "${fail_msg}"
 
     # maybe the module obtained with --install-latest-eb-release is exactly the EasyBuild version we wanted?
+    IGNORE_CACHE=''
     module avail 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out}
     if [[ $? -eq 0 ]]; then
         echo_green ">> Module for EasyBuild v${EB_VERSION} found!"
     else
-        eb_ec=EasyBuild-${EB_VERSION}.eb
-        echo_yellow ">> Still no module for EasyBuild v${EB_VERSION}, trying with easyconfig ${eb_ec}..."
-        ${EB} --search ${eb_ec} | grep ${eb_ec} > /dev/null
+        module --ignore_cache avail 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out}
         if [[ $? -eq 0 ]]; then
-            echo "Easyconfig ${eb_ec} found for EasyBuild v${EB_VERSION}, so installing it..."
-            ok_msg="EasyBuild v${EB_VERSION} installed, alright!"
-            fail_msg="Installing EasyBuild v${EB_VERSION}, yikes! (output: ${eb_install_out})"
-            ${EB} EasyBuild-${EB_VERSION}.eb 2>&1 | tee -a ${eb_install_out}
-            check_exit_code $? "${ok_msg}" "${fail_msg}"
+            echo_green ">> Module for EasyBuild v${EB_VERSION} found!"
+            IGNORE_CACHE='--ignore_cache'
         else
-            fatal_error "No easyconfig found for EasyBuild v${EB_VERSION}"
+            eb_ec=EasyBuild-${EB_VERSION}.eb
+            echo_yellow ">> Still no module for EasyBuild v${EB_VERSION}, trying with easyconfig ${eb_ec}..."
+            ${EB} --search ${eb_ec} | grep ${eb_ec} > /dev/null
+            if [[ $? -eq 0 ]]; then
+                echo "Easyconfig ${eb_ec} found for EasyBuild v${EB_VERSION}, so installing it..."
+                ok_msg="EasyBuild v${EB_VERSION} installed, alright!"
+                fail_msg="Installing EasyBuild v${EB_VERSION}, yikes! (output: ${eb_install_out})"
+                ${EB} EasyBuild-${EB_VERSION}.eb 2>&1 | tee -a ${eb_install_out}
+                check_exit_code $? "${ok_msg}" "${fail_msg}"
+            else
+                fatal_error "No easyconfig found for EasyBuild v${EB_VERSION}"
+            fi
         fi
     fi
 
@@ -103,7 +110,7 @@ else
 fi
 
 echo ">> Loading EasyBuild v${EB_VERSION} module..."
-module load EasyBuild/${EB_VERSION}
+module ${IGNORE_CACHE} load EasyBuild/${EB_VERSION}
 eb_show_system_info_out=${TMPDIR}/eb_show_system_info.out
 ${EB} --show-system-info > ${eb_show_system_info_out}
 if [[ $? -eq 0 ]]; then

run_in_compat_layer_env.sh

@@ -3,25 +3,28 @@
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
 
-if [ -z $EESSI_PILOT_VERSION ]; then
-    echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2
+if [ -z $EESSI_VERSION ]; then
+    echo "ERROR: \$EESSI_VERSION must be set!" >&2
     exit 1
 fi
-EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)"
+EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)"
 if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then
     echo "ERROR: ${EESSI_COMPAT_LAYER_DIR} does not exist!" >&2
     exit 1
 fi
 
 INPUT=$(echo "$@")
+if [ ! -z ${SLURM_JOB_ID} ]; then
+    INPUT="export SLURM_JOB_ID=${SLURM_JOB_ID}; ${INPUT}"
+fi
 if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
     INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}"
 fi
 if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then
     INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}"
 fi
-if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then
-    INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}"
+if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then
+    INPUT="export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}; ${INPUT}"
 fi
 if [ ! -z ${http_proxy} ]; then
     INPUT="export http_proxy=${http_proxy}; ${INPUT}"
@@ -30,5 +33,5 @@ if [ ! -z ${https_proxy} ]; then
     INPUT="export https_proxy=${https_proxy}; ${INPUT}"
 fi
 
-echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..."
+echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_VERSION} compatibility layer environment..."
 ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}"