Merge pull request #771 from Neves-P/feature/dev.eessi.io

tweak scripts to support building for `dev.eessi.io` repository

EESSI-install-software.sh

@@ -218,6 +218,11 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then
     fi
 fi
 
+# If in dev.eessi.io, allow building on top of softw
+if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then
+    module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all
+fi
+
 module use $EASYBUILD_INSTALLPATH/modules/all
 
 if [[ -z ${MODULEPATH} ]]; then
@@ -232,7 +237,11 @@ pr_diff=$(ls [0-9]*.diff | head -1)
 # install any additional required scripts
 # order is important: these are needed to install a full CUDA SDK in host_injections
 # for now, this just reinstalls all scripts. Note the most elegant, but works
-${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX}
+
+# Only run install_scripts.sh if not dev.eessi.io for security
+if [[ "${EESSI_CVMFS_REPO}" != /cvmfs/dev.eessi.io ]]; then
+    ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX}
+fi
 
 # Install full CUDA SDK in host_injections
 # Hardcode this for now, see if it works
@@ -262,7 +271,7 @@ if command_exists "nvidia-smi"; then
 fi
 
 # use PR patch file to determine in which easystack files stuff was added
-changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') 
+changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing') 
 if [ -z "${changed_easystacks}" ]; then
     echo "No missing installations, party time!"  # Ensure the bot report success, as there was nothing to be build here
 else
@@ -288,7 +297,7 @@ else
         if [ -f ${easystack_file} ]; then
             echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."
 
-            ${EB} --easystack ${TOPDIR}/${easystack_file} --robot
+            ${EB} --easystack ${easystack_file} --robot
             ec=$?
 
             # copy EasyBuild log file if EasyBuild exited with an error
@@ -301,7 +310,7 @@ else
                 copy_build_log "${eb_last_log}" "${build_logs_dir}"
             fi
 
-            $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} ${TOPDIR}/${pr_diff}
+            $TOPDIR/check_missing_installations.sh ${easystack_file} ${pr_diff}
         else
             fatal_error "Easystack file ${easystack_file} not found!"
         fi

EESSI-remove-software.sh

@@ -102,7 +102,7 @@ pr_diff=$(ls [0-9]*.diff | head -1)
 
 # if this script is run as root, use PR patch file to determine if software needs to be removed first
 if [ $EUID -eq 0 ]; then
-    changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
+    changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
     if [ -z ${changed_easystacks_rebuilds} ]; then
         echo "No software needs to be removed."
     else

bot/build.sh

@@ -21,12 +21,14 @@
 # stop as soon as something fails
 set -e
 
+# Make sure we are referring to software-layer as working directory
+software_layer_dir=$(dirname $(dirname $(realpath $0)))
 # source utils.sh and cfg_files.sh
-source scripts/utils.sh
-source scripts/cfg_files.sh
+source $software_layer_dir/scripts/utils.sh
+source $software_layer_dir/scripts/cfg_files.sh
 
 # defaults
-export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=cfg/job.cfg}"
 HOST_ARCH=$(uname -m)
 
 # check if ${JOB_CFG_FILE} exists
@@ -169,6 +171,12 @@ COMMON_ARGS+=("--mode" "run")
 [[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
 [[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}")
 
+# Also expose software.eessi.io when configured for dev.eessi.io
+# Need software.eessi.io for the compat layer
+if [[ "${REPOSITORY}" == dev.eessi.io ]]; then
+    COMMON_ARGS+=("--repository" "software.eessi.io,access=ro")
+fi
+
 # make sure to use the same parent dir for storing tarballs of tmp
 PREVIOUS_TMP_DIR=${PWD}/previous_tmp
 
@@ -188,7 +196,7 @@ fi
 pr_diff=$(ls [0-9]*.diff | head -1)
 # the true at the end of the next command is important: grep will expectedly return 1 if there is no easystack file being added under rebuilds,
 # but due to "set -e" the entire script would otherwise fail
-changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | (grep "/rebuilds/" || true))
+changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | (grep "/rebuilds/" || true))
 if [[ -z "${changed_easystacks_rebuilds}" ]]; then
     echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step."
 else
@@ -208,10 +216,10 @@ else
     removal_outerr=$(mktemp remove.outerr.XXXX)
 
     echo "Executing command to remove software:"
-    echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}"
-    echo "                     -- ./EESSI-remove-software.sh \"${REMOVAL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}"
-    ./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \
-                         -- ./EESSI-remove-software.sh "${REMOVAL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr}
+    echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}"
+    echo "                     -- $software_layer_dir/EESSI-remove-software.sh \"${REMOVAL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}"
+    $software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \
+                         -- $software_layer_dir/EESSI-remove-software.sh "${REMOVAL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr}
 
     # make sure that the build step resumes from the same temporary directory
     # this is important, as otherwise the removed software will still be there
@@ -244,10 +252,10 @@ fi
 build_outerr=$(mktemp build.outerr.XXXX)
 
 echo "Executing command to build software:"
-echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}"
-echo "                     -- ./install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}"
-./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \
-                     -- ./install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr}
+echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}"
+echo "                     -- $software_layer_dir/install_software_layer.sh \"${INSTALL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${build_outerr}"
+$software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \
+                     -- $software_layer_dir/install_software_layer.sh "${INSTALL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${build_outerr}
 
 # prepare directory to store tarball of tmp for tarball step
 TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step
@@ -272,7 +280,7 @@ fi
 
 timestamp=$(date +%s)
 # to set EESSI_VERSION we need to source init/eessi_defaults now
-source init/eessi_defaults
+source $software_layer_dir/init/eessi_defaults
 export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp})
 
 # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be
@@ -281,9 +289,9 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI
 # /tmp as default?
 TMP_IN_CONTAINER=/tmp
 echo "Executing command to create tarball:"
-echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
-echo "                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
-./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
-                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+echo "$software_layer_dir/eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
+echo "                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+$software_layer_dir/eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
+                     -- $software_layer_dir/create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
 exit 0

check_missing_installations.sh

@@ -26,8 +26,11 @@ easystack=$1
 LOCAL_TMPDIR=$(mktemp -d)
 
 # Clone the develop branch of EasyBuild and use that to search for easyconfigs
-git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs
-export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
+
+if [[ -z ${EASYBUILD_ROBOT_PATHS} ]]; then
+    git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs
+    export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
+fi
 
 # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr
 tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack})

init/minimal_eessi_env

@@ -20,4 +20,9 @@ fi
 export EESSI_CPU_FAMILY=$(uname -m)
 
 # set $EPREFIX since that is basically a standard in Gentoo Prefix
-export EPREFIX=$EESSI_PREFIX/compat/$EESSI_OS_TYPE/$EESSI_CPU_FAMILY
+# if $EESSI_COMPAT_LAYER_DIR is defined (for example by run_in_compat_layer_env.sh script), we use that value
+if [ ! -z ${EESSI_COMPAT_LAYER_DIR} ]; then
+    export EPREFIX=$EESSI_COMPAT_LAYER_DIR
+else
+    export EPREFIX=$EESSI_PREFIX/compat/$EESSI_OS_TYPE/$EESSI_CPU_FAMILY
+fi

install_software_layer.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
-./run_in_compat_layer_env.sh ./EESSI-install-software.sh "$@"
+$base_dir/run_in_compat_layer_env.sh $base_dir/EESSI-install-software.sh "$@"

run_in_compat_layer_env.sh

@@ -7,7 +7,16 @@ if [ -z $EESSI_VERSION ]; then
     echo "ERROR: \$EESSI_VERSION must be set!" >&2
     exit 1
 fi
-EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)"
+
+echo "EESSI_COMPAT_LAYER_DIR_OVERRIDE: ${EESSI_COMPAT_LAYER_DIR_OVERRIDE}" 
+
+if [ ! -z ${EESSI_COMPAT_LAYER_DIR_OVERRIDE} ]; then
+    echo "EESSI_COMPAT_LAYER_DIR_OVERRIDE found. Setting EESSI_COMPAT_LAYER_DIR to ${EESSI_COMPAT_LAYER_DIR_OVERRIDE}"
+    EESSI_COMPAT_LAYER_DIR=${EESSI_COMPAT_LAYER_DIR_OVERRIDE}
+else 
+    EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)"
+fi
+
 if [ ! -d ${EESSI_COMPAT_LAYER_DIR} ]; then
     echo "ERROR: ${EESSI_COMPAT_LAYER_DIR} does not exist!" >&2
     exit 1
@@ -29,6 +38,9 @@ fi
 if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then
     INPUT="export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}; ${INPUT}"
 fi
+if [ ! -z ${EESSI_COMPAT_LAYER_DIR} ]; then
+    INPUT="export EESSI_COMPAT_LAYER_DIR=${EESSI_COMPAT_LAYER_DIR}; ${INPUT}"
+fi
 if [ ! -z ${EESSI_OVERRIDE_GPU_CHECK} ]; then
     INPUT="export EESSI_OVERRIDE_GPU_CHECK=${EESSI_OVERRIDE_GPU_CHECK}; ${INPUT}"
 fi
@@ -38,6 +50,9 @@ fi
 if [ ! -z ${https_proxy} ]; then
     INPUT="export https_proxy=${https_proxy}; ${INPUT}"
 fi
+if [ ! -z ${EASYBUILD_ROBOT_PATHS} ]; then
+    INPUT="export EASYBUILD_ROBOT_PATHS=${EASYBUILD_ROBOT_PATHS}; ${INPUT}"
+fi
 
 echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_VERSION} compatibility layer environment..."
 ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}"