diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 4c80a2649a..1d0f3ed470 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -218,6 +218,46 @@ fi # Don't run the Lmod GPU driver check when doing builds (may not have a GPU, and it's not relevant for vanilla builds anyway) export EESSI_OVERRIDE_GPU_CHECK=1 +# before we actually install software, we need to remove software that is requested +# to be rebuilt (need to do this here because installations of software are read-only; +# also, it should be done in the same container run or fuse-overlayfs might get confused) +changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/") +if [ -z ${changed_easystacks_rebuilds} ]; then + echo "No software needs to be removed." +else + for easystack_file in ${changed_easystacks_rebuilds}; do + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file + eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') + + # load EasyBuild module (will be installed if it's not available yet) + source ${TOPDIR}/load_easybuild_module.sh ${eb_version} + + if [ -f ${easystack_file} ]; then + echo_green "Software rebuild(s) requested in ${easystack_file}, so" + echo_green " determining which existing installation have to be removed (assuming contents" + echo_green " have been made writable/deletable)..." + # we need to remove existing installation directories first, + # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R]) + # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) + # rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') + for app in ${rebuild_apps}; do + app_dir=${EASYBUILD_INSTALLPATH}/software/${app} + app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua + ls -lisaR ${app_dir} + ls -lisaR ${app_module} + echo_yellow "Removing ${app_dir} and ${app_module}..." + rm -rdfv ${app_dir} + rm -rdfv ${app_module} + ls -lisaR ${app_dir} + ls -lisaR ${app_module} + done + else + fatal_error "Easystack file ${easystack_file} not found!" + fi + done +fi + # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') if [ -z "${changed_easystacks}" ]; then diff --git a/bot/build.sh b/bot/build.sh index a12d3669fc..2b08a8599c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -233,31 +233,33 @@ else ls -lR ${STORAGE}/lower_dirs - # prepare directory to store tarball of tmp for removal and build steps - TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step - mkdir -p ${TARBALL_TMP_REMOVAL_STEP_DIR} - - # prepare arguments to eessi_container.sh specific to remove step - declare -a REMOVAL_STEP_ARGS=() - REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_REMOVAL_STEP_DIR}") - REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}") - if [[ ! -z ${LOWER_DIRS} ]]; then - REMOVAL_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}") - fi - - # create tmp file for output of removal step - removal_outerr=$(mktemp remove.outerr.XXXX) - - echo "Executing command to remove software:" - echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}" - echo " -- ./EESSI-remove-software.sh \"${REMOVAL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}" - ./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \ - -- ./EESSI-remove-software.sh "${REMOVAL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr} - - # make sure that the build step resumes from the same temporary directory - # this is important, as otherwise the removed software will still be there - REMOVAL_TMPDIR=$(grep ' as tmp directory ' ${removal_outerr} | cut -d ' ' -f 2) - BUILD_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}") +# # prepare directory to store tarball of tmp for removal and build steps +# TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step +# mkdir -p ${TARBALL_TMP_REMOVAL_STEP_DIR} +# +#### +# # prepare arguments to eessi_container.sh specific to remove step +# declare -a REMOVAL_STEP_ARGS=() +# REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_REMOVAL_STEP_DIR}") +# REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}") +# if [[ ! -z ${LOWER_DIRS} ]]; then +# REMOVAL_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}") +# fi +# +# # create tmp file for output of removal step +# removal_outerr=$(mktemp remove.outerr.XXXX) +# +# echo "Executing command to remove software:" +# echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}" +# echo " -- ./EESSI-remove-software.sh \"${REMOVAL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}" +# ./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \ +# -- ./EESSI-remove-software.sh "${REMOVAL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr} +# +# # make sure that the build step resumes from the same temporary directory +# # this is important, as otherwise the removed software will still be there +# REMOVAL_TMPDIR=$(grep ' as tmp directory ' ${removal_outerr} | cut -d ' ' -f 2) +# BUILD_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}") +#### fi # prepare directory to store tarball of tmp for build step @@ -273,20 +275,21 @@ if [[ ! -z ${SHARED_FS_PATH} ]]; then BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections") fi if [[ ! -z ${LOWER_DIRS} ]]; then - # make copy of LOWER_DIRS but only retain directories - lower_parent_dir=$(dirname ${LOWER_DIRS}) - the_lower_dir=$(basename ${LOWER_DIRS}) - LOWER_DIRS_ONLY="${lower_parent_dir}/${the_lower_dir}_2" - mkdir -p ${LOWER_DIRS_ONLY} - echo "contents of LOWER_DIRS_ONLY (after mkdir -p)" - ls -lisaR ${LOWER_DIRS_ONLY} - cp -a ${LOWER_DIRS}/. ${LOWER_DIRS_ONLY} - echo "contents of LOWER_DIRS_ONLY (after cp -a)" - ls -lisaR ${LOWER_DIRS_ONLY} - find ${LOWER_DIRS_ONLY} -type f -exec rm {} \; - echo "contents of LOWER_DIRS_ONLY (find ... rm)" - ls -lisaR ${LOWER_DIRS_ONLY} - BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS_ONLY}") +# # make copy of LOWER_DIRS but only retain directories +# lower_parent_dir=$(dirname ${LOWER_DIRS}) +# the_lower_dir=$(basename ${LOWER_DIRS}) +# LOWER_DIRS_ONLY="${lower_parent_dir}/${the_lower_dir}_2" +# mkdir -p ${LOWER_DIRS_ONLY} +# echo "contents of LOWER_DIRS_ONLY (after mkdir -p)" +# ls -lisaR ${LOWER_DIRS_ONLY} +# cp -a ${LOWER_DIRS}/. ${LOWER_DIRS_ONLY} +# echo "contents of LOWER_DIRS_ONLY (after cp -a)" +# ls -lisaR ${LOWER_DIRS_ONLY} +# find ${LOWER_DIRS_ONLY} -type f -exec rm {} \; +# echo "contents of LOWER_DIRS_ONLY (find ... rm)" +# ls -lisaR ${LOWER_DIRS_ONLY} +# BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS_ONLY}") + BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}") fi # create tmp file for output of build step