Skip to content

Commit

Permalink
move removal step into installation script
Browse files Browse the repository at this point in the history
  • Loading branch information
truib committed May 19, 2024
1 parent 57555ac commit 9f41c8e
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 39 deletions.
40 changes: 40 additions & 0 deletions EESSI-install-software.sh
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,46 @@ fi
# Don't run the Lmod GPU driver check when doing builds (may not have a GPU, and it's not relevant for vanilla builds anyway)
export EESSI_OVERRIDE_GPU_CHECK=1

# before we actually install software, we need to remove software that is requested
# to be rebuilt (need to do this here because installations of software are read-only;
# also, it should be done in the same container run or fuse-overlayfs might get confused)
changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
if [ -z ${changed_easystacks_rebuilds} ]; then
echo "No software needs to be removed."
else
for easystack_file in ${changed_easystacks_rebuilds}; do
# determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')

# load EasyBuild module (will be installed if it's not available yet)
source ${TOPDIR}/load_easybuild_module.sh ${eb_version}

if [ -f ${easystack_file} ]; then
echo_green "Software rebuild(s) requested in ${easystack_file}, so"
echo_green " determining which existing installation have to be removed (assuming contents"
echo_green " have been made writable/deletable)..."
# we need to remove existing installation directories first,
# so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
# * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
# rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
for app in ${rebuild_apps}; do
app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
ls -lisaR ${app_dir}
ls -lisaR ${app_module}
echo_yellow "Removing ${app_dir} and ${app_module}..."
rm -rdfv ${app_dir}
rm -rdfv ${app_module}
ls -lisaR ${app_dir}
ls -lisaR ${app_module}
done
else
fatal_error "Easystack file ${easystack_file} not found!"
fi
done
fi

# use PR patch file to determine in which easystack files stuff was added
changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing')
if [ -z "${changed_easystacks}" ]; then
Expand Down
81 changes: 42 additions & 39 deletions bot/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -233,31 +233,33 @@ else

ls -lR ${STORAGE}/lower_dirs

# prepare directory to store tarball of tmp for removal and build steps
TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step
mkdir -p ${TARBALL_TMP_REMOVAL_STEP_DIR}

# prepare arguments to eessi_container.sh specific to remove step
declare -a REMOVAL_STEP_ARGS=()
REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_REMOVAL_STEP_DIR}")
REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}")
if [[ ! -z ${LOWER_DIRS} ]]; then
REMOVAL_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
fi

# create tmp file for output of removal step
removal_outerr=$(mktemp remove.outerr.XXXX)

echo "Executing command to remove software:"
echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}"
echo " -- ./EESSI-remove-software.sh \"${REMOVAL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}"
./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \
-- ./EESSI-remove-software.sh "${REMOVAL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr}

# make sure that the build step resumes from the same temporary directory
# this is important, as otherwise the removed software will still be there
REMOVAL_TMPDIR=$(grep ' as tmp directory ' ${removal_outerr} | cut -d ' ' -f 2)
BUILD_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}")
# # prepare directory to store tarball of tmp for removal and build steps
# TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step
# mkdir -p ${TARBALL_TMP_REMOVAL_STEP_DIR}
#
####
# # prepare arguments to eessi_container.sh specific to remove step
# declare -a REMOVAL_STEP_ARGS=()
# REMOVAL_STEP_ARGS+=("--save" "${TARBALL_TMP_REMOVAL_STEP_DIR}")
# REMOVAL_STEP_ARGS+=("--storage" "${STORAGE}")
# if [[ ! -z ${LOWER_DIRS} ]]; then
# REMOVAL_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
# fi
#
# # create tmp file for output of removal step
# removal_outerr=$(mktemp remove.outerr.XXXX)
#
# echo "Executing command to remove software:"
# echo "./eessi_container.sh ${COMMON_ARGS[@]} ${REMOVAL_STEP_ARGS[@]}"
# echo " -- ./EESSI-remove-software.sh \"${REMOVAL_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${removal_outerr}"
# ./eessi_container.sh "${COMMON_ARGS[@]}" "${REMOVAL_STEP_ARGS[@]}" \
# -- ./EESSI-remove-software.sh "${REMOVAL_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${removal_outerr}
#
# # make sure that the build step resumes from the same temporary directory
# # this is important, as otherwise the removed software will still be there
# REMOVAL_TMPDIR=$(grep ' as tmp directory ' ${removal_outerr} | cut -d ' ' -f 2)
# BUILD_STEP_ARGS+=("--resume" "${REMOVAL_TMPDIR}")
####
fi

# prepare directory to store tarball of tmp for build step
Expand All @@ -273,20 +275,21 @@ if [[ ! -z ${SHARED_FS_PATH} ]]; then
BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections")
fi
if [[ ! -z ${LOWER_DIRS} ]]; then
# make copy of LOWER_DIRS but only retain directories
lower_parent_dir=$(dirname ${LOWER_DIRS})
the_lower_dir=$(basename ${LOWER_DIRS})
LOWER_DIRS_ONLY="${lower_parent_dir}/${the_lower_dir}_2"
mkdir -p ${LOWER_DIRS_ONLY}
echo "contents of LOWER_DIRS_ONLY (after mkdir -p)"
ls -lisaR ${LOWER_DIRS_ONLY}
cp -a ${LOWER_DIRS}/. ${LOWER_DIRS_ONLY}
echo "contents of LOWER_DIRS_ONLY (after cp -a)"
ls -lisaR ${LOWER_DIRS_ONLY}
find ${LOWER_DIRS_ONLY} -type f -exec rm {} \;
echo "contents of LOWER_DIRS_ONLY (find ... rm)"
ls -lisaR ${LOWER_DIRS_ONLY}
BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS_ONLY}")
# # make copy of LOWER_DIRS but only retain directories
# lower_parent_dir=$(dirname ${LOWER_DIRS})
# the_lower_dir=$(basename ${LOWER_DIRS})
# LOWER_DIRS_ONLY="${lower_parent_dir}/${the_lower_dir}_2"
# mkdir -p ${LOWER_DIRS_ONLY}
# echo "contents of LOWER_DIRS_ONLY (after mkdir -p)"
# ls -lisaR ${LOWER_DIRS_ONLY}
# cp -a ${LOWER_DIRS}/. ${LOWER_DIRS_ONLY}
# echo "contents of LOWER_DIRS_ONLY (after cp -a)"
# ls -lisaR ${LOWER_DIRS_ONLY}
# find ${LOWER_DIRS_ONLY} -type f -exec rm {} \;
# echo "contents of LOWER_DIRS_ONLY (find ... rm)"
# ls -lisaR ${LOWER_DIRS_ONLY}
# BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS_ONLY}")
BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
fi

# create tmp file for output of build step
Expand Down

0 comments on commit 9f41c8e

Please sign in to comment.