revert to using standard method (fakeroot) to remove existing install…

…ation

EESSI-remove-software.sh

@@ -101,7 +101,7 @@ fi
 pr_diff=$(ls [0-9]*.diff | head -1)
 
 # if this script is run as root, use PR patch file to determine if software needs to be removed first
-if [ $EUID -ne 0 ]; then
+if [ $EUID -eq 0 ]; then
     changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep 'easystacks/.*yml$' | egrep -v 'known-issues|missing' | grep "/rebuilds/")
     if [ -z ${changed_easystacks_rebuilds} ]; then
         echo "No software needs to be removed."
@@ -114,57 +114,30 @@ if [ $EUID -ne 0 ]; then
             source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
 
             if [ -f ${easystack_file} ]; then
-                echo_green "Software rebuild(s) requested in ${easystack_file}, so"
-                echo_green "  determining which existing installation have to be removed (assuming contents"
-                echo_green "  have been made writable/deletable)..."
+                echo_green "Software rebuild(s) requested in ${easystack_file}, so determining which existing installation have to be removed..."
                 # we need to remove existing installation directories first,
                 # so let's figure out which modules have to be rebuilt by doing a dry-run and grepping "someapp/someversion" for the relevant lines (with [R])
                 #  * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion)
-                # rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
-                rebuild_apps=$(eb --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
+                rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}')
                 for app in ${rebuild_apps}; do
                     # Returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all:
                     app_modulepath=$(module --terse av ${app} 2>&1 | head -n 1 | sed 's/://')
                     # Two dirname invocations, so returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2
                     app_installprefix=$(dirname $(dirname ${app_modulepath}))
                     app_dir=${app_installprefix}/software/${app}
                     app_module=${app_installprefix}/modules/all/${app}.lua
-                    # app_dir=${EASYBUILD_INSTALLPATH}/software/${app}
-                    # app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua
                     echo_yellow "Removing ${app_dir} and ${app_module}..."
-                    # suggestion: use the recursive rm's and ls a specific
-                    #   directory only (${app_dir}/easybuild)
-                    rm -rdfv ${app_dir}
-                    rm -rdfv ${app_module}
-                    echo_yellow "Contents of ${app_dir}/easybuild (should not exist)"
-                    ls -l ${app_dir}/easybuild || true
-                    # ls didn't change the result (permission denied)
-                    # ls ${app_dir}/easybuild || true
-                    # 2nd idea: recreate some directory
+                    rm -rf ${app_dir}
+                    rm -rf ${app_module}
+                    # recreate some directory to work around permission denied
+                    # issues when rebuilding the package
                     mkdir -p ${app_dir}/easybuild
-                    echo_yellow "Contents of ${app_dir}/easybuild after it got recreated with 'mkdir -p' (should be empty)"
-                    ls -l ${app_dir}/easybuild || true
-
-                    ## 1st remove files in depth-first order
-                    #for filepath in $(find ${app_dir} -depth -type f); do
-                    #    echo "  removing file ${filepath}"
-                    #    rm -fv ${filepath}
-                    #done
-                    ## 2nd remove directories in depth-first order
-                    #for dirpath in $(find ${app_dir} -depth -type d); do
-                    #    echo "  removing directory ${dirpath}"
-                    #    rmdir -v ${dirpath}
-                    #done
-                    ## 3rd remove module file
-                    #echo "  removing module file ${app_module}"
-                    #rm -fv ${app_module}
                 done
             else
                 fatal_error "Easystack file ${easystack_file} not found!"
             fi
         done
     fi
 else
-    # fatal_error "This script can only be run by root!"
-    fatal_error "This script must not be run by root!"
+    fatal_error "This script can only be run by root!"
 fi

bot/build.sh

@@ -200,62 +200,6 @@ changed_easystacks_rebuilds=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed
 if [[ -z "${changed_easystacks_rebuilds}" ]]; then
     echo "This PR does not add any easystack files in a rebuilds subdirectory, so let's skip the removal step."
 else
-    # determine which software packages (and modules) have to be removed
-    TARBALL_TMP_DETERMINE_STEP_DIR=${PREVIOUS_TMP_DIR}/determine_step
-    mkdir -p ${TARBALL_TMP_DETERMINE_STEP_DIR}
-
-    # prepare arguments to eessi_container.sh specific to determine step
-    declare -a DETERMINE_STEP_ARGS=()
-    DETERMINE_STEP_ARGS+=("--save" "${TARBALL_TMP_DETERMINE_STEP_DIR}")
-    DETERMINE_STEP_ARGS+=("--storage" "${STORAGE}")
-
-    # create tmp file for output of determine step
-    determine_outerr=$(mktemp determine.outerr.XXXX)
-
-    echo "Executing command to determine software to be removed:"
-    echo "${software_layer_dir}/eessi_container.sh ${COMMON_ARGS[@]} ${DETERMINE_STEP_ARGS[@]}"
-    echo "                     -- ${software_layer_dir}/EESSI-determine-rebuilds.sh \"${DETERMINE_SCRIPT_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${determine_outerr}"
-    ${software_layer_dir}/eessi_container.sh "${COMMON_ARGS[@]}" "${DETERMINE_STEP_ARGS[@]}" \
-                         -- ${software_layer_dir}/EESSI-determine-rebuilds.sh "${DETERMINE_SCRIPT_ARGS[@]}" "$@" 2>&1 | tee -a ${determine_outerr}
-
-    # process output file
-    #   for each line containing 'REMOVE_DIRECTORY some_path'
-    #     create a new directory ${STORAGE}/lower_dirs/some_path_stripped
-    #       where the prefix /cvmfs/repo_name is removed from some_path
-    #     set permission of the directory to ug+rwx
-    # SKIP  for each line containing 'REMOVE_FILE some_file_path'
-    # SKIP    touch a new file ${STORAGE}/lower_dirs/some_file_path_stripped
-    # SKIP      where the prefix /cvmfs/repo_name is removed from some_file_path
-    # SKIP    set permission of the file to ug+rw
-    #   for each line containing 'REMOVE_MODULE some_file_path'
-    #     touch a new file ${STORAGE}/lower_dirs/some_file_path_stripped
-    #       where the prefix /cvmfs/repo_name is removed from some_file_path
-    #     set permission of the file to ug+rw
-
-    LOWER_DIRS="${STORAGE}/lower_dirs"
-    mkdir -p "${LOWER_DIRS}"
-
-    grep ^REMOVE_DIRECTORY ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_dirs
-    cat ${determine_outerr}.rm_dirs | while read remove_dir; do
-        mkdir -p ${STORAGE}/lower_dirs/${remove_dir}
-        chmod ug+rwx ${STORAGE}/lower_dirs/${remove_dir}
-    done
-
-    # grep ^REMOVE_FILE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_files
-    # cat ${determine_outerr}.rm_files | while read remove_file; do
-    #     touch ${STORAGE}/lower_dirs/${remove_file}
-    #     chmod ug+rw ${STORAGE}/lower_dirs/${remove_file}
-    # done
-
-    grep ^REMOVE_MODULE ${determine_outerr} | cut -f4- -d'/' > ${determine_outerr}.rm_modules
-    cat ${determine_outerr}.rm_modules | while read remove_module; do
-        module_parent_dir=$(dirname ${STORAGE}/lower_dirs/${remove_module})
-        mkdir -p ${module_parent_dir}
-        chmod ug+rw ${module_parent_dir}
-        touch ${STORAGE}/lower_dirs/${remove_module}
-        chmod ug+rw ${STORAGE}/lower_dirs/${remove_module}
-    done
-
     # prepare directory to store tarball of tmp for removal and build steps
     TARBALL_TMP_REMOVAL_STEP_DIR=${PREVIOUS_TMP_DIR}/removal_step
     mkdir -p ${TARBALL_TMP_REMOVAL_STEP_DIR}
@@ -267,11 +211,7 @@ else
 
     # add fakeroot option in order to be able to remove software, see:
     # https://github.com/EESSI/software-layer/issues/312
-    # REMOVAL_STEP_ARGS+=("--fakeroot")
-
-    if [[ ! -z ${LOWER_DIRS} ]]; then
-        REMOVAL_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
-    fi
+    REMOVAL_STEP_ARGS+=("--fakeroot")
 
     # create tmp file for output of removal step
     removal_outerr=$(mktemp remove.outerr.XXXX)
@@ -309,10 +249,6 @@ if [[ ! -z ${SHARED_FS_PATH} ]]; then
     BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections")
 fi
 
-if [[ ! -z ${LOWER_DIRS} ]]; then
-    BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
-fi
-
 # create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)
 

easystacks/software.eessi.io/2023.06/rebuilds/20241112-eb-4.9.4-EESSI-extend.yml

@@ -3,6 +3,8 @@
 # EESSI-extend should adjust EASYBUILD_INSTALLPATH and set
 # EASYBUILD_CUDA_COMPUTE_CAPABILITIES
 easyconfigs:
-  - EESSI-extend-2023.06-easybuild.eb:
-      options:
-        try-amend: keeppreviousinstall=True
+  - EESSI-extend-2023.06-easybuild.eb
+# the options are added to load_eessi_extend_module.sh
+#  - EESSI-extend-2023.06-easybuild.eb:
+#      options:
+#        try-amend: keeppreviousinstall=True

eessi_container.sh

@@ -89,11 +89,6 @@ display_help() {
   echo "  -n | --nvidia MODE      - configure the container to work with NVIDIA GPUs,"
   echo "                            MODE==install for a CUDA installation, MODE==run to"
   echo "                            attach a GPU, MODE==all for both [default: false]"
-  echo "  -o | --lower-dirs DIRS  - list of ':' separated directories that are used"
-  echo "                            in front of the default lower dir (CVMFS repo);"
-  echo "                            fuse-overlayfs will merge all lower directories;"
-  echo "                            the option can be used to make certain directories"
-  echo "                            in the CVMFS repo writable [default: none]"
   echo "  -r | --repository CFG   - configuration file or identifier defining the"
   echo "                            repository to use; can be given multiple times;"
   echo "                            CFG may include a suffix ',access={ro,rw}' to"
@@ -130,7 +125,6 @@ FAKEROOT=0
 VERBOSE=0
 STORAGE=
 LIST_REPOS=0
-LOWER_DIRS=
 MODE="shell"
 SETUP_NVIDIA=0
 REPOSITORIES=()
@@ -188,10 +182,6 @@ while [[ $# -gt 0 ]]; do
       NVIDIA_MODE="$2"
       shift 2
       ;;
-    -o|--lower-dirs)
-      LOWER_DIRS="$2"
-      shift 2
-      ;;
     -r|--repository)
       REPOSITORIES+=("$2")
       shift 2
@@ -799,14 +789,7 @@ do
         EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")
 
         EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs"
-        lowerdirs=/cvmfs_ro/${cvmfs_repo_name}
-        if [[ ! -z ${LOWER_DIRS} ]]; then
-            # need to convert ':' in LOWER_DIRS to ',' because bind mounts use ',' as
-            # separator while the lowerdir overlayfs option uses ':'
-            export BIND_PATHS="${BIND_PATHS},${LOWER_DIRS/:/,}"
-            lowerdirs=${LOWER_DIRS}:${lowerdirs}
-        fi
-        EESSI_WRITABLE_OVERLAY+=" -o lowerdir=${lowerdirs}"
+        EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}"
         EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper"
         EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work"
         EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"

load_eessi_extend_module.sh

@@ -91,6 +91,8 @@ else
         eb_install_out=${TMPDIR}/eb_install.out
         ok_msg="EESSI-extend/${EESSI_EXTEND_VERSION} installed, let's go!"
         fail_msg="Installing EESSI-extend/${EESSI_EXTEND_VERSION} failed, that's not good... (output: ${eb_install_out})"
+        # while always adding --try-amend=keep... may do no harm, we could make
+        # an attempt to figure out if it is needed, e.g., when we are rebuilding
         ${EB} "EESSI-extend-${EESSI_EXTEND_VERSION}.eb" --try-amend=keeppreviousinstall=True 2>&1 | tee ${eb_install_out}
         check_exit_code $? "${ok_msg}" "${fail_msg}"
     )