From 4e2f6d046a673f7d22106c505b116e5dc2cb7de5 Mon Sep 17 00:00:00 2001 From: Michael Huebner Date: Tue, 20 Dec 2022 10:23:17 +0100 Subject: [PATCH 001/106] Move bash utils to new scripts dir, add new dir to tarball creation --- .github/workflows/tests_scripts.yml | 4 +-- EESSI-pilot-install-software.sh | 2 +- ...tarball.sh => create_directory_tarballs.sh | 33 ++++++++++--------- utils.sh => scripts/utils.sh | 0 update_lmod_cache.sh | 2 +- 5 files changed, 22 insertions(+), 19 deletions(-) rename create_init_tarball.sh => create_directory_tarballs.sh (52%) rename utils.sh => scripts/utils.sh (100%) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 18fcd7b255..d224034c2a 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -6,7 +6,7 @@ on: - build_container.sh - install_software_layer.sh - run_in_compat_layer_env.sh - - utils.sh + - scripts/utils.sh - update_lmod_cache.sh pull_request: @@ -16,7 +16,7 @@ on: - build_container.sh - install_software_layer.sh - run_in_compat_layer_env.sh - - utils.sh + - scripts/utils.sh - update_lmod_cache.sh permissions: contents: read # to fetch code (actions/checkout) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 10311b7a15..4ca553dbe7 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -50,7 +50,7 @@ set -- "${POSITIONAL_ARGS[@]}" TOPDIR=$(dirname $(realpath $0)) -source $TOPDIR/utils.sh +source $TOPDIR/scripts/utils.sh # honor $TMPDIR if it is already defined, use /tmp otherwise if [ -z $TMPDIR ]; then diff --git a/create_init_tarball.sh b/create_directory_tarballs.sh similarity index 52% rename from create_init_tarball.sh rename to create_directory_tarballs.sh index 3393285a33..c254c2c229 100755 --- a/create_init_tarball.sh +++ b/create_directory_tarballs.sh @@ -4,19 +4,6 @@ SOFTWARE_LAYER_TARBALL_URL=https://github.com/EESSI/software-layer/tarball/main set -eo pipefail -function echo_green() { - echo -e "\e[32m$1\e[0m" -} - -function echo_red() { - echo -e "\e[31m$1\e[0m" -} - -function error() { - echo_red "ERROR: $1" >&2 - exit 1 -} - if [ $# -ne 1 ]; then echo "Usage: $0 " >&2 exit 1 @@ -24,13 +11,18 @@ fi version=$1 +TOPDIR=$(dirname $(realpath $0)) + +source $TOPDIR/scripts/utils.sh + # Check if the EESSI version number encoded in the filename # is valid, i.e. matches the format YYYY.DD if ! echo "${version}" | egrep -q '^20[0-9][0-9]\.(0[0-9]|1[0-2])$' then - error "${version} is not a valid EESSI version." + fatal_error "${version} is not a valid EESSI version." fi +# Create tarball of init directory tartmp=$(mktemp -t -d init.XXXXX) mkdir "${tartmp}/${version}" tarname="eessi-${version}-init-$(date +%s).tar.gz" @@ -38,9 +30,20 @@ curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --s source "${tartmp}/${version}/init/minimal_eessi_env" if [ "${EESSI_PILOT_VERSION}" != "${version}" ] then - error "Specified version ${version} does not match version ${EESSI_PILOT_VERSION} in the init files!" + fatal_error "Specified version ${version} does not match version ${EESSI_PILOT_VERSION} in the init files!" fi tar czf "${tarname}" -C "${tartmp}" "${version}" rm -rf "${tartmp}" echo_green "Done! Created tarball ${tarname}." + +# Create tarball of scripts directory +# Version check has already been performed and would have caused script to exit at this point in case of problems +tartmp=$(mktemp -t -d scripts.XXXXX) +mkdir "${tartmp}/${version}" +tarname="eessi-${version}-scripts-$(date +%s).tar.gz" +curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards */scripts/ +tar czf "${tarname}" -C "${tartmp}" "${version}" +rm -rf "${tartmp}" + +echo_green "Done! Created tarball ${tarname}." diff --git a/utils.sh b/scripts/utils.sh similarity index 100% rename from utils.sh rename to scripts/utils.sh diff --git a/update_lmod_cache.sh b/update_lmod_cache.sh index 89e2ecbeee..814bb2dae0 100755 --- a/update_lmod_cache.sh +++ b/update_lmod_cache.sh @@ -5,7 +5,7 @@ TOPDIR=$(dirname $(realpath $0)) -source $TOPDIR/utils.sh +source $TOPDIR/scripts/utils.sh if [ $# -ne 2 ]; then echo "Usage: $0 " >&2 From 2caad9bc9cc99544542725894c7ee11c8b6ea701 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 20:42:40 +0100 Subject: [PATCH 002/106] initial version of unified script - should support existing ways to access EESSI - no additional features implemented yet (--help may hint at new features) --- eessi_container.sh | 311 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 311 insertions(+) create mode 100755 eessi_container.sh diff --git a/eessi_container.sh b/eessi_container.sh new file mode 100755 index 0000000000..9219581582 --- /dev/null +++ b/eessi_container.sh @@ -0,0 +1,311 @@ +#!/bin/bash +# +# unified script to access EESSI in different scenarios: read-only +# for just using EESSI, read & write for building software to be +# added to the software stack +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# script overview +# -. initial settings & exit codes +# 0. parse args +# 1. check if argument values are valid +# 2. set up local disk/tmp +# 3. set up common vars and directories +# 4. set up vars specific to a scenario +# 5. initialize local disk/tmp from previous run if provided +# 6. run container + +# -. initial settings & exit codes +base_dir=$(dirname $(realpath $0)) + +# functions +function echo_red() { + echo -e "\e[31m$1\e[0m" +} + +function fatal_error() { + echo_red "ERROR: ${1}" >&2 + exit ${2} +} + +# exit codes: bitwise shift codes to allow for combination of exit codes +ANY_ERROR_EXITCODE=1 +CMDLINE_ARG_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 1)) +ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2)) +CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3)) +LOCAL_DISK_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4)) +MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) +PREVIOUS_RUN_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) +REPOSITORY_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) +HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) +HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) +RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) + +# CernVM-FS settings +CVMFS_VAR_LIB="var-lib-cvmfs" +CVMFS_VAR_RUN="var-run-cvmfs" + + +# 0. parse args +# see example parsing of command line arguments at +# https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop +# https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash + +display_help() { + echo "usage: $0 [OPTIONS] [SCRIPT]" + echo " OPTIONS:" + echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -c | --container IMAGE - image file or URL defining the container to use" + echo " [default: docker://ghcr.io/eessi/build-node:debian10]" + echo " -d | --dry-run - run script except for executing the container," + echo " print information about setup [default: false]" + echo " -h | --help - display this usage information [default: false]" + echo " -i | --info - display configured repositories [default: false]" + echo " -l | --local-disk DIR - directory space on local machine (used for" + echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -m | --mode {shell,run} - shell (launch interactive shell)" + echo " run (run a script) [default: shell]" + echo " -p | --previous-run PRUN - init local disk with data from previous run" + echo " format is PATH[:TAR/ZIP] where PATH is pointing" + echo " to the previously used local disk, and TAR/ZIP" + echo " is used to initialize the local disk if PATH" + echo " doesn't exist currently; if PATH exists and" + echo " a TAR/ZIP is provided an error is reported" + echo " [default: not set]" + echo " -r | --repository CFG - configuration file or identifier defining the" + echo " repository to use [default: EESSI-pilot]" + echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" + echo " [default: not set]" + echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" + echo " [default: not set]" + echo + echo " If value for --mode is 'run', the SCRIPT provided is executed." +} + +# set defaults for command line arguments +ACCESS="ro" +CONTAINER="docker://ghcr.io/eessi/build-node:debian10" +DRY_RUN=0 +INFO=0 +LOCAL_DISK= +MODE="shell" +PREVIOUS_RUN= +REPOSITORY="EESSI-pilot" +HTTP_PROXY= +HTTPS_PROXY= +RUN_SCRIPT_AND_ARGS= + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -a|--access) + ACCESS="$2" + shift 2 + ;; + -c|--container) + CONTAINER="$2" + shift 2 + ;; + -d|--dry-run) + DRY_RUN=1 + ;; + -h|--help) + display_help + exit 0 + ;; + -i|--info) + INFO=1 + ;; + -l|--local-disk) + LOCAL_DISK="$2" + #EESSI_TMPDIR="$2" + shift 2 + ;; + -m|--mode) + MODE="$2" + shift 2 + ;; + -p|--previous-run) + PREVIOUS_RUN="$2" + shift 2 + ;; + -x|--http-proxy) + HTTP_PROXY="$2" + export http_proxy=${HTTP_PROXY} + shift 2 + ;; + -y|--https-proxy) + HTTPS_PROXY="$2" + export https_proxy=${HTTPS_PROXY} + shift 2 + ;; + -*|--*) + fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + + +# 1. check if argument values are valid +# (arg -a|--access) check if ACCESS is supported +if [[ ${ACCESS} != "ro" -a ${ACCESS} != "rw" ]]; then + fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" +fi + +# TODO (arg -c|--container) check container (is it a file or URL & access those) +# CONTAINER_ERROR_EXITCODE + +# TODO (arg -l|--local-disk) check if it exists, if user has write permission, +# if it contains no data, etc. +# LOCAL_DISK_ERROR_EXITCODE + +# (arg -m|--mode) check if MODE is known +if [[ ${MODE} != "shell" -a ${MODE} != "run" ]]; then + fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" +fi + +# TODO (arg -p|--previous-run) check if it exists, if user has read permission, +# if it contains data from a previous run +# PREVIOUS_RUN_ERROR_EXITCODE + +# TODO (arg -r|--repository) check if repository is known +# REPOSITORY_UNKNOWN_EXITCODE + +# TODO (arg -x|--http-proxy) check if http proxy is accessible +# HTTP_PROXY_ERROR_EXITCODE + +# TODO (arg -y|--https-proxy) check if https proxy is accessible +# HTTPS_PROXY_ERROR_EXITCODE + +# check if a script is provided if mode is 'run' +if [[ "${MODE}" == "run" ]]; then + if [[ $# -eq 0 ]]; then + fatal_error "no command specified to run?!" "${RUN_SCRIPT_MISSING_EXITCODE}" + else + RUN_SCRIPT_AND_ARGS=$@ + fi +fi + + +# 2. set up local disk/tmp +# as location for temporary data use in the following order +# a. command line argument -l|--local_disk +# b. env var TMPDIR +# c. /tmp +# note, we ensure that (a) takes precedence by setting TMPDIR to LOCAL_DISK +# if LOCAL_DISK is not empty +# note, (b) & (c) are automatically ensured by using mktemp -d to create +# a temporary directory +# note, if previous run is used the name of the temporary directory +# should be identical to previous run, ie, then we don't create a new +# temporary directory +if [[ ! -z ${LOCAL_DISK} ]]; then + TMPDIR=${LOCAL_DISK} +fi +if [[ ! -z ${TMPDIR} ]]; then + # TODO check if TMPDIR already exists + EESSI_LOCAL_DISK=${TMPDIR} +fi +if [[ -z ${TMPDIR} ]]; then + # mktemp falls back to using /tmp if TMPDIR is empty + # TODO check if /tmp is writable, large enough and usable (different + # features for ro-access and rw-access) + echo "skipping sanity checks for /tmp" +fi +EESSI_LOCAL_DISK=$(mktemp -d eessi.XXXXXXXXXX) +echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..." + + +# 3. set up common vars and directories +# directory structure should be: +# ${EESSI_LOCAL_DISK} +# |-singularity_cache +# |-${CVMFS_VAR_LIB} +# |-${CVMFS_VAR_RUN} + +source ${base_dir}/init/eessi_defaults + +# configure Singularity +export SINGULARITY_CACHEDIR=${EESSI_LOCAL_DISK}/singularity_cache +mkdir -p ${SINGULARITY_CACHEDIR} +[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" + +# set env vars and create directories for CernVM-FS +EESSI_CVMFS_VAR_LIB=${EESSI_LOCAL_DISK}/${CVMFS_VAR_LIB} +EESSI_CVMFS_VAR_RUN=${EESSI_LOCAL_DISK}/${CVMFS_VAR_RUN} +mkdir -p ${EESSI_CVMFS_VAR_LIB} +mkdir -p ${EESSI_CVMFS_VAR_RUN} +[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}" +[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}" + +# tmp dir for EESSI +EESSI_TMPDIR=${EESSI_LOCAL_DISK}/tmp +mkdir -p ${EESSI_TMPDIR} +[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" + +# allow that SINGULARITY_HOME is defined before script is run +if [[ -z ${SINGULARITY_HOME} ]]; then + export SINGULARITY_HOME="${EESSI_LOCAL_DISK}/home:/home/${USER}" + mkdir -p ${EESSI_LOCAL_DISK}/home + [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" +fi + +# define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) +BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvfms,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" +BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" + + +# 4. set up vars and dirs specific to a scenario + +# strip "/cvmfs/" from default setting +repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} + +if [[ "${ACCESS}" == "ro" ]]; then + export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} ${EESSI_CVMFS_REPO}" + export EESSI_FUSE_MOUNTS="--fusemount ${EESSI_PILOT_READONLY}" +fi + +if [[ "${ACCESS}" == "rw" ]]; then + EESSI_CVMFS_OVERLAY_UPPER=${EESSI_LOCAL_DISK}/overlay-upper + EESSI_CVMFS_OVERLAY_WORK=${EESSI_LOCAL_DISK}/overlay-work + mkdir -p ${EESSI_CMVFS_OVERLAY_UPPER} + mkdir -p ${EESSI_CMVFS_OVERLAY_WORK} + [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" + [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" + + # set environment variables for fuse mounts in Singularity container + export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${EESSI_CVMFS_OVERLAY_UPPER}" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${EESSI_CVMFS_OVERLAY_WORK}" + EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" + export EESSI_PILOT_WRITABLE_OVERLAY + + EESSI_FUSE_MOUNTS="--fusemount ${EESSI_PILOT_READONLY}" + EESSI_FUSE_MOUNTS+=" --fusemount ${EESSI_PILOT_WRITABLE_OVERLAY}" + export EESSI_FUSE_MOUNTS +fi + + +# 5. initialize local disk/tmp from previous run if provided + + +# 6. run container +echo "Launching container with command (next line):" +echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" +singularity ${MODE} ${EESSI_FUSE_MOUNTS} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} From 6f904cee6e6aa9d83e79e0bf7eabbc329885561b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 20:54:01 +0100 Subject: [PATCH 003/106] fix test syntax --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 9219581582..273d3bb297 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -162,7 +162,7 @@ set -- "${POSITIONAL_ARGS[@]}" # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported -if [[ ${ACCESS} != "ro" -a ${ACCESS} != "rw" ]]; then +if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi @@ -174,7 +174,7 @@ fi # LOCAL_DISK_ERROR_EXITCODE # (arg -m|--mode) check if MODE is known -if [[ ${MODE} != "shell" -a ${MODE} != "run" ]]; then +if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" fi From d6aae8d03247f2fe0e571989070ff94764dfdcd6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 21:44:53 +0100 Subject: [PATCH 004/106] use array for EESSI_FUSE_MOUNTS, fix var name typo --- eessi_container.sh | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 273d3bb297..23d6cd4bf5 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -274,31 +274,30 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" # strip "/cvmfs/" from default setting repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} +declare -a EESSI_FUSE_MOUNTS=() if [[ "${ACCESS}" == "ro" ]]; then export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} ${EESSI_CVMFS_REPO}" - export EESSI_FUSE_MOUNTS="--fusemount ${EESSI_PILOT_READONLY}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") fi if [[ "${ACCESS}" == "rw" ]]; then EESSI_CVMFS_OVERLAY_UPPER=${EESSI_LOCAL_DISK}/overlay-upper EESSI_CVMFS_OVERLAY_WORK=${EESSI_LOCAL_DISK}/overlay-work - mkdir -p ${EESSI_CMVFS_OVERLAY_UPPER} - mkdir -p ${EESSI_CMVFS_OVERLAY_WORK} + mkdir -p ${EESSI_CVMFS_OVERLAY_UPPER} + mkdir -p ${EESSI_CVMFS_OVERLAY_WORK} [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" # set environment variables for fuse mounts in Singularity container - export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") + EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${EESSI_CVMFS_OVERLAY_UPPER}" EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${EESSI_CVMFS_OVERLAY_WORK}" EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" - export EESSI_PILOT_WRITABLE_OVERLAY - - EESSI_FUSE_MOUNTS="--fusemount ${EESSI_PILOT_READONLY}" - EESSI_FUSE_MOUNTS+=" --fusemount ${EESSI_PILOT_WRITABLE_OVERLAY}" - export EESSI_FUSE_MOUNTS + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}") fi @@ -307,5 +306,5 @@ fi # 6. run container echo "Launching container with command (next line):" -echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" -singularity ${MODE} ${EESSI_FUSE_MOUNTS} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} +echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" +singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} From 4194421b5075bfc8714a5d6d0f3cd554d35992aa Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 21:46:01 +0100 Subject: [PATCH 005/106] add missing shift operations --- eessi_container.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index 23d6cd4bf5..fcc5cc1e31 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -116,6 +116,7 @@ while [[ $# -gt 0 ]]; do ;; -d|--dry-run) DRY_RUN=1 + shift 1 ;; -h|--help) display_help @@ -123,6 +124,7 @@ while [[ $# -gt 0 ]]; do ;; -i|--info) INFO=1 + shift 1 ;; -l|--local-disk) LOCAL_DISK="$2" From 8b2f7301fc9cf7e2b583397a4f603ab74f4efcc0 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 22:10:09 +0100 Subject: [PATCH 006/106] fixing creation of temporary directory --- eessi_container.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index fcc5cc1e31..3d64585d0d 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -210,17 +210,20 @@ fi # c. /tmp # note, we ensure that (a) takes precedence by setting TMPDIR to LOCAL_DISK # if LOCAL_DISK is not empty -# note, (b) & (c) are automatically ensured by using mktemp -d to create -# a temporary directory +# note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to +# create a temporary directory # note, if previous run is used the name of the temporary directory # should be identical to previous run, ie, then we don't create a new # temporary directory if [[ ! -z ${LOCAL_DISK} ]]; then TMPDIR=${LOCAL_DISK} + # mktemp fails if TMPDIR does not exist, so let's create it + mkdir -p ${TMPDIR} fi if [[ ! -z ${TMPDIR} ]]; then # TODO check if TMPDIR already exists - EESSI_LOCAL_DISK=${TMPDIR} + # mktemp fails if TMPDIR does not exist, so let's create it + mkdir -p ${TMPDIR} fi if [[ -z ${TMPDIR} ]]; then # mktemp falls back to using /tmp if TMPDIR is empty @@ -228,7 +231,7 @@ if [[ -z ${TMPDIR} ]]; then # features for ro-access and rw-access) echo "skipping sanity checks for /tmp" fi -EESSI_LOCAL_DISK=$(mktemp -d eessi.XXXXXXXXXX) +EESSI_LOCAL_DISK=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..." From 6aa5cac8d7335e47120c5504de2fb8e1aa86f95e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 22:32:10 +0100 Subject: [PATCH 007/106] fix BIND typo, define SINGULARITY_BIND --- eessi_container.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 3d64585d0d..0425d8759b 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -270,8 +270,9 @@ if [[ -z ${SINGULARITY_HOME} ]]; then fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) -BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvfms,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" +BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" +[[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" # 4. set up vars and dirs specific to a scenario @@ -310,6 +311,14 @@ fi # 6. run container +# final settings +if [[ -z ${SINGULARITY_BIND} ]]; then + export SINGULARITY_BIND="${BIND_PATHS}" +else + export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}" +fi +[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" + echo "Launching container with command (next line):" echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} From 3a2bf51687bb1c9b8e39518e2481939e84dfc751 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 1 Jan 2023 23:48:28 +0100 Subject: [PATCH 008/106] fix tmp dir for overlay fs --- eessi_container.sh | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 0425d8759b..1be3b566db 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -241,31 +241,34 @@ echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..." # |-singularity_cache # |-${CVMFS_VAR_LIB} # |-${CVMFS_VAR_RUN} +# |-overlay-upper +# |-overlay-work +# |-home source ${base_dir}/init/eessi_defaults +# tmp dir for EESSI +EESSI_TMPDIR=${EESSI_LOCAL_DISK} +mkdir -p ${EESSI_TMPDIR} +[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" + # configure Singularity -export SINGULARITY_CACHEDIR=${EESSI_LOCAL_DISK}/singularity_cache +export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache mkdir -p ${SINGULARITY_CACHEDIR} [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" # set env vars and create directories for CernVM-FS -EESSI_CVMFS_VAR_LIB=${EESSI_LOCAL_DISK}/${CVMFS_VAR_LIB} -EESSI_CVMFS_VAR_RUN=${EESSI_LOCAL_DISK}/${CVMFS_VAR_RUN} +EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} +EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} mkdir -p ${EESSI_CVMFS_VAR_LIB} mkdir -p ${EESSI_CVMFS_VAR_RUN} [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}" [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}" -# tmp dir for EESSI -EESSI_TMPDIR=${EESSI_LOCAL_DISK}/tmp -mkdir -p ${EESSI_TMPDIR} -[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" - # allow that SINGULARITY_HOME is defined before script is run if [[ -z ${SINGULARITY_HOME} ]]; then - export SINGULARITY_HOME="${EESSI_LOCAL_DISK}/home:/home/${USER}" - mkdir -p ${EESSI_LOCAL_DISK}/home + export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" + mkdir -p ${EESSI_TMPDIR}/home [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" fi @@ -287,10 +290,10 @@ if [[ "${ACCESS}" == "ro" ]]; then fi if [[ "${ACCESS}" == "rw" ]]; then - EESSI_CVMFS_OVERLAY_UPPER=${EESSI_LOCAL_DISK}/overlay-upper - EESSI_CVMFS_OVERLAY_WORK=${EESSI_LOCAL_DISK}/overlay-work - mkdir -p ${EESSI_CVMFS_OVERLAY_UPPER} - mkdir -p ${EESSI_CVMFS_OVERLAY_WORK} + EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper + EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work + mkdir -p ${EESSI_TMPDIR}/overlay-upper + mkdir -p ${EESSI_TMPDIR}/overlay-work [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" @@ -300,8 +303,8 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${EESSI_CVMFS_OVERLAY_UPPER}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${EESSI_CVMFS_OVERLAY_WORK}" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work" EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}") fi From 3fbea5e4faa3f4acbe31108fe8f08e2a5f088405 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 2 Jan 2023 18:53:53 +0100 Subject: [PATCH 009/106] fixing export bug, improving readability - added missing export for TMPDIR - moved functions to utils.sh - renamed local-disk/LOCAL_DISK to host-storage/HOST_STORAGE --- eessi_container.sh | 58 ++++++++++++++++++----------------------- utils.sh | 65 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 34 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 1be3b566db..b597b17647 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -16,37 +16,29 @@ # -. initial settings & exit codes # 0. parse args # 1. check if argument values are valid -# 2. set up local disk/tmp +# 2. set up host storage/tmp # 3. set up common vars and directories # 4. set up vars specific to a scenario -# 5. initialize local disk/tmp from previous run if provided +# 5. initialize host storage/tmp from previous run if provided # 6. run container # -. initial settings & exit codes base_dir=$(dirname $(realpath $0)) -# functions -function echo_red() { - echo -e "\e[31m$1\e[0m" -} - -function fatal_error() { - echo_red "ERROR: ${1}" >&2 - exit ${2} -} +source ${base_dir}/utils.sh # exit codes: bitwise shift codes to allow for combination of exit codes -ANY_ERROR_EXITCODE=1 +# ANY_ERROR_EXITCODE is sourced from ${base_dir}/utils.sh CMDLINE_ARG_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 1)) ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2)) CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3)) -LOCAL_DISK_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4)) +HOST_STORAGE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4)) MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) PREVIOUS_RUN_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) REPOSITORY_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) -HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) -HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) -RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) +HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) +HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) +RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) # CernVM-FS settings CVMFS_VAR_LIB="var-lib-cvmfs" @@ -68,7 +60,7 @@ display_help() { echo " print information about setup [default: false]" echo " -h | --help - display this usage information [default: false]" echo " -i | --info - display configured repositories [default: false]" - echo " -l | --local-disk DIR - directory space on local machine (used for" + echo " -l | --host-storage DIR - directory space on host machine (used for" echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" echo " -m | --mode {shell,run} - shell (launch interactive shell)" echo " run (run a script) [default: shell]" @@ -94,7 +86,7 @@ ACCESS="ro" CONTAINER="docker://ghcr.io/eessi/build-node:debian10" DRY_RUN=0 INFO=0 -LOCAL_DISK= +HOST_STORAGE= MODE="shell" PREVIOUS_RUN= REPOSITORY="EESSI-pilot" @@ -126,8 +118,8 @@ while [[ $# -gt 0 ]]; do INFO=1 shift 1 ;; - -l|--local-disk) - LOCAL_DISK="$2" + -l|--host-storage) + HOST_STORAGE="$2" #EESSI_TMPDIR="$2" shift 2 ;; @@ -171,9 +163,9 @@ fi # TODO (arg -c|--container) check container (is it a file or URL & access those) # CONTAINER_ERROR_EXITCODE -# TODO (arg -l|--local-disk) check if it exists, if user has write permission, +# TODO (arg -l|--host-storage) check if it exists, if user has write permission, # if it contains no data, etc. -# LOCAL_DISK_ERROR_EXITCODE +# HOST_STORAGE_ERROR_EXITCODE # (arg -m|--mode) check if MODE is known if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then @@ -203,20 +195,20 @@ if [[ "${MODE}" == "run" ]]; then fi -# 2. set up local disk/tmp +# 2. set up host storage/tmp # as location for temporary data use in the following order -# a. command line argument -l|--local_disk +# a. command line argument -l|--host-storage # b. env var TMPDIR # c. /tmp -# note, we ensure that (a) takes precedence by setting TMPDIR to LOCAL_DISK -# if LOCAL_DISK is not empty +# note, we ensure that (a) takes precedence by setting TMPDIR to HOST_STORAGE +# if HOST_STORAGE is not empty # note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to # create a temporary directory # note, if previous run is used the name of the temporary directory # should be identical to previous run, ie, then we don't create a new # temporary directory -if [[ ! -z ${LOCAL_DISK} ]]; then - TMPDIR=${LOCAL_DISK} +if [[ ! -z ${HOST_STORAGE} ]]; then + export TMPDIR=${HOST_STORAGE} # mktemp fails if TMPDIR does not exist, so let's create it mkdir -p ${TMPDIR} fi @@ -231,13 +223,13 @@ if [[ -z ${TMPDIR} ]]; then # features for ro-access and rw-access) echo "skipping sanity checks for /tmp" fi -EESSI_LOCAL_DISK=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) -echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..." +EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) +echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..." # 3. set up common vars and directories # directory structure should be: -# ${EESSI_LOCAL_DISK} +# ${EESSI_HOST_STORAGE} # |-singularity_cache # |-${CVMFS_VAR_LIB} # |-${CVMFS_VAR_RUN} @@ -248,7 +240,7 @@ echo "Using ${EESSI_LOCAL_DISK} as parent for temporary directories..." source ${base_dir}/init/eessi_defaults # tmp dir for EESSI -EESSI_TMPDIR=${EESSI_LOCAL_DISK} +EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} [[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" @@ -310,7 +302,7 @@ if [[ "${ACCESS}" == "rw" ]]; then fi -# 5. initialize local disk/tmp from previous run if provided +# 5. initialize host storage/tmp from previous run if provided # 6. run container diff --git a/utils.sh b/utils.sh index 0c98c86ec4..82dfd4a345 100644 --- a/utils.sh +++ b/utils.sh @@ -10,9 +10,14 @@ function echo_yellow() { echo -e "\e[33m$1\e[0m" } +ANY_ERROR_EXITCODE=1 function fatal_error() { echo_red "ERROR: $1" >&2 - exit 1 + if [[ $# -gt 1 ]]; then + exit $2 + else + exit "${ANY_ERROR_EXITCODE}" + fi } function check_exit_code { @@ -26,3 +31,61 @@ function check_exit_code { fatal_error "${fail_msg}" fi } + +declare -A repo_cfg + +function cfg_get_section { + if [[ "$1" =~ ^(\[)(.*)(\])$ ]]; then + echo ${BASH_REMATCH[2]} + else + echo "" + fi +} + +function cfg_get_key_value { + if [[ "$1" =~ ^([^=]+)=([^=]+)$ ]]; then + echo "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}" + else + echo "" + fi +} + +function cfg_load { + local cur_section="" + local cur_key="" + local cur_val="" + IFS= + while read -r line; do + new_section=$(cfg_get_section $line) + # got a new section + if [[ -n "$new_section" ]]; then + cur_section=$new_section + # not a section, try a key value + else + val=$(cfg_get_key_value $line) + # trim leading and trailing spaces as well + cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + if [[ -n "$cur_key" ]]; then + # section + key is the associative in bash array, the field separator is space + repo_cfg[${cur_section} ${cur_key}]=$cur_val + fi + fi + done <$1 +} + +function cfg_print { + for i in "${!repo_cfg[@]}" + do + # split the associative key in to section and key + echo -n "section : $(echo $i | cut -f1 -d ' ');" + echo -n "key : $(echo $i | cut -f2 -d ' ');" + echo "value: ${repo_cfg[$i]}" + done +} + +function cfg_get_value { + section=$1 + key=$2 + echo "${repo_cfg[$section $key]}" +} From 84daaa36c1e06baf3cd0de9f93ac9cf26c354cd3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 3 Jan 2023 14:43:15 +0100 Subject: [PATCH 010/106] moving cfg handling to separate file cfg_files.sh --- cfg_files.sh | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++ utils.sh | 58 ----------------------------------------- 2 files changed, 73 insertions(+), 58 deletions(-) create mode 100644 cfg_files.sh diff --git a/cfg_files.sh b/cfg_files.sh new file mode 100644 index 0000000000..c2c549d0bf --- /dev/null +++ b/cfg_files.sh @@ -0,0 +1,73 @@ +# functions for working with ini/cfg files +# +# This file is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + + +# global variables +# -a -> indexed array +# -A -> associative array +declare -A cfg_repos + + +# functions +function cfg_get_section { + if [[ "$1" =~ ^(\[)(.*)(\])$ ]]; then + echo ${BASH_REMATCH[2]} + else + echo "" + fi +} + +function cfg_get_key_value { + if [[ "$1" =~ ^([^=]+)=([^=]+)$ ]]; then + echo "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}" + else + echo "" + fi +} + +function cfg_load { + local cur_section="" + local cur_key="" + local cur_val="" + IFS= + while read -r line; do + new_section=$(cfg_get_section $line) + # got a new section + if [[ -n "$new_section" ]]; then + cur_section=$new_section + # not a section, try a key value + else + val=$(cfg_get_key_value $line) + # trim leading and trailing spaces as well + cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + if [[ -n "$cur_key" ]]; then + # section + key is the associative in bash array, the field separator is space + repo_cfg[${cur_section} ${cur_key}]=$cur_val + fi + fi + done <$1 +} + +function cfg_print { + for i in "${!repo_cfg[@]}" + do + # split the associative key in to section and key + echo -n "section : $(echo $i | cut -f1 -d ' ');" + echo -n "key : $(echo $i | cut -f2 -d ' ');" + echo "value: ${repo_cfg[$i]}" + done +} + +function cfg_get_value { + section=$1 + key=$2 + echo "${repo_cfg[$section $key]}" +} diff --git a/utils.sh b/utils.sh index 82dfd4a345..5d8455bb68 100644 --- a/utils.sh +++ b/utils.sh @@ -31,61 +31,3 @@ function check_exit_code { fatal_error "${fail_msg}" fi } - -declare -A repo_cfg - -function cfg_get_section { - if [[ "$1" =~ ^(\[)(.*)(\])$ ]]; then - echo ${BASH_REMATCH[2]} - else - echo "" - fi -} - -function cfg_get_key_value { - if [[ "$1" =~ ^([^=]+)=([^=]+)$ ]]; then - echo "${BASH_REMATCH[1]}=${BASH_REMATCH[2]}" - else - echo "" - fi -} - -function cfg_load { - local cur_section="" - local cur_key="" - local cur_val="" - IFS= - while read -r line; do - new_section=$(cfg_get_section $line) - # got a new section - if [[ -n "$new_section" ]]; then - cur_section=$new_section - # not a section, try a key value - else - val=$(cfg_get_key_value $line) - # trim leading and trailing spaces as well - cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') - cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') - if [[ -n "$cur_key" ]]; then - # section + key is the associative in bash array, the field separator is space - repo_cfg[${cur_section} ${cur_key}]=$cur_val - fi - fi - done <$1 -} - -function cfg_print { - for i in "${!repo_cfg[@]}" - do - # split the associative key in to section and key - echo -n "section : $(echo $i | cut -f1 -d ' ');" - echo -n "key : $(echo $i | cut -f2 -d ' ');" - echo "value: ${repo_cfg[$i]}" - done -} - -function cfg_get_value { - section=$1 - key=$2 - echo "${repo_cfg[$section $key]}" -} From 95c65049ac6148fb531841131e431a585c540a16 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 3 Jan 2023 20:31:27 +0100 Subject: [PATCH 011/106] functions to process repository config file and use data in eessi_container.sh --- cfg_files.sh | 87 ++++++++++++++++++++++++++++++++++++++++++---- eessi_container.sh | 64 ++++++++++++++++++++++++++++++++-- 2 files changed, 142 insertions(+), 9 deletions(-) diff --git a/cfg_files.sh b/cfg_files.sh index c2c549d0bf..f17b893280 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -13,6 +13,7 @@ # -a -> indexed array # -A -> associative array declare -A cfg_repos +declare -A cfg_file_map # functions @@ -46,28 +47,102 @@ function cfg_load { else val=$(cfg_get_key_value $line) # trim leading and trailing spaces as well - cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') - cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces) + #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') + cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces) if [[ -n "$cur_key" ]]; then # section + key is the associative in bash array, the field separator is space - repo_cfg[${cur_section} ${cur_key}]=$cur_val + cfg_repos[${cur_section} ${cur_key}]=$cur_val fi fi done <$1 } function cfg_print { - for i in "${!repo_cfg[@]}" + for i in "${!cfg_repos[@]}" do # split the associative key in to section and key echo -n "section : $(echo $i | cut -f1 -d ' ');" echo -n "key : $(echo $i | cut -f2 -d ' ');" - echo "value: ${repo_cfg[$i]}" + echo "value: ${cfg_repos[$i]}" done } function cfg_get_value { section=$1 key=$2 - echo "${repo_cfg[$section $key]}" + echo "${cfg_repos[$section $key]}" +} + +function cfg_trim_spaces { + # reads from argument $1 or stdin + if [[ $# -gt 0 ]]; then + sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<< ${1} + else + sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' < /dev/stdin + fi +} + +function cfg_trim_quotes { + # reads from argument $1 or stdin + if [[ $# -gt 0 ]]; then + sed -e 's/^"*//' -e 's/"*$//' <<< ${1} + else + sed -e 's/^"*//' -e 's/"*$//' < /dev/stdin + fi +} + +function cfg_trim_curly_brackets { + # reads from argument $1 or stdin + if [[ $# -gt 0 ]]; then + sed -e 's/^{*//' -e 's/}*$//' <<< ${1} + else + sed -e 's/^{*//' -e 's/}*$//' < /dev/stdin + fi +} + +function cfg_get_all_sections { + # first field in keys + # 1. get first field in all keys, 2. filter duplicates, 3. return them as string + declare -A all_sections + for key in "${!cfg_repos[@]}" + do + section=$(echo "$key" | cut -f1 -d' ') + all_sections[${section}]=1 + done + sections= + for sec_key in "${!all_sections[@]}" + do + sections="${sections} ${sec_key}" + done + echo "${sections}" +} + +function cfg_init_file_map { + # strip '{' and '}' from config_map + # split config_map at ',' + # for each item: split at ':' use first as key, second as value + + # reset global variable + cfg_file_map=() + + # expects a string containing the config_map from the cfg file + # trim leading and trailing curly brackets + cm_trimmed=$(cfg_trim_curly_brackets $1) + + # split into elements along ',' + declare -a cm_mappings +# IFS=',' read -r -a cm_mappings <<< "${cm_trimmed}" + + for index in "${!cm_mappings[@]}" + do + # split mapping into key and value + map_key=$(echo ${cm_mappings[index]} | cut -f1 -d':') + map_value=$(echo ${cm_mappings[index]} | cut -f2 -d':') + # trim spaces and double quotes at start and end + tr_key=$(cfg_trim_spaces "${map_key}" | cfg_trim_quotes) + tr_value=$(cfg_trim_spaces "${map_value}" | cfg_trim_quotes) + cfg_file_map[${tr_key}]=${tr_value} + done } diff --git a/eessi_container.sh b/eessi_container.sh index b597b17647..be0bcccab6 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -26,6 +26,7 @@ base_dir=$(dirname $(realpath $0)) source ${base_dir}/utils.sh +source ${base_dir}/cfg_files.sh # exit codes: bitwise shift codes to allow for combination of exit codes # ANY_ERROR_EXITCODE is sourced from ${base_dir}/utils.sh @@ -44,6 +45,9 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" +# repository cfg file +REPO_CFG_FILE=repos.cfg + # 0. parse args # see example parsing of command line arguments at @@ -131,6 +135,10 @@ while [[ $# -gt 0 ]]; do PREVIOUS_RUN="$2" shift 2 ;; + -r|--repository) + REPOSITORY="$2" + shift 2 + ;; -x|--http-proxy) HTTP_PROXY="$2" export http_proxy=${HTTP_PROXY} @@ -236,6 +244,7 @@ echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..." # |-overlay-upper # |-overlay-work # |-home +# |-cfg source ${base_dir}/init/eessi_defaults @@ -269,11 +278,60 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" +# set up repository config (always create cfg dir and populate it with info when +# arg -r|--repository is used) +mkdir -p ${EESSI_TMPDIR}/cfg +if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then + # strip "/cvmfs/" from default setting + repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} +else + # TODO implement more flexible specification of repo cfgs + # REPOSITORY => repo-id OR repo-cfg-file (with a single section) OR + # repo-cfg-file:repo-id (repo-id defined in repo-cfg-file) + # + # for now, assuming repo-id is defined in config file pointed to + # REPO_CFG_FILE, which is to be copied into the working directory + # (could also become part of the software layer to define multiple + # standard EESSI repositories) + cfg_load ${REPO_CFG_FILE} + + # cfg file should include: repo_name, repo_version, config_bundle, + # map { local_filepath -> container_filepath } + # + # repo_name_domain is the domain part of the repo_name, e.g., + # eessi-hpc.org for pilot.eessi-hpc.org + # + # where config bundle includes the files (-> target location in container) + # - default.local -> /etc/cvmfs/default.local + # contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ... + # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf + # contains CVMFS settings, e.g., CVMFS_SERVER_URL (Stratum 1s), + # CVMFS_KEYS_DIR, CVMFS_USE_GEOAPI, ... + # - ${repo_name_domain}/ -> /etc/cvmfs/keys/${repo_name_domain} + # a directory that contains the public key to access the repository, key + # itself then doesn't need to be BIND mounted + # - ${repo_name_domain}/${repo_name}.pub + # (-> /etc/cvmfs/keys/${repo_name_domain}/${repo_name}.pub + # the public key to access the repository, key itself is BIND mounted + # via directory ${repo_name_domain} + repo_name=$(cfg_get_value ${REPOSITORY} "repo_name") + # derive domain part from repo_name (everything after first '.') + repo_name_domain=${repo_name#*.} + repo_version=$(cfg_get_value ${REPOSITORY} "repo_version") + config_bundle=$(cfg_get_value ${REPOSITORY} "config_bundle") + config_map=$(cfg_get_value ${REPOSITORY} "config_map") + + # convert config_map into associative array cfg_file_map + cfg_init_file_map + declare -p cfg_file_map + + # TODO use information to set up dir ${EESSI_TMPDIR}/cfg and + # define BIND mounts + exit -1 +fi -# 4. set up vars and dirs specific to a scenario -# strip "/cvmfs/" from default setting -repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} +# 4. set up vars and dirs specific to a scenario declare -a EESSI_FUSE_MOUNTS=() if [[ "${ACCESS}" == "ro" ]]; then From f349b1ac841e82ea8c1b739ce5c883cc75c308d3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 3 Jan 2023 20:51:59 +0100 Subject: [PATCH 012/106] small fixes and code style improvements --- cfg_files.sh | 30 +++++++++++++++++++----------- eessi_container.sh | 4 ++-- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/cfg_files.sh b/cfg_files.sh index f17b893280..e1694bfc79 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -60,19 +60,19 @@ function cfg_load { } function cfg_print { - for i in "${!cfg_repos[@]}" - do + for index in "${!cfg_repos[@]}" + do # split the associative key in to section and key - echo -n "section : $(echo $i | cut -f1 -d ' ');" - echo -n "key : $(echo $i | cut -f2 -d ' ');" - echo "value: ${cfg_repos[$i]}" - done + echo -n "section : $(echo $index | cut -f1 -d ' ');" + echo -n "key : $(echo $index | cut -f2 -d ' ');" + echo "value: ${cfg_repos[$index]}" + done } function cfg_get_value { - section=$1 - key=$2 - echo "${cfg_repos[$section $key]}" + section=$1 + key=$2 + echo "${cfg_repos[$section $key]}" } function cfg_trim_spaces { @@ -116,7 +116,7 @@ function cfg_get_all_sections { do sections="${sections} ${sec_key}" done - echo "${sections}" + echo "${sections}" | cfg_trim_spaces } function cfg_init_file_map { @@ -129,7 +129,7 @@ function cfg_init_file_map { # expects a string containing the config_map from the cfg file # trim leading and trailing curly brackets - cm_trimmed=$(cfg_trim_curly_brackets $1) + cm_trimmed=$(cfg_trim_curly_brackets "$1") # split into elements along ',' declare -a cm_mappings @@ -146,3 +146,11 @@ function cfg_init_file_map { cfg_file_map[${tr_key}]=${tr_value} done } + +function cfg_print_map { + for index in "${!cfg_file_map[@]}" + do + echo "${index} --> ${cfg_file_map[${index}]}" + done +} + diff --git a/eessi_container.sh b/eessi_container.sh index be0bcccab6..85f1f55077 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -322,8 +322,8 @@ else config_map=$(cfg_get_value ${REPOSITORY} "config_map") # convert config_map into associative array cfg_file_map - cfg_init_file_map - declare -p cfg_file_map + cfg_init_file_map "${config_map}" + cfg_print_map # TODO use information to set up dir ${EESSI_TMPDIR}/cfg and # define BIND mounts From f1a583c22912c0b1a5bc9c218f3a75a6c5246322 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 3 Jan 2023 21:01:38 +0100 Subject: [PATCH 013/106] uncommented reading of config map --- cfg_files.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cfg_files.sh b/cfg_files.sh index e1694bfc79..885ebd0877 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -133,7 +133,7 @@ function cfg_init_file_map { # split into elements along ',' declare -a cm_mappings -# IFS=',' read -r -a cm_mappings <<< "${cm_trimmed}" + IFS=',' read -r -a cm_mappings <<< "${cm_trimmed}" for index in "${!cm_mappings[@]}" do From ff207f3ad866f3b7c815333f8c964a952f6511b2 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Jan 2023 10:33:55 +0100 Subject: [PATCH 014/106] preparing config information for custom repository --- eessi_container.sh | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 85f1f55077..1e1ed8e4ce 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -36,7 +36,7 @@ CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3)) HOST_STORAGE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4)) MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) PREVIOUS_RUN_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) -REPOSITORY_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) +REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) @@ -185,7 +185,7 @@ fi # PREVIOUS_RUN_ERROR_EXITCODE # TODO (arg -r|--repository) check if repository is known -# REPOSITORY_UNKNOWN_EXITCODE +# REPOSITORY_ERROR_EXITCODE # TODO (arg -x|--http-proxy) check if http proxy is accessible # HTTP_PROXY_ERROR_EXITCODE @@ -246,8 +246,6 @@ echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..." # |-home # |-cfg -source ${base_dir}/init/eessi_defaults - # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} @@ -282,6 +280,9 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/cfg if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then + # need to source defaults as late as possible (see other sourcing below) + source ${base_dir}/init/eessi_defaults + # strip "/cvmfs/" from default setting repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} else @@ -325,9 +326,22 @@ else cfg_init_file_map "${config_map}" cfg_print_map - # TODO use information to set up dir ${EESSI_TMPDIR}/cfg and - # define BIND mounts - exit -1 + # TODO use information to set up dir ${EESSI_TMPDIR}/cfg, + # define BIND mounts and override repo name and version + # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg + if [[ ! -r ${config_bundle} ]]; then + fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} + fi + tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg + for src in "${!cfg_file_map[@]}" + do + target=${cfg_file_map[${src}]} + BIND_PATHS="${BIND_PATHS},${src}:${target}" + done + export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} + export EESSI_CVMFS_REPO_OVERRIDE=${repo_name} + # need to source defaults as late as possible (after *_OVERRIDEs) + source ${base_dir}/init/eessi_defaults fi From b2543b9f70a27ad0be01159797d408362c718836 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Jan 2023 10:35:42 +0100 Subject: [PATCH 015/106] fix source location for config files --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 1e1ed8e4ce..cc8ab956da 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -336,7 +336,7 @@ else for src in "${!cfg_file_map[@]}" do target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${src}:${target}" + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}" done export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} export EESSI_CVMFS_REPO_OVERRIDE=${repo_name} From 3f161976d2ff1567e6a197bf91562f8d4aa122e6 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Jan 2023 14:37:32 +0100 Subject: [PATCH 016/106] using correct repository paths --- eessi_container.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index cc8ab956da..1df3574a6a 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -339,7 +339,7 @@ else BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}" done export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} - export EESSI_CVMFS_REPO_OVERRIDE=${repo_name} + export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" # need to source defaults as late as possible (after *_OVERRIDEs) source ${base_dir}/init/eessi_defaults fi @@ -349,8 +349,10 @@ fi declare -a EESSI_FUSE_MOUNTS=() if [[ "${ACCESS}" == "ro" ]]; then - export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} ${EESSI_CVMFS_REPO}" + export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") + export EESSI_FUSE_MOUNTS fi if [[ "${ACCESS}" == "rw" ]]; then @@ -362,7 +364,8 @@ if [[ "${ACCESS}" == "rw" ]]; then [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" # set environment variables for fuse mounts in Singularity container - EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_READONLY}") EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" @@ -370,7 +373,10 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper" EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work" EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" + export EESSI_PILOT_WRITABLE_OVERLAY + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_PILOT_WRITABLE_OVERLAY}") + export EESSI_FUSE_MOUNTS fi From 935517be94c0ea8b4ca5107aa7251da34a5473f4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Jan 2023 19:29:51 +0100 Subject: [PATCH 017/106] make repository cfg file path configurable --- eessi_container.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 1df3574a6a..b8867d2d78 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -45,8 +45,9 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" -# repository cfg file -REPO_CFG_FILE=repos.cfg +# repository cfg file, default name (default location: $PWD) +# can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" # 0. parse args @@ -291,10 +292,10 @@ else # repo-cfg-file:repo-id (repo-id defined in repo-cfg-file) # # for now, assuming repo-id is defined in config file pointed to - # REPO_CFG_FILE, which is to be copied into the working directory + # EESSI_REPOS_CFG_FILE, which is to be copied into the working directory # (could also become part of the software layer to define multiple # standard EESSI repositories) - cfg_load ${REPO_CFG_FILE} + cfg_load ${EESSI_REPOS_CFG_FILE} # cfg file should include: repo_name, repo_version, config_bundle, # map { local_filepath -> container_filepath } From d28cc696374cb0523a8583a4dcce58899628778b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Jan 2023 20:51:54 +0100 Subject: [PATCH 018/106] short section about eessi_container.sh --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index 4d463b1c5b..daf02eebc2 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,22 @@ Environment set up to use EESSI pilot software stack, have fun! [EESSI pilot 2021.12] $ ``` +### Accessing EESSI via a container + +You need Singularity version 3.7 or newer. Then, simply run + +``` +$ ./eessi_container.sh +``` +Once you get presented the prompt `Singularity>` run the above `source` command. + +If you want to build a package for the software repository, simply add the arguments `--access rw`, e.g., full command would be + +``` +$ ./eessi_container.sh --access rw +``` +Note, not all features/arguments listed via `./eessi_container.sh --help` are implemented. + # License The software in this repository is distributed under the terms of the From eca4f9798213ddc275b7496c1923fd8915af906b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 4 Jan 2023 20:54:31 +0100 Subject: [PATCH 019/106] show which features are not yet implemented --- eessi_container.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index b8867d2d78..fafa54193b 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -61,14 +61,20 @@ display_help() { echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" echo " -c | --container IMAGE - image file or URL defining the container to use" echo " [default: docker://ghcr.io/eessi/build-node:debian10]" - echo " -d | --dry-run - run script except for executing the container," - echo " print information about setup [default: false]" echo " -h | --help - display this usage information [default: false]" - echo " -i | --info - display configured repositories [default: false]" echo " -l | --host-storage DIR - directory space on host machine (used for" echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" echo " -m | --mode {shell,run} - shell (launch interactive shell)" echo " run (run a script) [default: shell]" + echo " -r | --repository CFG - configuration file or identifier defining the" + echo " repository to use [default: EESSI-pilot]" + echo + echo " If value for --mode is 'run', the SCRIPT provided is executed." + echo + echo " FEATURES/OPTIONS to be implemented:" + echo " -d | --dry-run - run script except for executing the container," + echo " print information about setup [default: false]" + echo " -i | --info - display configured repositories [default: false]" echo " -p | --previous-run PRUN - init local disk with data from previous run" echo " format is PATH[:TAR/ZIP] where PATH is pointing" echo " to the previously used local disk, and TAR/ZIP" @@ -76,14 +82,10 @@ display_help() { echo " doesn't exist currently; if PATH exists and" echo " a TAR/ZIP is provided an error is reported" echo " [default: not set]" - echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot]" echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" echo " [default: not set]" echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" echo " [default: not set]" - echo - echo " If value for --mode is 'run', the SCRIPT provided is executed." } # set defaults for command line arguments From 576897821c2eca3c21f7f168804315dec186466e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 5 Jan 2023 19:53:55 +0100 Subject: [PATCH 020/106] adding support for resuming a previous session --- eessi_container.sh | 146 +++++++++++++++++++++++++-------------------- 1 file changed, 82 insertions(+), 64 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index fafa54193b..9048b1ae7d 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -35,7 +35,7 @@ ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2)) CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3)) HOST_STORAGE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4)) MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) -PREVIOUS_RUN_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) +RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) @@ -58,16 +58,22 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" display_help() { echo "usage: $0 [OPTIONS] [SCRIPT]" echo " OPTIONS:" - echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" - echo " -c | --container IMAGE - image file or URL defining the container to use" - echo " [default: docker://ghcr.io/eessi/build-node:debian10]" - echo " -h | --help - display this usage information [default: false]" - echo " -l | --host-storage DIR - directory space on host machine (used for" - echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" - echo " -m | --mode {shell,run} - shell (launch interactive shell)" - echo " run (run a script) [default: shell]" - echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot]" + echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -c | --container IMG - image file or URL defining the container to use" + echo " [default: docker://ghcr.io/eessi/build-node:debian10]" + echo " -h | --help - display this usage information [default: false]" + echo " -g | --storage DIR - directory space on host machine (used for" + echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" + echo " MODE==run (run a script) [default: shell]" + echo " -r | --repository CFG - configuration file or identifier defining the" + echo " repository to use [default: EESSI-pilot]" + echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," + echo " where DIR points to a previously used tmp directory" + echo " (check for output 'Using DIR as tmp' of previous run)" + echo " and TGZ is the path to a tarball which is unpacked" + echo " the tmp dir stored on the local storage space (see" + echo " option --storage above) [default: not set]" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." echo @@ -75,13 +81,6 @@ display_help() { echo " -d | --dry-run - run script except for executing the container," echo " print information about setup [default: false]" echo " -i | --info - display configured repositories [default: false]" - echo " -p | --previous-run PRUN - init local disk with data from previous run" - echo " format is PATH[:TAR/ZIP] where PATH is pointing" - echo " to the previously used local disk, and TAR/ZIP" - echo " is used to initialize the local disk if PATH" - echo " doesn't exist currently; if PATH exists and" - echo " a TAR/ZIP is provided an error is reported" - echo " [default: not set]" echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" echo " [default: not set]" echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" @@ -93,10 +92,10 @@ ACCESS="ro" CONTAINER="docker://ghcr.io/eessi/build-node:debian10" DRY_RUN=0 INFO=0 -HOST_STORAGE= +STORAGE= MODE="shell" -PREVIOUS_RUN= REPOSITORY="EESSI-pilot" +RESUME= HTTP_PROXY= HTTPS_PROXY= RUN_SCRIPT_AND_ARGS= @@ -117,6 +116,10 @@ while [[ $# -gt 0 ]]; do DRY_RUN=1 shift 1 ;; + -g|--storage) + STORAGE="$2" + shift 2 + ;; -h|--help) display_help exit 0 @@ -125,23 +128,18 @@ while [[ $# -gt 0 ]]; do INFO=1 shift 1 ;; - -l|--host-storage) - HOST_STORAGE="$2" - #EESSI_TMPDIR="$2" - shift 2 - ;; -m|--mode) MODE="$2" shift 2 ;; - -p|--previous-run) - PREVIOUS_RUN="$2" - shift 2 - ;; -r|--repository) REPOSITORY="$2" shift 2 ;; + -u|--resume) + RESUME="$2" + shift 2 + ;; -x|--http-proxy) HTTP_PROXY="$2" export http_proxy=${HTTP_PROXY} @@ -174,7 +172,7 @@ fi # TODO (arg -c|--container) check container (is it a file or URL & access those) # CONTAINER_ERROR_EXITCODE -# TODO (arg -l|--host-storage) check if it exists, if user has write permission, +# TODO (arg -g|--storage) check if it exists, if user has write permission, # if it contains no data, etc. # HOST_STORAGE_ERROR_EXITCODE @@ -183,13 +181,13 @@ if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" fi -# TODO (arg -p|--previous-run) check if it exists, if user has read permission, -# if it contains data from a previous run -# PREVIOUS_RUN_ERROR_EXITCODE - # TODO (arg -r|--repository) check if repository is known # REPOSITORY_ERROR_EXITCODE +# TODO (arg -u|--resume) check if it exists, if user has read permission, +# if it contains data from a previous run +# RESUME_ERROR_EXITCODE + # TODO (arg -x|--http-proxy) check if http proxy is accessible # HTTP_PROXY_ERROR_EXITCODE @@ -206,37 +204,52 @@ if [[ "${MODE}" == "run" ]]; then fi -# 2. set up host storage/tmp -# as location for temporary data use in the following order -# a. command line argument -l|--host-storage -# b. env var TMPDIR -# c. /tmp -# note, we ensure that (a) takes precedence by setting TMPDIR to HOST_STORAGE -# if HOST_STORAGE is not empty -# note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to -# create a temporary directory -# note, if previous run is used the name of the temporary directory -# should be identical to previous run, ie, then we don't create a new -# temporary directory -if [[ ! -z ${HOST_STORAGE} ]]; then - export TMPDIR=${HOST_STORAGE} - # mktemp fails if TMPDIR does not exist, so let's create it - mkdir -p ${TMPDIR} -fi -if [[ ! -z ${TMPDIR} ]]; then - # TODO check if TMPDIR already exists - # mktemp fails if TMPDIR does not exist, so let's create it - mkdir -p ${TMPDIR} -fi -if [[ -z ${TMPDIR} ]]; then - # mktemp falls back to using /tmp if TMPDIR is empty - # TODO check if /tmp is writable, large enough and usable (different - # features for ro-access and rw-access) - echo "skipping sanity checks for /tmp" +# 2. set up host storage/tmp if necessary +# if session to be resumed from a previous one (--resume ARG) and ARG is a directory +# just reuse ARG, define environment variables accordingly and skip creating a new +# tmp storage +if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then + # resume from directory ${RESUME} + # skip creating a new tmp directory, just set environment variables + echo "Resuming from previous run using temporary storage at ${RESUME}" + EESSI_HOST_STORAGE=${RESUME} +else + # we need a tmp location (and possibly init it with ${RESUME} if it was not + # a directory + + # as location for temporary data use in the following order + # a. command line argument -l|--host-storage + # b. env var TMPDIR + # c. /tmp + # note, we ensure that (a) takes precedence by setting TMPDIR to STORAGE + # if STORAGE is not empty + # note, (b) & (c) are automatically ensured by using 'mktemp -d --tmpdir' to + # create a temporary directory + if [[ ! -z ${STORAGE} ]]; then + export TMPDIR=${STORAGE} + # mktemp fails if TMPDIR does not exist, so let's create it + mkdir -p ${TMPDIR} + fi + if [[ ! -z ${TMPDIR} ]]; then + # TODO check if TMPDIR already exists + # mktemp fails if TMPDIR does not exist, so let's create it + mkdir -p ${TMPDIR} + fi + if [[ -z ${TMPDIR} ]]; then + # mktemp falls back to using /tmp if TMPDIR is empty + # TODO check if /tmp is writable, large enough and usable (different + # features for ro-access and rw-access) + echo "skipping sanity checks for /tmp" + fi + EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) + echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..." fi -EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) -echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..." +# if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} +if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then + tar xf ${RESUME} -C ${EESSI_HOST_STORAGE} + echo "Resuming from previous run using temporary storage ${RESUME} unpacked into ${EESSI_HOST_STORAGE}" +fi # 3. set up common vars and directories # directory structure should be: @@ -335,7 +348,12 @@ else if [[ ! -r ${config_bundle} ]]; then fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} fi - tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg + + # only unpack config_bundle if we're not resuming from a previous run + if [[ -z ${RESUME} ]]; then + tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg + fi + for src in "${!cfg_file_map[@]}" do target=${cfg_file_map[${src}]} From bf90a48478f810efbf12e227041069145b5423b9 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 5 Jan 2023 20:06:23 +0100 Subject: [PATCH 021/106] improving message for tmp and resume option --- eessi_container.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 9048b1ae7d..71efdd0c01 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -70,10 +70,10 @@ display_help() { echo " repository to use [default: EESSI-pilot]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," echo " where DIR points to a previously used tmp directory" - echo " (check for output 'Using DIR as tmp' of previous run)" - echo " and TGZ is the path to a tarball which is unpacked" - echo " the tmp dir stored on the local storage space (see" - echo " option --storage above) [default: not set]" + echo " (check for output 'Using DIR as tmp ...' of a previous" + echo " run) and TGZ is the path to a tarball which is" + echo " unpacked the tmp dir stored on the local storage space" + echo " (see option --storage above) [default: not set]" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." echo @@ -242,7 +242,7 @@ else echo "skipping sanity checks for /tmp" fi EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) - echo "Using ${EESSI_HOST_STORAGE} as parent for temporary directories..." + echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." fi # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} From 6808ec587246a1d92f6fd06cf5e705218d57e001 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 6 Jan 2023 11:24:16 +0100 Subject: [PATCH 022/106] first attempt at implementing --save option --- eessi_container.sh | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 71efdd0c01..e19e44ca91 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -35,8 +35,9 @@ ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2)) CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3)) HOST_STORAGE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 4)) MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) -RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) -REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) +REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) +RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) +SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) @@ -74,6 +75,11 @@ display_help() { echo " run) and TGZ is the path to a tarball which is" echo " unpacked the tmp dir stored on the local storage space" echo " (see option --storage above) [default: not set]" + echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" + echo " directory DIR or provided with the fixed full path TGZ" + echo " when a directory is provided, the format of the" + echo " tarball's name will be" + echo " {REPO_ID}-{OS}-{ARCH}-{TIMESTAMP}.tgz [default: not set]" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." echo @@ -96,6 +102,7 @@ STORAGE= MODE="shell" REPOSITORY="EESSI-pilot" RESUME= +SAVE= HTTP_PROXY= HTTPS_PROXY= RUN_SCRIPT_AND_ARGS= @@ -136,6 +143,10 @@ while [[ $# -gt 0 ]]; do REPOSITORY="$2" shift 2 ;; + -s|--save) + SAVE="$2" + shift 2 + ;; -u|--resume) RESUME="$2" shift 2 @@ -188,6 +199,11 @@ fi # if it contains data from a previous run # RESUME_ERROR_EXITCODE +# TODO (arg -s|--save) check if DIR exists, if user has write permission, +# if TGZ already exists, if user has write permission to directory to which +# TGZ should be written +# SAVE_ERROR_EXITCODE + # TODO (arg -x|--http-proxy) check if http proxy is accessible # HTTP_PROXY_ERROR_EXITCODE @@ -416,3 +432,26 @@ fi echo "Launching container with command (next line):" echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} + +# 7. save tmp if requested (arg -s|--save) +if [[ ! -z ${SAVE} ]]; then + if [[ -d ${SAVE} ]]; then + # TODO to be implemented; best way a bit unclear + # assume SAVE is name of a directory to which tarball shall be written to + # name format: {REPO_ID}-{OS}-{ARCH}-{TIMESTAMP}.tgz + # use container to determine OS and ARCH (same approach as in eessi-bot-build.slurm) + #RUN_CMD="python3 /mnt/os_arch_detect/eessi_software_subdir.py ${GENERIC_OPT}" + # only need READONLY part for CVMFS + #singularity run -B ... ${CONTAINER} ${RUN_CMD} 2>&1) + os=foo + arch=bar + ts=$(date +%s) + TGZ=${REPOSITORY}-${os}-${arch}-${ts}.tgz + else + # assume SAVE is the full path to a tarball's name + tar cf ${SAVE} -C ${EESSI_TMPDIR} . + echo "Save contents of '${EESSI_TMPDIR}' to '${SAVE}' (to resume, add '--resume ${SAVE}')" + fi +fi + +# TODO clean up tmp by default? only retain if another option provided (--retain-tmp) From efe93c72c6e782bc08535cb9045ee1474c3e98e2 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 6 Jan 2023 11:57:05 +0100 Subject: [PATCH 023/106] added support for providing just a directory to deposit a tarball of tmp --- eessi_container.sh | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index e19e44ca91..01638ebf03 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -78,8 +78,8 @@ display_help() { echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" echo " directory DIR or provided with the fixed full path TGZ" echo " when a directory is provided, the format of the" - echo " tarball's name will be" - echo " {REPO_ID}-{OS}-{ARCH}-{TIMESTAMP}.tgz [default: not set]" + echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" + echo " [default: not set]" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." echo @@ -435,23 +435,21 @@ singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS # 7. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then + # Note, for now we don't try to be smart and record in any way the OS and + # ARCH which might have been used internally, eg, when software packages + # were built ... we rather keep the script here "stupid" and leave the handling + # of these aspects to where the script is used if [[ -d ${SAVE} ]]; then - # TODO to be implemented; best way a bit unclear # assume SAVE is name of a directory to which tarball shall be written to - # name format: {REPO_ID}-{OS}-{ARCH}-{TIMESTAMP}.tgz - # use container to determine OS and ARCH (same approach as in eessi-bot-build.slurm) - #RUN_CMD="python3 /mnt/os_arch_detect/eessi_software_subdir.py ${GENERIC_OPT}" - # only need READONLY part for CVMFS - #singularity run -B ... ${CONTAINER} ${RUN_CMD} 2>&1) - os=foo - arch=bar + # name format: {REPO_ID}-{TIMESTAMP}.tgz ts=$(date +%s) - TGZ=${REPOSITORY}-${os}-${arch}-${ts}.tgz + TGZ=${SAVE}/${REPOSITORY}-${ts}.tgz else # assume SAVE is the full path to a tarball's name - tar cf ${SAVE} -C ${EESSI_TMPDIR} . - echo "Save contents of '${EESSI_TMPDIR}' to '${SAVE}' (to resume, add '--resume ${SAVE}')" + TGZ=${SAVE} fi + tar cf ${TGZ} -C ${EESSI_TMPDIR} . + echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) From f896c51b73272b7b9f3dab6a12b6a700292e0a12 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 6 Jan 2023 12:15:07 +0100 Subject: [PATCH 024/106] cleaning up code, making script more silent by default --- eessi_container.sh | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 01638ebf03..57b003368b 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -38,8 +38,8 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) -HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) -HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) +#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) +#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) # CernVM-FS settings @@ -87,10 +87,10 @@ display_help() { echo " -d | --dry-run - run script except for executing the container," echo " print information about setup [default: false]" echo " -i | --info - display configured repositories [default: false]" - echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" - echo " [default: not set]" - echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" - echo " [default: not set]" + #echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" + #echo " [default: not set]" + #echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" + #echo " [default: not set]" } # set defaults for command line arguments @@ -103,8 +103,8 @@ MODE="shell" REPOSITORY="EESSI-pilot" RESUME= SAVE= -HTTP_PROXY= -HTTPS_PROXY= +#HTTP_PROXY= +#HTTPS_PROXY= RUN_SCRIPT_AND_ARGS= POSITIONAL_ARGS=() @@ -151,16 +151,16 @@ while [[ $# -gt 0 ]]; do RESUME="$2" shift 2 ;; - -x|--http-proxy) - HTTP_PROXY="$2" - export http_proxy=${HTTP_PROXY} - shift 2 - ;; - -y|--https-proxy) - HTTPS_PROXY="$2" - export https_proxy=${HTTPS_PROXY} - shift 2 - ;; +# -x|--http-proxy) +# HTTP_PROXY="$2" +# export http_proxy=${HTTP_PROXY} +# shift 2 +# ;; +# -y|--https-proxy) +# HTTPS_PROXY="$2" +# export https_proxy=${HTTPS_PROXY} +# shift 2 +# ;; -*|--*) fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" ;; @@ -255,7 +255,7 @@ else # mktemp falls back to using /tmp if TMPDIR is empty # TODO check if /tmp is writable, large enough and usable (different # features for ro-access and rw-access) - echo "skipping sanity checks for /tmp" + [[ ${INFO} -eq 1 ]] && echo "skipping sanity checks for /tmp" fi EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." @@ -356,7 +356,7 @@ else # convert config_map into associative array cfg_file_map cfg_init_file_map "${config_map}" - cfg_print_map + [[ ${INFO} -eq 1 ]] && cfg_print_map # TODO use information to set up dir ${EESSI_TMPDIR}/cfg, # define BIND mounts and override repo name and version @@ -431,7 +431,9 @@ fi echo "Launching container with command (next line):" echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" -singularity ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} +# TODO for now we run singularity with '-q' (quiet), later adjust this to the log level +# provided to the script +singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} # 7. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then From 2e66057724c2a3b51b92e10c11edd910691d8075 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 15 Jan 2023 21:01:17 +0100 Subject: [PATCH 025/106] Fix hash for checkout action in README CI --- .github/workflows/tests_readme.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests_readme.yml b/.github/workflows/tests_readme.yml index 29dedcc3ff..5c6d0318d4 100644 --- a/.github/workflows/tests_readme.yml +++ b/.github/workflows/tests_readme.yml @@ -18,14 +18,15 @@ jobs: build: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - name: Check out software-layer repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults - run: | - source init/eessi_defaults - grep "${EESSI_PILOT_VERSION}" README.md + - name: verify if README.md is consistent with EESSI_PILOT_VERSION from init/eessi_defaults + run: | + source init/eessi_defaults + grep "${EESSI_PILOT_VERSION}" README.md - - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults - run: | - source init/eessi_defaults - grep "${EESSI_CVMFS_REPO}" README.md + - name: verify if README.md is consistent with EESSI_CVMFS_REPO from init/eessi_defaults + run: | + source init/eessi_defaults + grep "${EESSI_CVMFS_REPO}" README.md From 447bb2d0da7f6b41a149344eb055ed16ca188663 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 24 Jan 2023 21:59:41 +0100 Subject: [PATCH 026/106] added a function to obtain full path to a tool --- utils.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/utils.sh b/utils.sh index 5d8455bb68..cd7f758a0d 100644 --- a/utils.sh +++ b/utils.sh @@ -31,3 +31,35 @@ function check_exit_code { fatal_error "${fail_msg}" fi } + +function get_path_for_tool { + tool_name=$1 + tool_envvar_name=$2 + + which_out=$(which ${tool_name} 2>&1) + exit_code=$? + if [[ ${exit_code} -eq 0 ]]; then + echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2 + echo "${which_out}" + return 0 + fi + if [[ -z "${tool_envvar_name}" ]]; then + msg="no env var holding the full path to tool '${tool_name}' provided" + echo "${msg}" >&2 + return 1 + else + tool_envvar_value=${!tool_envvar_name} + if [[ -x "${tool_envvar_value}" ]]; then + msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}" + echo "${msg}" >&2 + echo "${tool_envvar_value}" + return 0 + else + msg="ERROR: tool '${tool_name}' not in PATH\n" + msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH" + echo "${msg}" >&2 + echo "" + return 2 + fi + fi +} From e2c24cf757b912aef167364aa34d0abf723b7b03 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 24 Jan 2023 22:01:13 +0100 Subject: [PATCH 027/106] partial bot/build.sh for eessi_container.sh --- bot/build.sh | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100755 bot/build.sh diff --git a/bot/build.sh b/bot/build.sh new file mode 100755 index 0000000000..90753db8a8 --- /dev/null +++ b/bot/build.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# +# script to build the EESSI software layer. Intended use is that it is called +# by a (batch) job running on a compute node. +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# ASSUMPTIONs: +# - working directory has been prepared by the bot with a checkout of a +# pull request (OR by some other means) +# - the working directory contains a directory 'cfg' where the main config +# file 'job.cfg' has been deposited +# - the directory may contain any additional files references in job.cfg +# - the tool 'yq' for working with json files is available via the PATH or +# the environment variable BOT_YQ (see https://github.com/mikefarah/yq) + +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" + +# source utils.sh +source utils.sh + +# check setup / define key variables +# get path for 'yq' (if not found, an empty string is returned) +YQ=$(get_path_for_tool "yq" "BOT_YQ") +exit_code=$? +if [[ ${exit_code} -ne 0 ]]; then + fatal_error "could not find path to 'yq'; exiting" +else + echo_green "found yq (${YQ})" +fi + +# check if './cfg/job.cfg' exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +fi +echo "obtaining configuration settings from '${JOB_CFG_FILE}'" + +LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' < ${JOB_CFG_FILE}) +echo "LOCAL_TMP='${LOCAL_TMP}'" +echo -n "setting \$storage by replacing any var in '${LOCAL_TMP}' -> " +# replace any env variable in ${LOCAL_TMP} with its +# current value (e.g., a value that is local to the job) +storage=$(envsubst <<< ${LOCAL_TMP}) +echo "'${storage}'" + + +# singularity/apptainer settings: load_modules, HOME, TMPDIR, BIND +LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' < ${JOB_CFG_FILE}) +echo "LOAD_MODULES='${LOAD_MODULES}'" + +export SINGULARITY_HOME="$(pwd):/eessi_bot_job" +export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" +mkdir -p ${SINGULARITY_TMPDIR} + +if [[ ${storage} != /tmp* ]] ; +then + export SINGULARITY_BIND="${storage}:/tmp" +fi +echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" + +# load modules LOAD_MODULES is not empty +if [[ ! -z ${LOAD_MODULES} ]]; then + for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') + do + echo "bot/build.sh: loading module '${mod}'" + module load ${mod} + done +else + echo "bot/build.sh: no modules to be loaded" +fi + From 8cac3dfeb43a59f841f33baa62330bb38fe29f01 Mon Sep 17 00:00:00 2001 From: Michael Huebner Date: Wed, 25 Jan 2023 14:37:33 +0100 Subject: [PATCH 028/106] Update of where utils.sh is located when sourcing --- check_missing_installations.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 30f9cc6ff7..e4c75aebd8 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -16,7 +16,7 @@ fi LOCAL_TMPDIR=$(mktemp -d) -source $TOPDIR/utils.sh +source $TOPDIR/scripts/utils.sh source $TOPDIR/configure_easybuild From 3390a2d56ab5c28200ff487e124c74e87945679c Mon Sep 17 00:00:00 2001 From: Michael Huebner Date: Wed, 25 Jan 2023 14:40:06 +0100 Subject: [PATCH 029/106] Add create_directory_tarballs.sh to github CI --- .github/workflows/tests_scripts.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index dbf8933555..53f4bd13be 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -8,6 +8,7 @@ on: - run_in_compat_layer_env.sh - scripts/utils.sh - update_lmod_cache.sh + - create_directory_tarballs.sh pull_request: branches: @@ -18,6 +19,7 @@ on: - run_in_compat_layer_env.sh - scripts/utils.sh - update_lmod_cache.sh + - create_directory_tarballs.sh permissions: contents: read # to fetch code (actions/checkout) jobs: From a71f8e4f79d53f5be848d5f7064ce3569c4e0477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 25 Jan 2023 19:37:55 +0100 Subject: [PATCH 030/106] added function to determine IPv4 address --- utils.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/utils.sh b/utils.sh index cd7f758a0d..db533adc12 100644 --- a/utils.sh +++ b/utils.sh @@ -63,3 +63,12 @@ function get_path_for_tool { fi fi } + +function get_ipv4_address { + hname=$1 + hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) + # TODO try other methods if the one above does not work --> tool that verifies + # what method can be used? + echo "${hipv4}" + return 0 +} From b5bf008e4136b7877e06ca0602b6fd6aa49128c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 25 Jan 2023 19:39:53 +0100 Subject: [PATCH 031/106] various updates for bot/build.sh and eessi_container.sh --- bot/build.sh | 69 ++++++++++++++++++++++---- eessi_container.sh | 117 ++++++++++++++++++++++++--------------------- 2 files changed, 121 insertions(+), 65 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 90753db8a8..e7cc7983bb 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -42,30 +42,37 @@ if [[ ! -r "${JOB_CFG_FILE}" ]]; then fi echo "obtaining configuration settings from '${JOB_CFG_FILE}'" -LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' < ${JOB_CFG_FILE}) +# if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy +HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE}) +echo "HTTP_PROXY='${HTTP_PROXY}'" + +LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) echo "LOCAL_TMP='${LOCAL_TMP}'" -echo -n "setting \$storage by replacing any var in '${LOCAL_TMP}' -> " +# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided + +echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " # replace any env variable in ${LOCAL_TMP} with its # current value (e.g., a value that is local to the job) -storage=$(envsubst <<< ${LOCAL_TMP}) -echo "'${storage}'" - +STORAGE=$(envsubst <<< ${LOCAL_TMP}) +echo "'${STORAGE}'" -# singularity/apptainer settings: load_modules, HOME, TMPDIR, BIND -LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' < ${JOB_CFG_FILE}) +# obtain list of modules to be loaded +LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE}) echo "LOAD_MODULES='${LOAD_MODULES}'" +# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND +CONTAINER=$(${YQ} '.site_config.container // ""' ${JOB_CFG_FILE}) export SINGULARITY_HOME="$(pwd):/eessi_bot_job" export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} -if [[ ${storage} != /tmp* ]] ; +if [[ ${STORAGE} != /tmp* ]] ; then - export SINGULARITY_BIND="${storage}:/tmp" + export SINGULARITY_BIND="${STORAGE}:/tmp" fi echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" -# load modules LOAD_MODULES is not empty +# load modules if LOAD_MODULES is not empty if [[ ! -z ${LOAD_MODULES} ]]; then for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') do @@ -76,3 +83,45 @@ else echo "bot/build.sh: no modules to be loaded" fi +# determine repository to be used from entry .repository in cfg/job.cfg +REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) +EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos.cfg"' ${JOB_CFG_FILE}) + +# determine architecture to be used from entry .architecture in cfg/job.cfg +# default: leave empty to let downstream script(s) determine subdir to be used +EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) + +source init/minimal_eessi_env + +# TODO +# - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh +# - add handling of http(s)_proxy to eessi_container.sh, in there needs the +# CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust +# way to determine the IP address of a proxy) +# - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying +# files into './cfg/.' and defining '.repository.repos_cfg_file' in './cfg/job.cfg') + +# prepare options and directories for calling eessi_container.sh +mkdir -p previous_tmp +run_outerr=$(mktemp eessi_container.outerr.XXXXXXXXXX) +CONTAINER_OPT= +if [[ ! -z ${CONTAINER} ]]; then + CONTAINER_OPT="--container ${CONTAINER}" +fi +HTTP_PROXY_OPT= +if [[ ! -z ${HTTP_PROXY} ]]; then + HTTP_PROXY_OPT="--container ${HTTP_PROXY}" +fi +REPOSITORY_OPT= +if [[ ! -z ${REPOSITORY} ]]; then + REPOSITORY_OPT="--repository ${REPOSITORY}" +fi +./eessi_container.sh --access rw \ + ${CONTAINER_OPT} \ + ${HTTP_PROXY_OPT} \ + --info \ + --mode run \ + ${REPOSITORY_OPT} \ + --save $(pwd)/previous_tmp \ + --storage ${STORAGE} \ + ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a $(run_outerr) diff --git a/eessi_container.sh b/eessi_container.sh index 57b003368b..0016c8c9f0 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -38,8 +38,8 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) -#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) -#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) +HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) +HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) # CernVM-FS settings @@ -59,43 +59,43 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" display_help() { echo "usage: $0 [OPTIONS] [SCRIPT]" echo " OPTIONS:" - echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" - echo " -c | --container IMG - image file or URL defining the container to use" - echo " [default: docker://ghcr.io/eessi/build-node:debian10]" - echo " -h | --help - display this usage information [default: false]" - echo " -g | --storage DIR - directory space on host machine (used for" - echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" - echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" - echo " MODE==run (run a script) [default: shell]" - echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot]" - echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," - echo " where DIR points to a previously used tmp directory" - echo " (check for output 'Using DIR as tmp ...' of a previous" - echo " run) and TGZ is the path to a tarball which is" - echo " unpacked the tmp dir stored on the local storage space" - echo " (see option --storage above) [default: not set]" - echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" - echo " directory DIR or provided with the fixed full path TGZ" - echo " when a directory is provided, the format of the" - echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" - echo " [default: not set]" + echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -c | --container IMG - image file or URL defining the container to use" + echo " [default: docker://ghcr.io/eessi/build-node:debian11]" + echo " -h | --help - display this usage information [default: false]" + echo " -i | --info - display configuration information [default: false]" + echo " -g | --storage DIR - directory space on host machine (used for" + echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" + echo " MODE==run (run a script) [default: shell]" + echo " -r | --repository CFG - configuration file or identifier defining the" + echo " repository to use [default: EESSI-pilot]" + echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," + echo " where DIR points to a previously used tmp directory" + echo " (check for output 'Using DIR as tmp ...' of a previous" + echo " run) and TGZ is the path to a tarball which is" + echo " unpacked the tmp dir stored on the local storage space" + echo " (see option --storage above) [default: not set]" + echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" + echo " directory DIR or provided with the fixed full path TGZ" + echo " when a directory is provided, the format of the" + echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" + echo " [default: not set]" + echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" + echo " [default: not set]; uses env var \$http_proxy if set" + echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" + echo " [default: not set]; uses env var \$https_proxy if set" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." echo echo " FEATURES/OPTIONS to be implemented:" - echo " -d | --dry-run - run script except for executing the container," - echo " print information about setup [default: false]" - echo " -i | --info - display configured repositories [default: false]" - #echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" - #echo " [default: not set]" - #echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" - #echo " [default: not set]" + echo " -d | --dry-run - run script except for executing the container," + echo " print information about setup [default: false]" } # set defaults for command line arguments ACCESS="ro" -CONTAINER="docker://ghcr.io/eessi/build-node:debian10" +CONTAINER="docker://ghcr.io/eessi/build-node:debian11" DRY_RUN=0 INFO=0 STORAGE= @@ -103,8 +103,8 @@ MODE="shell" REPOSITORY="EESSI-pilot" RESUME= SAVE= -#HTTP_PROXY= -#HTTPS_PROXY= +HTTP_PROXY=${http_proxy:-} +HTTPS_PROXY=${https_proxy:-} RUN_SCRIPT_AND_ARGS= POSITIONAL_ARGS=() @@ -151,16 +151,16 @@ while [[ $# -gt 0 ]]; do RESUME="$2" shift 2 ;; -# -x|--http-proxy) -# HTTP_PROXY="$2" -# export http_proxy=${HTTP_PROXY} -# shift 2 -# ;; -# -y|--https-proxy) -# HTTPS_PROXY="$2" -# export https_proxy=${HTTPS_PROXY} -# shift 2 -# ;; + -x|--http-proxy) + HTTP_PROXY="$2" + export http_proxy=${HTTP_PROXY} + shift 2 + ;; + -y|--https-proxy) + HTTPS_PROXY="$2" + export https_proxy=${HTTPS_PROXY} + shift 2 + ;; -*|--*) fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" ;; @@ -281,26 +281,26 @@ fi # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} -[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" +[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR='${EESSI_TMPDIR}'" # configure Singularity export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache mkdir -p ${SINGULARITY_CACHEDIR} -[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" +[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" # set env vars and create directories for CernVM-FS EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} mkdir -p ${EESSI_CVMFS_VAR_LIB} mkdir -p ${EESSI_CVMFS_VAR_RUN} -[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}" -[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}" +[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB='${EESSI_CVMFS_VAR_LIB}'" +[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN='${EESSI_CVMFS_VAR_RUN}'" # allow that SINGULARITY_HOME is defined before script is run if [[ -z ${SINGULARITY_HOME} ]]; then export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" mkdir -p ${EESSI_TMPDIR}/home - [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" + [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME='${SINGULARITY_HOME}'" fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) @@ -308,9 +308,9 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" -# set up repository config (always create cfg dir and populate it with info when +# set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) -mkdir -p ${EESSI_TMPDIR}/cfg +mkdir -p ${EESSI_TMPDIR}/repos_cfg if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then # need to source defaults as late as possible (see other sourcing below) source ${base_dir}/init/eessi_defaults @@ -367,13 +367,13 @@ else # only unpack config_bundle if we're not resuming from a previous run if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg + tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg fi for src in "${!cfg_file_map[@]}" do target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}" + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" done export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" @@ -397,8 +397,8 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work mkdir -p ${EESSI_TMPDIR}/overlay-upper mkdir -p ${EESSI_TMPDIR}/overlay-work - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" + [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER='${EESSI_CVMFS_OVERLAY_UPPER}'" + [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK='${EESSI_CVMFS_OVERLAY_WORK}'" # set environment variables for fuse mounts in Singularity container export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" @@ -427,7 +427,14 @@ if [[ -z ${SINGULARITY_BIND} ]]; then else export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}" fi -[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" +[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" + +# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) +if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then + export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility) + export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +fi echo "Launching container with command (next line):" echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" From b5c07eeb99e77191d818e64788b0d419031cf526 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 27 Jan 2023 20:30:58 +0100 Subject: [PATCH 032/106] provide CVMFS_HTTP_PROXY if necessary --- bot/build.sh | 3 ++- eessi_container.sh | 39 ++++++++++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index e7cc7983bb..a4f54cba3c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -95,7 +95,8 @@ source init/minimal_eessi_env # TODO # - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh -# - add handling of http(s)_proxy to eessi_container.sh, in there needs the +# TODO ensure that the bot makes use of that. +# - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the # CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust # way to determine the IP address of a proxy) # - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying diff --git a/eessi_container.sh b/eessi_container.sh index 0016c8c9f0..c2b4988162 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -19,8 +19,8 @@ # 2. set up host storage/tmp # 3. set up common vars and directories # 4. set up vars specific to a scenario -# 5. initialize host storage/tmp from previous run if provided -# 6. run container +# 5. run container +# 6. save tmp (if requested) # -. initial settings & exit codes base_dir=$(dirname $(realpath $0)) @@ -276,7 +276,7 @@ fi # |-overlay-upper # |-overlay-work # |-home -# |-cfg +# |-repos_cfg # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} @@ -358,9 +358,9 @@ else cfg_init_file_map "${config_map}" [[ ${INFO} -eq 1 ]] && cfg_print_map - # TODO use information to set up dir ${EESSI_TMPDIR}/cfg, - # define BIND mounts and override repo name and version - # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg + # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, + # define BIND mounts and override repo name and version + # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg if [[ ! -r ${config_bundle} ]]; then fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} fi @@ -381,6 +381,26 @@ else source ${base_dir}/init/eessi_defaults fi +# if http_proxy is not empty, we assume that the machine accesses internet +# via a proxy. then we need to add CVMFS_HTTP_PROXY to +# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND +# MOUNT if it was not yet in BIND_PATHS) +if [[ ! -z ${http_proxy} ]]; then + # TODO tolerate other formats for proxy URLs, for now assume format is + # http://SOME_HOSTNAME:SOME_PORT + PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http:// + PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port + HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/} + HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) + echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \ + "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ + >> ${EESSI_TMPDIR}/repos_cfg/default.local + cat ${EESSI_TMPDIR}/repos_cfg/default.local + # if default.local is not BIND mounted into container, add it to BIND_PATHS + if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then + export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" + fi +fi # 4. set up vars and dirs specific to a scenario @@ -417,10 +437,7 @@ if [[ "${ACCESS}" == "rw" ]]; then fi -# 5. initialize host storage/tmp from previous run if provided - - -# 6. run container +# 5. run container # final settings if [[ -z ${SINGULARITY_BIND} ]]; then export SINGULARITY_BIND="${BIND_PATHS}" @@ -442,7 +459,7 @@ echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ # provided to the script singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} -# 7. save tmp if requested (arg -s|--save) +# 6. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then # Note, for now we don't try to be smart and record in any way the OS and # ARCH which might have been used internally, eg, when software packages From 4ff37218793559a6e3a8ea9cef58198e33b7dcdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Fri, 27 Jan 2023 22:41:36 +0100 Subject: [PATCH 033/106] add functions to derive hostname and port number from proxy URL --- utils.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/utils.sh b/utils.sh index db533adc12..fb4eff1e3f 100644 --- a/utils.sh +++ b/utils.sh @@ -64,6 +64,30 @@ function get_path_for_tool { fi } +function get_host_from_url { + url=$1 + re="(http|https)://([^/:]+)" + if [[ $url =~ $re ]]; then + echo ${BASH_REMATCH[2]} + return 0 + else + echo "" + return 1 + fi +} + +function get_port_from_url { + url=$1 + re="(http|https)://[^:]+:([0-9]+)" + if [[ $url =~ $re ]]; then + echo ${BASH_REMATCH[2]} + return 0 + else + echo "" + return 1 + fi +} + function get_ipv4_address { hname=$1 hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) From 7a58fbff63712d81dc7dd3206ae2d35535c58b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Fri, 27 Jan 2023 22:43:27 +0100 Subject: [PATCH 034/106] add function to determine config sections --- cfg_files.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/cfg_files.sh b/cfg_files.sh index 885ebd0877..ffd29db50a 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -47,9 +47,7 @@ function cfg_load { else val=$(cfg_get_key_value $line) # trim leading and trailing spaces as well - #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces) - #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces) if [[ -n "$cur_key" ]]; then # section + key is the associative in bash array, the field separator is space @@ -69,6 +67,20 @@ function cfg_print { done } +function cfg_sections { + declare -A sections + for key in "${!cfg_repos[@]}" + do + # extract section from the associative key + section=$(echo $key | cut -f1 -d ' ') + sections[${section}]=1 + done + for repo in "${!sections[@]}" + do + echo "${repo}" + done +} + function cfg_get_value { section=$1 key=$2 From a5ea7d8aea2c92380839ecbf290c28d278c43835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Fri, 27 Jan 2023 22:44:06 +0100 Subject: [PATCH 035/106] add parameter to list repositories, fix bugs fixed bugs - determining IP address for proxy host was wrong - handling of positional parameters for executing commands with arguments was wrong --- eessi_container.sh | 43 ++++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index c2b4988162..d8d7cb2bdb 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -66,6 +66,7 @@ display_help() { echo " -i | --info - display configuration information [default: false]" echo " -g | --storage DIR - directory space on host machine (used for" echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -l | --list-repos - list available repository identifiers [default: false]" echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" echo " MODE==run (run a script) [default: shell]" echo " -r | --repository CFG - configuration file or identifier defining the" @@ -99,13 +100,13 @@ CONTAINER="docker://ghcr.io/eessi/build-node:debian11" DRY_RUN=0 INFO=0 STORAGE= +LIST_REPOS=0 MODE="shell" REPOSITORY="EESSI-pilot" RESUME= SAVE= HTTP_PROXY=${http_proxy:-} HTTPS_PROXY=${https_proxy:-} -RUN_SCRIPT_AND_ARGS= POSITIONAL_ARGS=() @@ -135,6 +136,10 @@ while [[ $# -gt 0 ]]; do INFO=1 shift 1 ;; + -l|--list-repos) + LIST_REPOS=1 + shift 1 + ;; -m|--mode) MODE="$2" shift 2 @@ -173,6 +178,17 @@ done set -- "${POSITIONAL_ARGS[@]}" +if [[ ${LIST_REPOS} -eq 1 ]]; then + echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':" + echo " EESSI-pilot [default]" + cfg_load ${EESSI_REPOS_CFG_FILE} + sections=$(cfg_sections) + while IFS= read -r repo_id + do + echo " ${repo_id}" + done <<< "${sections}" + exit 0 +fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported @@ -214,8 +230,6 @@ fi if [[ "${MODE}" == "run" ]]; then if [[ $# -eq 0 ]]; then fatal_error "no command specified to run?!" "${RUN_SCRIPT_MISSING_EXITCODE}" - else - RUN_SCRIPT_AND_ARGS=$@ fi fi @@ -387,17 +401,20 @@ fi # MOUNT if it was not yet in BIND_PATHS) if [[ ! -z ${http_proxy} ]]; then # TODO tolerate other formats for proxy URLs, for now assume format is - # http://SOME_HOSTNAME:SOME_PORT - PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http:// - PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port - HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/} - HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) - echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \ - "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ + # http://SOME_HOSTNAME:SOME_PORT/ + [[ ${INFO} -eq 1 ]] && echo "http_proxy='${http_proxy}'" + PROXY_HOST=$(get_host_from_url ${http_proxy}) + [[ ${INFO} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'" + PROXY_PORT=$(get_port_from_url ${http_proxy}) + [[ ${INFO} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'" + HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) + [[ ${INFO} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'" + echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ >> ${EESSI_TMPDIR}/repos_cfg/default.local cat ${EESSI_TMPDIR}/repos_cfg/default.local + # if default.local is not BIND mounted into container, add it to BIND_PATHS - if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then + if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" fi fi @@ -454,10 +471,10 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then fi echo "Launching container with command (next line):" -echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" +echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level # provided to the script -singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} +singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" # 6. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then From 6ed3d85ff5f11bca37f838c533320733c3ddaf08 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 14:27:26 +0100 Subject: [PATCH 036/106] fix bash pattern matching operator --- eessi_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index c2b4988162..0cab47e8fb 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -397,7 +397,7 @@ if [[ ! -z ${http_proxy} ]]; then >> ${EESSI_TMPDIR}/repos_cfg/default.local cat ${EESSI_TMPDIR}/repos_cfg/default.local # if default.local is not BIND mounted into container, add it to BIND_PATHS - if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then + if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" fi fi From fd5cc3d339bffd7db248c43be909d195db8a64bd Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 14:29:25 +0100 Subject: [PATCH 037/106] various updates to bot/build.sh - added handling of HTTPS_PROXY - moved CONTAINER setting from site_config to repository section (different repositories could require different containers) - added handling of CPU_TARGET and EESSI_SOFTWARE_SUBDIR_OVERRIDE - updated TODOs - added setting of EESSI_REPOS_CFG_FILE_OVERRIDE - bug fixes --- bot/build.sh | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index a4f54cba3c..3d06477a15 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -46,6 +46,10 @@ echo "obtaining configuration settings from '${JOB_CFG_FILE}'" HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE}) echo "HTTP_PROXY='${HTTP_PROXY}'" +# if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy +HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // "${https_proxy}"' ${JOB_CFG_FILE}) +echo "HTTPS_PROXY='${HTTPS_PROXY}'" + LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) echo "LOCAL_TMP='${LOCAL_TMP}'" # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided @@ -61,7 +65,7 @@ LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE}) echo "LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND -CONTAINER=$(${YQ} '.site_config.container // ""' ${JOB_CFG_FILE}) +CONTAINER=$(${YQ} '.repository.container // ""' ${JOB_CFG_FILE}) export SINGULARITY_HOME="$(pwd):/eessi_bot_job" export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} @@ -89,13 +93,19 @@ EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos. # determine architecture to be used from entry .architecture in cfg/job.cfg # default: leave empty to let downstream script(s) determine subdir to be used -EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) +if [[ ! -z "${CPU_TARGET}" ]]; then + EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET} +else + EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) +fi source init/minimal_eessi_env # TODO # - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh -# TODO ensure that the bot makes use of that. +# TODO ensure that the bot makes use of that. (currently sets env var +# CPU_TARGET & adds --export=ALL,CPU_TARGET=val to sbatch command ... also +# add it to cfg/job.cfg - .architecture.software_subdir) # - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the # CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust # way to determine the IP address of a proxy) @@ -111,18 +121,25 @@ if [[ ! -z ${CONTAINER} ]]; then fi HTTP_PROXY_OPT= if [[ ! -z ${HTTP_PROXY} ]]; then - HTTP_PROXY_OPT="--container ${HTTP_PROXY}" + HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}" +fi +HTTPS_PROXY_OPT= +if [[ ! -z ${HTTPS_PROXY} ]]; then + HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}" fi REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi +# set EESSI_REPOS_CFG_FILE_OVERRIDE to ./cfg/repos.cfg +export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ + ${HTTPS_PROXY_OPT} \ --info \ --mode run \ ${REPOSITORY_OPT} \ --save $(pwd)/previous_tmp \ --storage ${STORAGE} \ - ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a $(run_outerr) + ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a ${run_outerr} From b2eab080cba313c72e2c72f91429242b1bf04990 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 19:10:40 +0100 Subject: [PATCH 038/106] replace FILE with DIR in EESSI_REPOS_CFG_FILE_OVERRIDE --- bot/build.sh | 8 ++++---- eessi_container.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 3d06477a15..7de3594d41 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -89,7 +89,7 @@ fi # determine repository to be used from entry .repository in cfg/job.cfg REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) -EESSI_REPOS_CFG_FILE_OVERRIDE=$(${YQ} '.repository.repos_cfg_file // "cfg/repos.cfg"' ${JOB_CFG_FILE}) +EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // "cfg"' ${JOB_CFG_FILE}) # determine architecture to be used from entry .architecture in cfg/job.cfg # default: leave empty to let downstream script(s) determine subdir to be used @@ -131,8 +131,8 @@ REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi -# set EESSI_REPOS_CFG_FILE_OVERRIDE to ./cfg/repos.cfg -export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg +# set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg +export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ @@ -142,4 +142,4 @@ export EESSI_REPOS_CFG_FILE_OVERRIDE=$(pwd)/cfg/repos.cfg ${REPOSITORY_OPT} \ --save $(pwd)/previous_tmp \ --storage ${STORAGE} \ - ./install_software_layer.sh TODO installopts TODO commonopts TODO remainingopts "$@" 2>&1 | tee -a ${run_outerr} + ./install_software_layer.sh "$@" 2>&1 | tee -a ${run_outerr} diff --git a/eessi_container.sh b/eessi_container.sh index d8d7cb2bdb..4ca494e136 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -47,8 +47,8 @@ CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" # repository cfg file, default name (default location: $PWD) -# can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE -export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" +# can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg" # 0. parse args From 7e200a185782727ad61f7c1e78f4b52228300a4d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 19:56:41 +0100 Subject: [PATCH 039/106] fix whitespace issues --- cfg_files.sh | 1 - eessi_container.sh | 2 +- init/eessi_defaults | 1 - utils.sh | 4 ++-- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cfg_files.sh b/cfg_files.sh index ffd29db50a..57ea2f7c03 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -165,4 +165,3 @@ function cfg_print_map { echo "${index} --> ${cfg_file_map[${index}]}" done } - diff --git a/eessi_container.sh b/eessi_container.sh index 4ca494e136..3a9fac2955 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -407,7 +407,7 @@ if [[ ! -z ${http_proxy} ]]; then [[ ${INFO} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'" PROXY_PORT=$(get_port_from_url ${http_proxy}) [[ ${INFO} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'" - HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) + HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) [[ ${INFO} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'" echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ >> ${EESSI_TMPDIR}/repos_cfg/default.local diff --git a/init/eessi_defaults b/init/eessi_defaults index 1b5ce07fb1..f482cbc269 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -10,4 +10,3 @@ export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" - diff --git a/utils.sh b/utils.sh index fb4eff1e3f..d0da95e87f 100644 --- a/utils.sh +++ b/utils.sh @@ -73,7 +73,7 @@ function get_host_from_url { else echo "" return 1 - fi + fi } function get_port_from_url { @@ -85,7 +85,7 @@ function get_port_from_url { else echo "" return 1 - fi + fi } function get_ipv4_address { From 57b6f27191ffce26c05065b1ce2a7e0841309e4d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 30 Jan 2023 17:48:52 +0100 Subject: [PATCH 040/106] fix handling of HTTP(S)_PROXY settings --- bot/build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 7de3594d41..d82048e86f 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -43,11 +43,13 @@ fi echo "obtaining configuration settings from '${JOB_CFG_FILE}'" # if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy -HTTP_PROXY=$(${YQ} '.site_config.http_proxy // "${http_proxy}"' ${JOB_CFG_FILE}) +HTTP_PROXY=$(${YQ} '.site_config.http_proxy // ""' ${JOB_CFG_FILE}) +HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} echo "HTTP_PROXY='${HTTP_PROXY}'" # if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy -HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // "${https_proxy}"' ${JOB_CFG_FILE}) +HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // ""' ${JOB_CFG_FILE}) +HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} echo "HTTPS_PROXY='${HTTPS_PROXY}'" LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) From f513b120b7264c0a4af1479feac533d86b95b24f Mon Sep 17 00:00:00 2001 From: Michael Huebner Date: Thu, 2 Feb 2023 15:35:42 +0100 Subject: [PATCH 041/106] Add CI test for the create_directory_tarballs.sh script --- .github/workflows/tests_scripts.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 53f4bd13be..a78c71dc9a 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -51,3 +51,13 @@ jobs: # force using x86_64/generic, to avoid triggering an installation from scratch sed -i "s@./EESSI-pilot-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-pilot-install-software.sh\"@g" install_software_layer.sh ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh + + - name: test create_directory_tarballs.sh script + run: | + # scripts need to be copied to /tmp, + # since create_directory_tarballs.sh must be accessible from within build container + cp -a * /tmp/ + cd /tmp + ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh + # check if tarballs have been produced + ls -l *.tar.gz From e4702ee06c71fba563e9d4f362fdf3aac41d394f Mon Sep 17 00:00:00 2001 From: Michael Huebner Date: Thu, 2 Feb 2023 17:47:24 +0100 Subject: [PATCH 042/106] Need to specify the EESSI version to test create_directory_tarballs --- .github/workflows/tests_scripts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index a78c71dc9a..23bb36a7e8 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -58,6 +58,6 @@ jobs: # since create_directory_tarballs.sh must be accessible from within build container cp -a * /tmp/ cd /tmp - ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh + ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2021.12 # check if tarballs have been produced ls -l *.tar.gz From c4b9cf5e2a4dcd75fca54e9fc9da2c6b69011d9e Mon Sep 17 00:00:00 2001 From: Michael Huebner Date: Thu, 2 Feb 2023 18:15:34 +0100 Subject: [PATCH 043/106] Fix wildcard usage when building tarball of the scripts dir --- create_directory_tarballs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_directory_tarballs.sh b/create_directory_tarballs.sh index c254c2c229..be282463eb 100755 --- a/create_directory_tarballs.sh +++ b/create_directory_tarballs.sh @@ -42,7 +42,7 @@ echo_green "Done! Created tarball ${tarname}." tartmp=$(mktemp -t -d scripts.XXXXX) mkdir "${tartmp}/${version}" tarname="eessi-${version}-scripts-$(date +%s).tar.gz" -curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards */scripts/ +curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards '*/scripts/' tar czf "${tarname}" -C "${tartmp}" "${version}" rm -rf "${tartmp}" From fce504fb466be005f1d417e96e47f442c8f29c52 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 3 Feb 2023 11:54:47 +0100 Subject: [PATCH 044/106] include changes made to test PR for building with the bot --- bot/build.sh | 87 ++++++++++++++++++++++++++++++++------ eessi_container.sh | 50 +++++++++++++++------- run_in_compat_layer_env.sh | 15 ++++++- 3 files changed, 121 insertions(+), 31 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index d82048e86f..8565511bfa 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -72,12 +72,6 @@ export SINGULARITY_HOME="$(pwd):/eessi_bot_job" export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} -if [[ ${STORAGE} != /tmp* ]] ; -then - export SINGULARITY_BIND="${STORAGE}:/tmp" -fi -echo "SINGULARITY_BIND='${SINGULARITY_BIND}'" - # load modules if LOAD_MODULES is not empty if [[ ! -z ${LOAD_MODULES} ]]; then for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') @@ -91,7 +85,20 @@ fi # determine repository to be used from entry .repository in cfg/job.cfg REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) -EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // "cfg"' ${JOB_CFG_FILE}) +EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE}) +export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} + +# determine pilot version to be used from .repository.repo_version in cfg/job.cfg +# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# next script (eessi_container.sh) makes use of it via sourcing init scripts +# (e.g., init/eessi_defaults or init/minimal_eessi_env) +export EESSI_PILOT_VERSION_OVERRIDE=$(${YQ} '.repository.repo_version // ""' ${JOB_CFG_FILE}) + +# determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg +# here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down +# "source init/eessi_defaults" via sourcing init/minimal_eessi_env +export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG_FILE}) + # determine architecture to be used from entry .architecture in cfg/job.cfg # default: leave empty to let downstream script(s) determine subdir to be used @@ -100,8 +107,11 @@ if [[ ! -z "${CPU_TARGET}" ]]; then else EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) fi +export EESSI_SOFTWARE_SUBDIR_OVERRIDE -source init/minimal_eessi_env +# get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux) +EESSI_OS_TYPE=$(${YQ} '.architecture.os_type // ""' ${JOB_CFG_FILE}) +export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} # TODO # - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh @@ -112,11 +122,9 @@ source init/minimal_eessi_env # CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust # way to determine the IP address of a proxy) # - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying -# files into './cfg/.' and defining '.repository.repos_cfg_file' in './cfg/job.cfg') +# files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg') # prepare options and directories for calling eessi_container.sh -mkdir -p previous_tmp -run_outerr=$(mktemp eessi_container.outerr.XXXXXXXXXX) CONTAINER_OPT= if [[ ! -z ${CONTAINER} ]]; then CONTAINER_OPT="--container ${CONTAINER}" @@ -133,8 +141,21 @@ REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi +mkdir -p previous_tmp +build_outerr=$(mktemp build.outerr.XXXX) +echo "Executing command to build software:" +echo "./eessi_container.sh --access rw" +echo " ${CONTAINER_OPT}" +echo " ${HTTP_PROXY_OPT}" +echo " ${HTTPS_PROXY_OPT}" +echo " --info" +echo " --mode run" +echo " ${REPOSITORY_OPT}" +echo " --save ${PWD}/previous_tmp" +echo " --storage ${STORAGE}" +echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg -export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg +export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ @@ -142,6 +163,44 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=$(pwd)/cfg --info \ --mode run \ ${REPOSITORY_OPT} \ - --save $(pwd)/previous_tmp \ + --save ${PWD}/previous_tmp \ --storage ${STORAGE} \ - ./install_software_layer.sh "$@" 2>&1 | tee -a ${run_outerr} + ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr} + +# determine temporary directory to resume from +BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //") + +tar_outerr=$(mktemp tar.outerr.XXXX) +timestamp=$(date +%s) +# to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now +source init/eessi_defaults +export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) + +# value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be +# synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh +# TODO should we make this a configurable parameter of eessi_container.sh using +# /tmp as default? +TMP_IN_CONTAINER=/tmp +echo "Executing command to create tarball:" +echo "./eessi_container.sh --access rw" +echo " ${CONTAINER_OPT}" +echo " ${HTTP_PROXY_OPT}" +echo " ${HTTPS_PROXY_OPT}" +echo " --info" +echo " --mode run" +echo " ${REPOSITORY_OPT}" +echo " --resume ${BUILD_TMPDIR}" +echo " --save ${PWD}/previous_tmp" +echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +./eessi_container.sh --access rw \ + ${CONTAINER_OPT} \ + ${HTTP_PROXY_OPT} \ + ${HTTPS_PROXY_OPT} \ + --info \ + --mode run \ + ${REPOSITORY_OPT} \ + --resume ${BUILD_TMPDIR} \ + --save ${PWD}/previous_tmp \ + ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + +exit 0 diff --git a/eessi_container.sh b/eessi_container.sh index 3a9fac2955..1eb630a096 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -46,9 +46,15 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" +# directory for tmp used inside container +export TMP_IN_CONTAINER=/tmp + # repository cfg file, default name (default location: $PWD) # can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE -export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg" +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg" +# other repository cfg files in directory, default location: $PWD +# can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE +export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}" # 0. parse args @@ -193,7 +199,7 @@ fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then - fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" + fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi # TODO (arg -c|--container) check container (is it a file or URL & access those) @@ -205,11 +211,14 @@ fi # (arg -m|--mode) check if MODE is known if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then - fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" + fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" fi # TODO (arg -r|--repository) check if repository is known # REPOSITORY_ERROR_EXITCODE +if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then + fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" +fi # TODO (arg -u|--resume) check if it exists, if user has read permission, # if it contains data from a previous run @@ -243,6 +252,7 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then # skip creating a new tmp directory, just set environment variables echo "Resuming from previous run using temporary storage at ${RESUME}" EESSI_HOST_STORAGE=${RESUME} + echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}" else # we need a tmp location (and possibly init it with ${RESUME} if it was not # a directory @@ -319,7 +329,9 @@ fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" -BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" +# provide a '/tmp' inside the container +BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" + [[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" # set up repository config (always create directory repos_cfg and populate it with info when @@ -375,13 +387,22 @@ else # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, # define BIND mounts and override repo name and version # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg - if [[ ! -r ${config_bundle} ]]; then - fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} + # if config_bundle is relative path (no '/' at start) prepend it with + # EESSI_REPOS_CFG_DIR + config_bundle_path= + if [[ ! "${config_bundle}" =~ ^/ ]]; then + config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle} + else + config_bundle_path=${config_bundle} + fi + + if [[ ! -r ${config_bundle_path} ]]; then + fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} fi # only unpack config_bundle if we're not resuming from a previous run if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg + tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg fi for src in "${!cfg_file_map[@]}" @@ -430,12 +451,8 @@ if [[ "${ACCESS}" == "ro" ]]; then fi if [[ "${ACCESS}" == "rw" ]]; then - EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper - EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work mkdir -p ${EESSI_TMPDIR}/overlay-upper mkdir -p ${EESSI_TMPDIR}/overlay-work - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER='${EESSI_CVMFS_OVERLAY_UPPER}'" - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK='${EESSI_CVMFS_OVERLAY_WORK}'" # set environment variables for fuse mounts in Singularity container export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" @@ -444,8 +461,8 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" export EESSI_PILOT_WRITABLE_OVERLAY @@ -470,11 +487,13 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi +# if INFO is set to 1 (arg --info), add argument '-q' +RUN_QUIET=${INFO:--q} echo "Launching container with command (next line):" -echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" +echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level # provided to the script -singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" +singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" # 6. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then @@ -493,6 +512,7 @@ if [[ ! -z ${SAVE} ]]; then fi tar cf ${TGZ} -C ${EESSI_TMPDIR} . echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')" + echo "RESUME_FROM_TGZ ${TGZ}" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index e4850d677d..c70077bf15 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -3,7 +3,6 @@ base_dir=$(dirname $(realpath $0)) source ${base_dir}/init/eessi_defaults -BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian10" if [ -z $EESSI_PILOT_VERSION ]; then echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2 exit 1 @@ -18,6 +17,18 @@ INPUT=$(echo "$@") if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" fi +if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then + INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" +fi +if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then + INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}" +fi +if [ ! -z ${http_proxy} ]; then + INPUT="export http_proxy=${http_proxy}; ${INPUT}" +fi +if [ ! -z ${https_proxy} ]; then + INPUT="export https_proxy=${https_proxy}; ${INPUT}" +fi -echo "Running '${INPUT}' in EESSI ${EESSI_PILOT_VERSION} compatibility layer environment..." +echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..." ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}" From 3a6fe0e4ec0b10387bfd43cf346c3f052b4b85f3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 8 Feb 2023 08:49:37 +0100 Subject: [PATCH 045/106] update location of utils.sh, uniform way to determine 'TOPDIR' --- eessi_container.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 57b003368b..29a2ea4565 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -23,13 +23,13 @@ # 6. run container # -. initial settings & exit codes -base_dir=$(dirname $(realpath $0)) +TOPDIR=$(dirname $(realpath $0)) -source ${base_dir}/utils.sh -source ${base_dir}/cfg_files.sh +source ${TOPDIR}/scripts/utils.sh +source ${TOPDIR}/cfg_files.sh # exit codes: bitwise shift codes to allow for combination of exit codes -# ANY_ERROR_EXITCODE is sourced from ${base_dir}/utils.sh +# ANY_ERROR_EXITCODE is sourced from ${TOPDIR}/scripts/utils.sh CMDLINE_ARG_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 1)) ACCESS_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 2)) CONTAINER_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 3)) From 756b1b62407c250dfe8b7dcf9d87511375c7580c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 8 Feb 2023 08:53:54 +0100 Subject: [PATCH 046/106] bump default container version --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 29a2ea4565..2b2c0716ca 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -61,7 +61,7 @@ display_help() { echo " OPTIONS:" echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" echo " -c | --container IMG - image file or URL defining the container to use" - echo " [default: docker://ghcr.io/eessi/build-node:debian10]" + echo " [default: docker://ghcr.io/eessi/build-node:debian11]" echo " -h | --help - display this usage information [default: false]" echo " -g | --storage DIR - directory space on host machine (used for" echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" @@ -95,7 +95,7 @@ display_help() { # set defaults for command line arguments ACCESS="ro" -CONTAINER="docker://ghcr.io/eessi/build-node:debian10" +CONTAINER="docker://ghcr.io/eessi/build-node:debian11" DRY_RUN=0 INFO=0 STORAGE= From 4c435ab94185c8ab1cb64eebc97ffa96d47e83fb Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 8 Feb 2023 08:57:00 +0100 Subject: [PATCH 047/106] clarifying default value for repository arg --- eessi_container.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 2b2c0716ca..b6eb3894ef 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -68,7 +68,8 @@ display_help() { echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" echo " MODE==run (run a script) [default: shell]" echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot]" + echo " repository to use [default: EESSI-pilot via" + echo " container configuration]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," echo " where DIR points to a previously used tmp directory" echo " (check for output 'Using DIR as tmp ...' of a previous" From 990ebdae4dd77e396add2b2237efa1f6a70708d6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Feb 2023 08:57:54 +0100 Subject: [PATCH 048/106] move commands to install Apptainer in GitHub Actions Ubuntu VM to a script --- .github/workflows/tests_scripts.yml | 11 +---------- install_apptainer_ubuntu.sh | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 10 deletions(-) create mode 100755 install_apptainer_ubuntu.sh diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 9c4975c381..ca23a15c40 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -27,18 +27,9 @@ jobs: - name: checkout uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - # see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 - name: install Apptainer run: | - sudo apt-get install alien - alien --version - apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') - curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/$apptainer_rpm - sudo alien -d $apptainer_rpm - sudo apt install ./apptainer*.deb - apptainer --version - # also check whether 'singularity' command is still provided by Apptainer installation - singularity --version + ./install_apptainer_ubuntu.sh - name: test install_software_layer.sh script run: | diff --git a/install_apptainer_ubuntu.sh b/install_apptainer_ubuntu.sh new file mode 100755 index 0000000000..c35c34cda6 --- /dev/null +++ b/install_apptainer_ubuntu.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +# see https://github.com/apptainer/singularity/issues/5390#issuecomment-899111181 +sudo apt-get install alien +alien --version +apptainer_rpm=$(curl --silent -L https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/ | grep 'apptainer-[0-9]' | sed 's/.*\(apptainer[0-9._a-z-]*.rpm\).*/\1/g') +curl -OL https://dl.fedoraproject.org/pub/epel/8/Everything/x86_64/Packages/a/${apptainer_rpm} +sudo alien -d ${apptainer_rpm} +sudo apt install ./apptainer*.deb +apptainer --version +# also check whether 'singularity' command is still provided by Apptainer installation +singularity --version From 3d143d853dbecffb746d5ab3610d188d4ca12bda Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Feb 2023 08:58:00 +0100 Subject: [PATCH 049/106] add GitHub Actions workflow for testing eessi_container.sh script --- .../workflows/test_eessi_container_script.yml | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/test_eessi_container_script.yml diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml new file mode 100644 index 0000000000..c02cd0dacf --- /dev/null +++ b/.github/workflows/test_eessi_container_script.yml @@ -0,0 +1,28 @@ +# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions +name: Tests for eessi_container.sh script +on: [push, pull_request, workflow_dispatch] +permissions: + contents: read # to fetch code (actions/checkout) +jobs: + eessi_container_script: + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + SCRIPT_ARGS: + - "--help" + - "--info" + steps: + - name: Check out software-layer repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Mount EESSI CernVM-FS pilot repository + uses: eessi/github-action-eessi@v2 + + - name: install Apptainer + run: | + ./install_apptainer_ubuntu.sh + + - name: Test eessi_container.sh script + run: | + ./eessi_container.sh ${{matrix.SCRIPT_ARGS}} From 5fe467a04cb188f47f2b52c0e7159ef64a38ab58 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Feb 2023 09:13:03 +0100 Subject: [PATCH 050/106] stop script as soon as any command does non-zero exit + undefined env vars are errors --- eessi_container.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index b6eb3894ef..c09ea75ebf 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -12,6 +12,10 @@ # license: GPLv2 # +# -e: stop script as soon as any command has non-zero exit code +# -u: treat usage of undefined variables as errors +set -e -u + # script overview # -. initial settings & exit codes # 0. parse args From 57b831526fadf55ca1f886fd92efc11d75aa93f4 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Feb 2023 09:24:42 +0100 Subject: [PATCH 051/106] fix typo --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index c09ea75ebf..283c41ddc0 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -318,7 +318,7 @@ BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" mkdir -p ${EESSI_TMPDIR}/cfg if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then # need to source defaults as late as possible (see other sourcing below) - source ${base_dir}/init/eessi_defaults + source ${TOPDIR}/init/eessi_defaults # strip "/cvmfs/" from default setting repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} @@ -383,7 +383,7 @@ else export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" # need to source defaults as late as possible (after *_OVERRIDEs) - source ${base_dir}/init/eessi_defaults + source ${TOPDIR}/init/eessi_defaults fi From fba368d19686c1f2edaeb524d0a78f86673b453e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Feb 2023 09:25:20 +0100 Subject: [PATCH 052/106] comment out 'set -e -u' for now --- eessi_container.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eessi_container.sh b/eessi_container.sh index 283c41ddc0..a63eb3ad44 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -14,7 +14,8 @@ # -e: stop script as soon as any command has non-zero exit code # -u: treat usage of undefined variables as errors -set -e -u +# FIXME commented out because it's OK (?) if some environment variables are not set (like $SINGULARITY_HOME) +# set -e -u # script overview # -. initial settings & exit codes From 09feff74d11d4cf14393cde33abd0672eb9f3bcf Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Feb 2023 10:27:05 +0100 Subject: [PATCH 053/106] rename --info to --verbose --- .../workflows/test_eessi_container_script.yml | 2 +- eessi_container.sh | 34 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index c02cd0dacf..a940a86924 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -11,7 +11,7 @@ jobs: matrix: SCRIPT_ARGS: - "--help" - - "--info" + - "--verbose" steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/eessi_container.sh b/eessi_container.sh index a63eb3ad44..166f5ff2a4 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -92,7 +92,7 @@ display_help() { echo " FEATURES/OPTIONS to be implemented:" echo " -d | --dry-run - run script except for executing the container," echo " print information about setup [default: false]" - echo " -i | --info - display configured repositories [default: false]" + echo " -v | --verbose - display more information [default: false]" #echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" #echo " [default: not set]" #echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" @@ -103,7 +103,7 @@ display_help() { ACCESS="ro" CONTAINER="docker://ghcr.io/eessi/build-node:debian11" DRY_RUN=0 -INFO=0 +VERBOSE=0 STORAGE= MODE="shell" REPOSITORY="EESSI-pilot" @@ -137,10 +137,6 @@ while [[ $# -gt 0 ]]; do display_help exit 0 ;; - -i|--info) - INFO=1 - shift 1 - ;; -m|--mode) MODE="$2" shift 2 @@ -157,6 +153,10 @@ while [[ $# -gt 0 ]]; do RESUME="$2" shift 2 ;; + -v|--verbose) + VERBOSE=1 + shift 1 + ;; # -x|--http-proxy) # HTTP_PROXY="$2" # export http_proxy=${HTTP_PROXY} @@ -261,7 +261,7 @@ else # mktemp falls back to using /tmp if TMPDIR is empty # TODO check if /tmp is writable, large enough and usable (different # features for ro-access and rw-access) - [[ ${INFO} -eq 1 ]] && echo "skipping sanity checks for /tmp" + [[ ${VERBOSE} -eq 1 ]] && echo "skipping sanity checks for /tmp" fi EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." @@ -287,32 +287,32 @@ fi # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} -[[ ${INFO} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" +[[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" # configure Singularity export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache mkdir -p ${SINGULARITY_CACHEDIR} -[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" +[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" # set env vars and create directories for CernVM-FS EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} mkdir -p ${EESSI_CVMFS_VAR_LIB} mkdir -p ${EESSI_CVMFS_VAR_RUN} -[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}" -[[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}" +[[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_VAR_LIB=${EESSI_CVMFS_VAR_LIB}" +[[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_VAR_RUN=${EESSI_CVMFS_VAR_RUN}" # allow that SINGULARITY_HOME is defined before script is run if [[ -z ${SINGULARITY_HOME} ]]; then export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" mkdir -p ${EESSI_TMPDIR}/home - [[ ${INFO} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" + [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" -[[ ${INFO} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" +[[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" # set up repository config (always create cfg dir and populate it with info when # arg -r|--repository is used) @@ -362,7 +362,7 @@ else # convert config_map into associative array cfg_file_map cfg_init_file_map "${config_map}" - [[ ${INFO} -eq 1 ]] && cfg_print_map + [[ ${VERBOSE} -eq 1 ]] && cfg_print_map # TODO use information to set up dir ${EESSI_TMPDIR}/cfg, # define BIND mounts and override repo name and version @@ -403,8 +403,8 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work mkdir -p ${EESSI_TMPDIR}/overlay-upper mkdir -p ${EESSI_TMPDIR}/overlay-work - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" - [[ ${INFO} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" + [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" + [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" # set environment variables for fuse mounts in Singularity container export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" @@ -433,7 +433,7 @@ if [[ -z ${SINGULARITY_BIND} ]]; then else export SINGULARITY_BIND="${SINGULARITY_BIND},${BIND_PATHS}" fi -[[ ${INFO} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" +[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" echo "Launching container with command (next line):" echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" From fe83f85fe93bdb492946355ce611fcc96906cf8f Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Feb 2023 11:07:58 +0100 Subject: [PATCH 054/106] pin eessi/github-action-eessi by hash --- .github/workflows/test_eessi_container_script.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index a940a86924..fc4cd01f48 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Mount EESSI CernVM-FS pilot repository - uses: eessi/github-action-eessi@v2 + uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13 - name: install Apptainer run: | From 125bfa25d30e849e783bacb68c16200bea75dd7e Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Feb 2023 11:23:38 +0100 Subject: [PATCH 055/106] use exit code of container command in eessi_container.sh --- eessi_container.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index 166f5ff2a4..81ad7b4b7a 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -440,6 +440,7 @@ echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level # provided to the script singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} +exit_code=$? # 7. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then @@ -461,3 +462,6 @@ if [[ ! -z ${SAVE} ]]; then fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) + +# use exit code of container command +exit ${exit_code} From be3e39909797986845f2be4274ee5c4375c2ae7b Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 8 Feb 2023 13:01:21 +0100 Subject: [PATCH 056/106] replace RUN_SCRIPT_AND_ARGS with --- eessi_container.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 166f5ff2a4..4d61677e46 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -111,7 +111,6 @@ RESUME= SAVE= #HTTP_PROXY= #HTTPS_PROXY= -RUN_SCRIPT_AND_ARGS= POSITIONAL_ARGS=() @@ -220,8 +219,6 @@ fi if [[ "${MODE}" == "run" ]]; then if [[ $# -eq 0 ]]; then fatal_error "no command specified to run?!" "${RUN_SCRIPT_MISSING_EXITCODE}" - else - RUN_SCRIPT_AND_ARGS=$@ fi fi @@ -436,10 +433,10 @@ fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" echo "Launching container with command (next line):" -echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} ${RUN_SCRIPT_AND_ARGS}" +echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level # provided to the script -singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} ${RUN_SCRIPT_AND_ARGS} +singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" # 7. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then From 27cbdf3e0c3507925148945e41dc0683772489c6 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Feb 2023 15:52:49 +0100 Subject: [PATCH 057/106] add checks for various use cases of eessi_container.sh script (+ no need to natively mount EESSI, since that's done in the container) --- .../workflows/test_eessi_container_script.yml | 86 +++++++++++++++++-- 1 file changed, 79 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index fc4cd01f48..d1b6b1cb74 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -9,20 +9,92 @@ jobs: strategy: fail-fast: false matrix: - SCRIPT_ARGS: - - "--help" - - "--verbose" + SCRIPT_TEST: + - help + - run + - shell + - container + - readwrite + - resume + - save steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - name: Mount EESSI CernVM-FS pilot repository - uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13 - - name: install Apptainer run: | ./install_apptainer_ubuntu.sh - name: Test eessi_container.sh script run: | - ./eessi_container.sh ${{matrix.SCRIPT_ARGS}} + test_cmd="cat /etc/os-release" + out_pattern="Debian GNU/Linux 11" + + if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then + ./eessi_container.sh --help + + # test use of --mode run + elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then + outfile=out_run.txt + echo "${test_cmd}" > test_script.sh + chmod u+x test_script.sh + export SINGULARITY_BIND="$PWD:/test" + ./eessi_container.sh --mode run /test/test_script.sh | tee ${outfile} + grep "${out_pattern}" ${outfile} + + # test use of --mode shell + elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then + outfile=out_shell.txt + ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} + grep "${out_pattern}" ${outfile} + + # test use of --container option, using a totally different container; + # cfr. https://github.com/easybuilders/easybuild-containers + elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then + outfile=out_container.txt + container="docker://ghcr.io/eessi/build-node:debian10" + ./eessi_container.sh --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} + grep "${out_pattern}" ${outfile} + + # test use of '--access rw' to get write access in container + elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then + outfile=out_readwrite.txt + fn="test_${RANDOM}.txt" + echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh + chmod u+x test_script.sh + export SINGULARITY_BIND="$PWD:/test" + ./eessi_container.sh --access rw --mode run /test/test_script.sh > ${outfile} + + tmpdir=$(grep "--resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") + # note: must use '--access rw' again here, since touched file is in overlay upper dir + ./eessi_container.sh --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} + grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile + + # test use of --resume + elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then + outfile=out_resume.txt + ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} + rm -f ${outfile} + + tmpdir=$(grep "--resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") + ./eessi_container.sh --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile} + grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile} + grep "${out_pattern}" ${outfile} + + # test use of --save (+ --resume) + elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then + outfile=out_save.txt + fn="test_${RANDOM}.txt" + test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}" + ./eessi_container.sh --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile} + rm -f ${outfile} + + ./eessi_container.sh --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} + grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile + + tar tfv test-save.tar | grep "overlay-upper/${fn}" + + else + echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2 + exit 1 + fi From b34afca53a24c72ba69f23956e1d55c7a5cc6585 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 8 Feb 2023 16:18:08 +0100 Subject: [PATCH 058/106] commented dry-run option, moved verbose option in help msg --- eessi_container.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index e0f4ead1af..96a9900a5e 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -86,13 +86,13 @@ display_help() { echo " when a directory is provided, the format of the" echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" echo " [default: not set]" + echo " -v | --verbose - display more information [default: false]" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." - echo - echo " FEATURES/OPTIONS to be implemented:" - echo " -d | --dry-run - run script except for executing the container," - echo " print information about setup [default: false]" - echo " -v | --verbose - display more information [default: false]" + #echo + #echo " FEATURES/OPTIONS to be implemented:" + #echo " -d | --dry-run - run script except for executing the container," + #echo " print information about setup [default: false]" #echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" #echo " [default: not set]" #echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" @@ -102,7 +102,7 @@ display_help() { # set defaults for command line arguments ACCESS="ro" CONTAINER="docker://ghcr.io/eessi/build-node:debian11" -DRY_RUN=0 +#DRY_RUN=0 VERBOSE=0 STORAGE= MODE="shell" @@ -124,10 +124,10 @@ while [[ $# -gt 0 ]]; do CONTAINER="$2" shift 2 ;; - -d|--dry-run) - DRY_RUN=1 - shift 1 - ;; +# -d|--dry-run) +# DRY_RUN=1 +# shift 1 +# ;; -g|--storage) STORAGE="$2" shift 2 From 7d3818d539111186f7d33dbf1b3a34813cd4e58b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Feb 2023 15:52:49 +0100 Subject: [PATCH 059/106] add checks for various use cases of eessi_container.sh script (+ no need to natively mount EESSI, since that's done in the container) --- .../workflows/test_eessi_container_script.yml | 86 +++++++++++++++++-- 1 file changed, 79 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index fc4cd01f48..362bd3893c 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -9,20 +9,92 @@ jobs: strategy: fail-fast: false matrix: - SCRIPT_ARGS: - - "--help" - - "--verbose" + SCRIPT_TEST: + - help + - run + - shell + - container + - readwrite + - resume + - save steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - name: Mount EESSI CernVM-FS pilot repository - uses: eessi/github-action-eessi@58b50fd2eead2162c2b9ac258d4fb60cc9f30503 # v2.0.13 - - name: install Apptainer run: | ./install_apptainer_ubuntu.sh - name: Test eessi_container.sh script run: | - ./eessi_container.sh ${{matrix.SCRIPT_ARGS}} + test_cmd="cat /etc/os-release" + out_pattern="Debian GNU/Linux 11" + + if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then + ./eessi_container.sh --help + + # test use of --mode run + elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then + outfile=out_run.txt + echo "${test_cmd}" > test_script.sh + chmod u+x test_script.sh + export SINGULARITY_BIND="$PWD:/test" + ./eessi_container.sh --mode run /test/test_script.sh | tee ${outfile} + grep "${out_pattern}" ${outfile} + + # test use of --mode shell + elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then + outfile=out_shell.txt + ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} + grep "${out_pattern}" ${outfile} + + # test use of --container option, using a totally different container; + # cfr. https://github.com/easybuilders/easybuild-containers + elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then + outfile=out_container.txt + container="docker://ghcr.io/eessi/build-node:debian10" + ./eessi_container.sh --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} + grep "Debian GNU/Linux 10" ${outfile} + + # test use of '--access rw' to get write access in container + elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then + outfile=out_readwrite.txt + fn="test_${RANDOM}.txt" + echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh + chmod u+x test_script.sh + export SINGULARITY_BIND="$PWD:/test" + ./eessi_container.sh --access rw --mode run /test/test_script.sh > ${outfile} + + tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") + # note: must use '--access rw' again here, since touched file is in overlay upper dir + ./eessi_container.sh --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} + grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile + + # test use of --resume + elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then + outfile=out_resume.txt + ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} + rm -f ${outfile} + + tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") + ./eessi_container.sh --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile} + grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile} + grep "${out_pattern}" ${outfile} + + # test use of --save (+ --resume) + elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then + outfile=out_save.txt + fn="test_${RANDOM}.txt" + test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}" + ./eessi_container.sh --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile} + rm -f ${outfile} + + ./eessi_container.sh --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} + grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile + + tar tfv test-save.tar | grep "overlay-upper/${fn}" + + else + echo "Unknown test case: ${{matrix.SCRIPT_TEST}}" >&2 + exit 1 + fi From 602aa68068b275b56b181af9bbcbf12f8395eedb Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 8 Feb 2023 16:28:52 +0100 Subject: [PATCH 060/106] fix check for './eessi_container.sh --resume' --- .github/workflows/test_eessi_container_script.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 362bd3893c..4d1c3bf898 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -73,11 +73,13 @@ jobs: # test use of --resume elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then outfile=out_resume.txt - ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} - rm -f ${outfile} + ./eessi_container.sh --mode shell <<< "${test_cmd}" > ${outfile} tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") + rm -f ${outfile} + ./eessi_container.sh --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile} + cat ${outfile} grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile} grep "${out_pattern}" ${outfile} From adcc655f99c65a609bc1cf21f1fe9d6fc89a8c7c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 8 Feb 2023 17:01:26 +0100 Subject: [PATCH 061/106] fix small typos in grep commands --- .github/workflows/test_eessi_container_script.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 362bd3893c..3974b42506 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -68,7 +68,7 @@ jobs: tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") # note: must use '--access rw' again here, since touched file is in overlay upper dir ./eessi_container.sh --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} - grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile + grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile # test use of --resume elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then @@ -90,7 +90,7 @@ jobs: rm -f ${outfile} ./eessi_container.sh --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} - grep "/cvmfs/pilot.eessi-hpc.org/${fn}$"> $outfile + grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile tar tfv test-save.tar | grep "overlay-upper/${fn}" From 7e470233684d456830f9cb7559992a7140845a57 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Feb 2023 09:11:46 +0100 Subject: [PATCH 062/106] enable --verbose + add step to collect test env info --- .../workflows/test_eessi_container_script.yml | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index c8c2aef47e..0920678c24 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -25,6 +25,11 @@ jobs: run: | ./install_apptainer_ubuntu.sh + - name: Collect info on test environment + run: | + mount + df -h + - name: Test eessi_container.sh script run: | test_cmd="cat /etc/os-release" @@ -39,13 +44,13 @@ jobs: echo "${test_cmd}" > test_script.sh chmod u+x test_script.sh export SINGULARITY_BIND="$PWD:/test" - ./eessi_container.sh --mode run /test/test_script.sh | tee ${outfile} + ./eessi_container.sh --verbose --mode run /test/test_script.sh | tee ${outfile} grep "${out_pattern}" ${outfile} # test use of --mode shell elif [[ ${{matrix.SCRIPT_TEST}} == 'shell' ]]; then outfile=out_shell.txt - ./eessi_container.sh --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} + ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} grep "${out_pattern}" ${outfile} # test use of --container option, using a totally different container; @@ -53,7 +58,7 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'container' ]]; then outfile=out_container.txt container="docker://ghcr.io/eessi/build-node:debian10" - ./eessi_container.sh --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} + ./eessi_container.sh --verbose --container ${container} --mode shell <<< "${test_cmd}" 2>&1 | tee ${outfile} grep "Debian GNU/Linux 10" ${outfile} # test use of '--access rw' to get write access in container @@ -63,22 +68,22 @@ jobs: echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh chmod u+x test_script.sh export SINGULARITY_BIND="$PWD:/test" - ./eessi_container.sh --access rw --mode run /test/test_script.sh > ${outfile} + ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile} tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") # note: must use '--access rw' again here, since touched file is in overlay upper dir - ./eessi_container.sh --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} + ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile # test use of --resume elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then outfile=out_resume.txt - ./eessi_container.sh --mode shell <<< "${test_cmd}" > ${outfile} + ./eessi_container.sh --verbose --mode shell <<< "${test_cmd}" > ${outfile} tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") rm -f ${outfile} - ./eessi_container.sh --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile} + ./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile} cat ${outfile} grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile} grep "${out_pattern}" ${outfile} @@ -88,10 +93,10 @@ jobs: outfile=out_save.txt fn="test_${RANDOM}.txt" test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}" - ./eessi_container.sh --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile} + ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile} rm -f ${outfile} - ./eessi_container.sh --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} + ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile} grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile tar tfv test-save.tar | grep "overlay-upper/${fn}" From a1bb7df72857be55e7e626de09634d1015cb45c0 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 10 Feb 2023 09:35:56 +0100 Subject: [PATCH 063/106] disable broken tests for eessi_container.sh script for now --- .github/workflows/test_eessi_container_script.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 0920678c24..7678aedc64 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -14,9 +14,10 @@ jobs: - run - shell - container - - readwrite - resume - - save + # FIXME disabled because '--access rw' is not working in CI environment + #- readwrite + #- save steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 From b1a36355de66fa9a9557edbef79508f88c763cbb Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 24 Jan 2023 21:59:41 +0100 Subject: [PATCH 064/106] added a function to obtain full path to a tool --- scripts/utils.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/scripts/utils.sh b/scripts/utils.sh index 5d8455bb68..cd7f758a0d 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -31,3 +31,35 @@ function check_exit_code { fatal_error "${fail_msg}" fi } + +function get_path_for_tool { + tool_name=$1 + tool_envvar_name=$2 + + which_out=$(which ${tool_name} 2>&1) + exit_code=$? + if [[ ${exit_code} -eq 0 ]]; then + echo "INFO: found tool ${tool_name} in PATH (${which_out})" >&2 + echo "${which_out}" + return 0 + fi + if [[ -z "${tool_envvar_name}" ]]; then + msg="no env var holding the full path to tool '${tool_name}' provided" + echo "${msg}" >&2 + return 1 + else + tool_envvar_value=${!tool_envvar_name} + if [[ -x "${tool_envvar_value}" ]]; then + msg="INFO: found tool ${tool_envvar_value} via env var ${tool_envvar_name}" + echo "${msg}" >&2 + echo "${tool_envvar_value}" + return 0 + else + msg="ERROR: tool '${tool_name}' not in PATH\n" + msg+="ERROR: tool '${tool_envvar_value}' via '${tool_envvar_name}' not in PATH" + echo "${msg}" >&2 + echo "" + return 2 + fi + fi +} From 27bc2035b00f59442fd46bcdbd6e104c09944f18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Wed, 25 Jan 2023 19:37:55 +0100 Subject: [PATCH 065/106] added function to determine IPv4 address --- scripts/utils.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/utils.sh b/scripts/utils.sh index cd7f758a0d..db533adc12 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -63,3 +63,12 @@ function get_path_for_tool { fi fi } + +function get_ipv4_address { + hname=$1 + hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) + # TODO try other methods if the one above does not work --> tool that verifies + # what method can be used? + echo "${hipv4}" + return 0 +} From bfe6d36bf056c9e4e9719fa3a8110899fb125dc2 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 18 Feb 2023 20:44:16 +0100 Subject: [PATCH 066/106] various updates for bot/build.sh and eessi_container.sh cherry-picked via commit b5bf008e4136b7877e06ca0602b6fd6aa49128c0 ONLY picked chages to eessi_container.sh --- eessi_container.sh | 103 ++++++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 48 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 96a9900a5e..3045fd5a89 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -43,8 +43,8 @@ MODE_UNKNOWN_EXITCODE=$((${ANY_ERROR_EXITCODE} << 5)) REPOSITORY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 6)) RESUME_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 7)) SAVE_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 8)) -#HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) -#HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) +HTTP_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 9)) +HTTPS_PROXY_ERROR_EXITCODE=$((${ANY_ERROR_EXITCODE} << 10)) RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) # CernVM-FS settings @@ -64,39 +64,39 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" display_help() { echo "usage: $0 [OPTIONS] [SCRIPT]" echo " OPTIONS:" - echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" - echo " -c | --container IMG - image file or URL defining the container to use" - echo " [default: docker://ghcr.io/eessi/build-node:debian11]" - echo " -h | --help - display this usage information [default: false]" - echo " -g | --storage DIR - directory space on host machine (used for" - echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" - echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" - echo " MODE==run (run a script) [default: shell]" - echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI-pilot via" + echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -c | --container IMG - image file or URL defining the container to use" + echo " [default: docker://ghcr.io/eessi/build-node:debian11]" + echo " -h | --help - display this usage information [default: false]" + echo " -g | --storage DIR - directory space on host machine (used for" + echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" + echo " MODE==run (run a script) [default: shell]" + echo " -r | --repository CFG - configuration file or identifier defining the" + echo " repository to use [default: EESSI-pilot via" echo " container configuration]" - echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," - echo " where DIR points to a previously used tmp directory" - echo " (check for output 'Using DIR as tmp ...' of a previous" - echo " run) and TGZ is the path to a tarball which is" - echo " unpacked the tmp dir stored on the local storage space" - echo " (see option --storage above) [default: not set]" - echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" - echo " directory DIR or provided with the fixed full path TGZ" - echo " when a directory is provided, the format of the" - echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" - echo " [default: not set]" - echo " -v | --verbose - display more information [default: false]" + echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," + echo " where DIR points to a previously used tmp directory" + echo " (check for output 'Using DIR as tmp ...' of a previous" + echo " run) and TGZ is the path to a tarball which is" + echo " unpacked the tmp dir stored on the local storage space" + echo " (see option --storage above) [default: not set]" + echo " -s | --save DIR/TGZ - save contents of tmp directory to a tarball in" + echo " directory DIR or provided with the fixed full path TGZ" + echo " when a directory is provided, the format of the" + echo " tarball's name will be {REPO_ID}-{TIMESTAMP}.tgz" + echo " [default: not set]" + echo " -v | --verbose - display more information [default: false]" + echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" + echo " [default: not set]; uses env var \$http_proxy if set" + echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" + echo " [default: not set]; uses env var \$https_proxy if set" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." - #echo - #echo " FEATURES/OPTIONS to be implemented:" - #echo " -d | --dry-run - run script except for executing the container," - #echo " print information about setup [default: false]" - #echo " -x | --http-proxy URL - provides URL for the env variable http_proxy" - #echo " [default: not set]" - #echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" - #echo " [default: not set]" + echo + echo " FEATURES/OPTIONS to be implemented:" + echo " -d | --dry-run - run script except for executing the container," + echo " print information about setup [default: false]" } # set defaults for command line arguments @@ -109,8 +109,8 @@ MODE="shell" REPOSITORY="EESSI-pilot" RESUME= SAVE= -#HTTP_PROXY= -#HTTPS_PROXY= +HTTP_PROXY=${http_proxy:-} +HTTPS_PROXY=${https_proxy:-} POSITIONAL_ARGS=() @@ -156,16 +156,16 @@ while [[ $# -gt 0 ]]; do VERBOSE=1 shift 1 ;; -# -x|--http-proxy) -# HTTP_PROXY="$2" -# export http_proxy=${HTTP_PROXY} -# shift 2 -# ;; -# -y|--https-proxy) -# HTTPS_PROXY="$2" -# export https_proxy=${HTTPS_PROXY} -# shift 2 -# ;; + -x|--http-proxy) + HTTP_PROXY="$2" + export http_proxy=${HTTP_PROXY} + shift 2 + ;; + -y|--https-proxy) + HTTPS_PROXY="$2" + export https_proxy=${HTTPS_PROXY} + shift 2 + ;; -*|--*) fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" ;; @@ -311,9 +311,9 @@ BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/ru BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" -# set up repository config (always create cfg dir and populate it with info when +# set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) -mkdir -p ${EESSI_TMPDIR}/cfg +mkdir -p ${EESSI_TMPDIR}/repos_cfg if [[ "${REPOSITORY}" == "EESSI-pilot" ]]; then # need to source defaults as late as possible (see other sourcing below) source ${TOPDIR}/init/eessi_defaults @@ -370,13 +370,13 @@ else # only unpack config_bundle if we're not resuming from a previous run if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle} -C ${EESSI_TMPDIR}/cfg + tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg fi for src in "${!cfg_file_map[@]}" do target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/cfg/${src}:${target}" + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" done export EESSI_PILOT_VERSION_OVERRIDE=${repo_version} export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" @@ -432,6 +432,13 @@ else fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_BIND=${SINGULARITY_BIND}" +# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) +if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then + export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} + # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility) + export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +fi + echo "Launching container with command (next line):" echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level From 837234107d28cd9b2d759a66af8f7e2ec5244923 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 18 Feb 2023 20:55:30 +0100 Subject: [PATCH 067/106] provide CVMFS_HTTP_PROXY if necessary cherry-picked via commit b5c07eeb99e77191d818e64788b0d419031cf526 NOTE, only applied changes to eessi_container.sh --- eessi_container.sh | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 3045fd5a89..9238d3c793 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -24,8 +24,8 @@ # 2. set up host storage/tmp # 3. set up common vars and directories # 4. set up vars specific to a scenario -# 5. initialize host storage/tmp from previous run if provided -# 6. run container +# 5. run container +# 6. save tmp (if requested) # -. initial settings & exit codes TOPDIR=$(dirname $(realpath $0)) @@ -279,7 +279,7 @@ fi # |-overlay-upper # |-overlay-work # |-home -# |-cfg +# |-repos_cfg # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} @@ -361,9 +361,9 @@ else cfg_init_file_map "${config_map}" [[ ${VERBOSE} -eq 1 ]] && cfg_print_map - # TODO use information to set up dir ${EESSI_TMPDIR}/cfg, - # define BIND mounts and override repo name and version - # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/cfg + # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, + # define BIND mounts and override repo name and version + # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg if [[ ! -r ${config_bundle} ]]; then fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} fi @@ -384,6 +384,26 @@ else source ${TOPDIR}/init/eessi_defaults fi +# if http_proxy is not empty, we assume that the machine accesses internet +# via a proxy. then we need to add CVMFS_HTTP_PROXY to +# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND +# MOUNT if it was not yet in BIND_PATHS) +if [[ ! -z ${http_proxy} ]]; then + # TODO tolerate other formats for proxy URLs, for now assume format is + # http://SOME_HOSTNAME:SOME_PORT + PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http:// + PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port + HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/} + HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) + echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \ + "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ + >> ${EESSI_TMPDIR}/repos_cfg/default.local + cat ${EESSI_TMPDIR}/repos_cfg/default.local + # if default.local is not BIND mounted into container, add it to BIND_PATHS + if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then + export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" + fi +fi # 4. set up vars and dirs specific to a scenario @@ -420,10 +440,7 @@ if [[ "${ACCESS}" == "rw" ]]; then fi -# 5. initialize host storage/tmp from previous run if provided - - -# 6. run container +# 5. run container # final settings if [[ -z ${SINGULARITY_BIND} ]]; then export SINGULARITY_BIND="${BIND_PATHS}" @@ -446,7 +463,7 @@ echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" exit_code=$? -# 7. save tmp if requested (arg -s|--save) +# 6. save tmp if requested (arg -s|--save) if [[ ! -z ${SAVE} ]]; then # Note, for now we don't try to be smart and record in any way the OS and # ARCH which might have been used internally, eg, when software packages From 3f1aed9a2b5da7fa7e017e415d24a412dbfa18fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Fri, 27 Jan 2023 22:41:36 +0100 Subject: [PATCH 068/106] add functions to derive hostname and port number from proxy URL --- scripts/utils.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/scripts/utils.sh b/scripts/utils.sh index db533adc12..fb4eff1e3f 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -64,6 +64,30 @@ function get_path_for_tool { fi } +function get_host_from_url { + url=$1 + re="(http|https)://([^/:]+)" + if [[ $url =~ $re ]]; then + echo ${BASH_REMATCH[2]} + return 0 + else + echo "" + return 1 + fi +} + +function get_port_from_url { + url=$1 + re="(http|https)://[^:]+:([0-9]+)" + if [[ $url =~ $re ]]; then + echo ${BASH_REMATCH[2]} + return 0 + else + echo "" + return 1 + fi +} + function get_ipv4_address { hname=$1 hipv4=$(grep ${hname} /etc/hosts | grep -v '^[[:space:]]*#' | cut -d ' ' -f 1) From b7abfc6e7de85e2b46e3241d8946a1211dcde568 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Fri, 27 Jan 2023 22:43:27 +0100 Subject: [PATCH 069/106] add function to determine config sections --- cfg_files.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/cfg_files.sh b/cfg_files.sh index 885ebd0877..ffd29db50a 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -47,9 +47,7 @@ function cfg_load { else val=$(cfg_get_key_value $line) # trim leading and trailing spaces as well - #cur_key=$(echo $val | cut -f1 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') cur_key=$(echo $val | cut -f1 -d'=' | cfg_trim_spaces) - #cur_val=$(echo $val | cut -f2 -d'=' | sed -e 's/^[[:space:]]*//' | sed -e 's/[[:space:]]*$//') cur_val=$(echo $val | cut -f2 -d'=' | cfg_trim_spaces) if [[ -n "$cur_key" ]]; then # section + key is the associative in bash array, the field separator is space @@ -69,6 +67,20 @@ function cfg_print { done } +function cfg_sections { + declare -A sections + for key in "${!cfg_repos[@]}" + do + # extract section from the associative key + section=$(echo $key | cut -f1 -d ' ') + sections[${section}]=1 + done + for repo in "${!sections[@]}" + do + echo "${repo}" + done +} + function cfg_get_value { section=$1 key=$2 From 9b7b8a4db00261b0fe2bd40152d7adacde11ffe3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 18 Feb 2023 21:17:11 +0100 Subject: [PATCH 070/106] add parameter to list repositories, fix bugs fixed bugs - determining IP address for proxy host was wrong - handling of positional parameters for executing commands with arguments was wrong --- eessi_container.sh | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 9238d3c793..06b77a5acf 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -70,6 +70,7 @@ display_help() { echo " -h | --help - display this usage information [default: false]" echo " -g | --storage DIR - directory space on host machine (used for" echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" + echo " -l | --list-repos - list available repository identifiers [default: false]" echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" echo " MODE==run (run a script) [default: shell]" echo " -r | --repository CFG - configuration file or identifier defining the" @@ -105,6 +106,7 @@ CONTAINER="docker://ghcr.io/eessi/build-node:debian11" #DRY_RUN=0 VERBOSE=0 STORAGE= +LIST_REPOS=0 MODE="shell" REPOSITORY="EESSI-pilot" RESUME= @@ -136,6 +138,10 @@ while [[ $# -gt 0 ]]; do display_help exit 0 ;; + -l|--list-repos) + LIST_REPOS=1 + shift 1 + ;; -m|--mode) MODE="$2" shift 2 @@ -178,6 +184,17 @@ done set -- "${POSITIONAL_ARGS[@]}" +if [[ ${LIST_REPOS} -eq 1 ]]; then + echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':" + echo " EESSI-pilot [default]" + cfg_load ${EESSI_REPOS_CFG_FILE} + sections=$(cfg_sections) + while IFS= read -r repo_id + do + echo " ${repo_id}" + done <<< "${sections}" + exit 0 +fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported @@ -390,17 +407,20 @@ fi # MOUNT if it was not yet in BIND_PATHS) if [[ ! -z ${http_proxy} ]]; then # TODO tolerate other formats for proxy URLs, for now assume format is - # http://SOME_HOSTNAME:SOME_PORT - PROXY_HOST_AND_PORT=${http_proxy#http:\/\//} # strip http:// - PROXY_PORT=${PROXY_HOST_AND_PORT#.*:/} # remove hostname: to get port - HTTP_PROXY_HOSTNAME=${PROXY_HOST_AND_PORT%:${PROX_PORT}/} - HTTP_PROXY_IPV4=$(get_ipv4_address ${HTTP_PROXY_HOSTNAME}) - echo "CVMFS_HTTP_PROXY=\"${http_proxy}|" \ - "http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ + # http://SOME_HOSTNAME:SOME_PORT/ + [[ ${VERBOSE} -eq 1 ]] && echo "http_proxy='${http_proxy}'" + PROXY_HOST=$(get_host_from_url ${http_proxy}) + [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'" + PROXY_PORT=$(get_port_from_url ${http_proxy}) + [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'" + HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) + [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'" + echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ >> ${EESSI_TMPDIR}/repos_cfg/default.local cat ${EESSI_TMPDIR}/repos_cfg/default.local + # if default.local is not BIND mounted into container, add it to BIND_PATHS - if [[ ${BIND_PATHS} !~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then + if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" fi fi From 4a414f593d6ff02629d1374e67006dcd1a85f71c Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 18 Feb 2023 21:26:19 +0100 Subject: [PATCH 071/106] replace FILE with DIR in EESSI_REPOS_CFG_FILE_OVERRIDE NOTE only applied changes to eessi_container.sh --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 06b77a5acf..d59bb38687 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -52,8 +52,8 @@ CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" # repository cfg file, default name (default location: $PWD) -# can be overwritten by setting env var EESSI_REPOS_CFG_FILE_OVERRIDE -export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_FILE_OVERRIDE:=repos.cfg}" +# can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg" # 0. parse args From 37abf4f8aab9d138ad763cd457d2476bef16a6aa Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 29 Jan 2023 19:56:41 +0100 Subject: [PATCH 072/106] fix whitespace issues --- cfg_files.sh | 1 - eessi_container.sh | 2 +- init/eessi_defaults | 1 - scripts/utils.sh | 4 ++-- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cfg_files.sh b/cfg_files.sh index ffd29db50a..57ea2f7c03 100644 --- a/cfg_files.sh +++ b/cfg_files.sh @@ -165,4 +165,3 @@ function cfg_print_map { echo "${index} --> ${cfg_file_map[${index}]}" done } - diff --git a/eessi_container.sh b/eessi_container.sh index d59bb38687..1a2e61227b 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -413,7 +413,7 @@ if [[ ! -z ${http_proxy} ]]; then [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_HOST='${PROXY_HOST}'" PROXY_PORT=$(get_port_from_url ${http_proxy}) [[ ${VERBOSE} -eq 1 ]] && echo "PROXY_PORT='${PROXY_PORT}'" - HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) + HTTP_PROXY_IPV4=$(get_ipv4_address ${PROXY_HOST}) [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'" echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ >> ${EESSI_TMPDIR}/repos_cfg/default.local diff --git a/init/eessi_defaults b/init/eessi_defaults index 1b5ce07fb1..f482cbc269 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -10,4 +10,3 @@ export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" - diff --git a/scripts/utils.sh b/scripts/utils.sh index fb4eff1e3f..d0da95e87f 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -73,7 +73,7 @@ function get_host_from_url { else echo "" return 1 - fi + fi } function get_port_from_url { @@ -85,7 +85,7 @@ function get_port_from_url { else echo "" return 1 - fi + fi } function get_ipv4_address { From d468a1ababa8072c4069917f4c16529cd362011f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 18 Feb 2023 21:42:34 +0100 Subject: [PATCH 073/106] include changes made to test PR for building with the bot cherry-picked via commit fce504fb466be005f1d417e96e47f442c8f29c52 NOTE, only changed eessi_container.sh --- eessi_container.sh | 50 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 1a2e61227b..14fb716abb 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -51,9 +51,15 @@ RUN_SCRIPT_MISSING_EXITCODE=$((${ANY_ERROR_EXITCODE} << 11)) CVMFS_VAR_LIB="var-lib-cvmfs" CVMFS_VAR_RUN="var-run-cvmfs" +# directory for tmp used inside container +export TMP_IN_CONTAINER=/tmp + # repository cfg file, default name (default location: $PWD) # can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE -export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=.}/repos.cfg" +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg" +# other repository cfg files in directory, default location: $PWD +# can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE +export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}" # 0. parse args @@ -199,7 +205,7 @@ fi # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then - fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" + fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi # TODO (arg -c|--container) check container (is it a file or URL & access those) @@ -211,11 +217,14 @@ fi # (arg -m|--mode) check if MODE is known if [[ "${MODE}" != "shell" && "${MODE}" != "run" ]]; then - fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" + fatal_error "unknown execution mode '${MODE}'" "${MODE_UNKNOWN_EXITCODE}" fi # TODO (arg -r|--repository) check if repository is known # REPOSITORY_ERROR_EXITCODE +if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI-pilot" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then + fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" +fi # TODO (arg -u|--resume) check if it exists, if user has read permission, # if it contains data from a previous run @@ -249,6 +258,7 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then # skip creating a new tmp directory, just set environment variables echo "Resuming from previous run using temporary storage at ${RESUME}" EESSI_HOST_STORAGE=${RESUME} + echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}" else # we need a tmp location (and possibly init it with ${RESUME} if it was not # a directory @@ -325,7 +335,9 @@ fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" -BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:/tmp" +# provide a '/tmp' inside the container +BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" + [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" # set up repository config (always create directory repos_cfg and populate it with info when @@ -381,13 +393,22 @@ else # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, # define BIND mounts and override repo name and version # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg - if [[ ! -r ${config_bundle} ]]; then - fatal_error "config bundle '${config_bundle}' is not readable" ${REPOSITORY_ERROR_EXITCODE} + # if config_bundle is relative path (no '/' at start) prepend it with + # EESSI_REPOS_CFG_DIR + config_bundle_path= + if [[ ! "${config_bundle}" =~ ^/ ]]; then + config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle} + else + config_bundle_path=${config_bundle} + fi + + if [[ ! -r ${config_bundle_path} ]]; then + fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} fi # only unpack config_bundle if we're not resuming from a previous run if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle} -C ${EESSI_TMPDIR}/repos_cfg + tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg fi for src in "${!cfg_file_map[@]}" @@ -436,12 +457,8 @@ if [[ "${ACCESS}" == "ro" ]]; then fi if [[ "${ACCESS}" == "rw" ]]; then - EESSI_CVMFS_OVERLAY_UPPER=/tmp/overlay-upper - EESSI_CVMFS_OVERLAY_WORK=/tmp/overlay-work mkdir -p ${EESSI_TMPDIR}/overlay-upper mkdir -p ${EESSI_TMPDIR}/overlay-work - [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_UPPER=${EESSI_CVMFS_OVERLAY_UPPER}" - [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_CVMFS_OVERLAY_WORK=${EESSI_CVMFS_OVERLAY_WORK}" # set environment variables for fuse mounts in Singularity container export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" @@ -450,8 +467,8 @@ if [[ "${ACCESS}" == "rw" ]]; then EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs" EESSI_PILOT_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=/tmp/overlay-upper" - EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=/tmp/overlay-work" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" + EESSI_PILOT_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" EESSI_PILOT_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" export EESSI_PILOT_WRITABLE_OVERLAY @@ -476,11 +493,13 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi +# if INFO is set to 1 (arg --info), add argument '-q' +RUN_QUIET=${INFO:--q} echo "Launching container with command (next line):" -echo "singularity ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" +echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level # provided to the script -singularity -q ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" +singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" exit_code=$? # 6. save tmp if requested (arg -s|--save) @@ -500,6 +519,7 @@ if [[ ! -z ${SAVE} ]]; then fi tar cf ${TGZ} -C ${EESSI_TMPDIR} . echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')" + echo "RESUME_FROM_TGZ ${TGZ}" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) From af658ee9229f36eac972bef87a36bbb6b883ef48 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Fri, 3 Feb 2023 20:58:01 +0100 Subject: [PATCH 074/106] fix issues in eessi_container.sh --- eessi_container.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 14fb716abb..23118d0d72 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -258,7 +258,6 @@ if [[ ! -z ${RESUME} && -d ${RESUME} ]]; then # skip creating a new tmp directory, just set environment variables echo "Resuming from previous run using temporary storage at ${RESUME}" EESSI_HOST_STORAGE=${RESUME} - echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}" else # we need a tmp location (and possibly init it with ${RESUME} if it was not # a directory @@ -290,6 +289,7 @@ else EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." fi +echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}" # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then @@ -494,7 +494,12 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then fi # if INFO is set to 1 (arg --info), add argument '-q' -RUN_QUIET=${INFO:--q} +if [[ -z ${INFO} ]]; then + RUN_QUIET='-q' +else + RUN_QUIET='' +fi + echo "Launching container with command (next line):" echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" # TODO for now we run singularity with '-q' (quiet), later adjust this to the log level From eb66e1c4361f458114d2186a846fa9fe56b1df16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20R=C3=B6blitz?= Date: Sat, 4 Feb 2023 15:07:23 +0100 Subject: [PATCH 075/106] fix RUN_QUIET setting --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 23118d0d72..89d4bd98f6 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -493,8 +493,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi -# if INFO is set to 1 (arg --info), add argument '-q' -if [[ -z ${INFO} ]]; then +# if INFO is set to 0 (no arg --info), add argument '-q' +if [[ ${INFO} -eq 0 ]]; then RUN_QUIET='-q' else RUN_QUIET='' From 1cb4674978bee259253330d769ca220ad795202a Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 18 Feb 2023 21:59:39 +0100 Subject: [PATCH 076/106] improvements to resuming job environment cherry-picked via commit 23e773ce0d8c6504e1440629a4178318265b73d2 NOTE only applied changes to eessi_container.sh --- eessi_container.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 89d4bd98f6..e714f8e2d0 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -360,6 +360,9 @@ else # standard EESSI repositories) cfg_load ${EESSI_REPOS_CFG_FILE} + # copy repos.cfg to job directory --> makes it easier to inspect the job + cp ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. + # cfg file should include: repo_name, repo_version, config_bundle, # map { local_filepath -> container_filepath } # @@ -493,8 +496,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi -# if INFO is set to 0 (no arg --info), add argument '-q' -if [[ ${INFO} -eq 0 ]]; then +# if INFO is set to 1 (arg --info), add argument '-q' +if [[ -z ${INFO} ]]; then RUN_QUIET='-q' else RUN_QUIET='' From 0c465da3de12ed1bb7494651031d64d7f17928e4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 7 Feb 2023 19:27:07 +0100 Subject: [PATCH 077/106] address requested change --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index e714f8e2d0..e8698cd719 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -496,8 +496,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi -# if INFO is set to 1 (arg --info), add argument '-q' -if [[ -z ${INFO} ]]; then +# if INFO is set to 0 (no arg --info), add argument '-q' +if [[ ${INFO} -eq 0 ]]; then RUN_QUIET='-q' else RUN_QUIET='' From 26ab98c1f1cc901d6905424c9fdb2f5c9395ed79 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 18 Feb 2023 22:16:36 +0100 Subject: [PATCH 078/106] cleaning up leftovers after adding updates from NESSI --- eessi_container.sh | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index e8698cd719..e7ad625a15 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -100,10 +100,6 @@ display_help() { echo " [default: not set]; uses env var \$https_proxy if set" echo echo " If value for --mode is 'run', the SCRIPT provided is executed." - echo - echo " FEATURES/OPTIONS to be implemented:" - echo " -d | --dry-run - run script except for executing the container," - echo " print information about setup [default: false]" } # set defaults for command line arguments @@ -496,8 +492,8 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi -# if INFO is set to 0 (no arg --info), add argument '-q' -if [[ ${INFO} -eq 0 ]]; then +# if VERBOSE is set to 0 (no arg --verbose), add argument '-q' +if [[ ${VERBOSE} -eq 0 ]]; then RUN_QUIET='-q' else RUN_QUIET='' @@ -505,8 +501,6 @@ fi echo "Launching container with command (next line):" echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" -# TODO for now we run singularity with '-q' (quiet), later adjust this to the log level -# provided to the script singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" exit_code=$? From a44088e4c33694c0047a6e5874df780e3c2ff15e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 18 Feb 2023 22:21:42 +0100 Subject: [PATCH 079/106] moving cfg_files.sh to scripts dir --- eessi_container.sh | 2 +- cfg_files.sh => scripts/cfg_files.sh | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename cfg_files.sh => scripts/cfg_files.sh (100%) diff --git a/eessi_container.sh b/eessi_container.sh index e7ad625a15..6d3fbc0adc 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -31,7 +31,7 @@ TOPDIR=$(dirname $(realpath $0)) source ${TOPDIR}/scripts/utils.sh -source ${TOPDIR}/cfg_files.sh +source ${TOPDIR}/scripts/cfg_files.sh # exit codes: bitwise shift codes to allow for combination of exit codes # ANY_ERROR_EXITCODE is sourced from ${TOPDIR}/scripts/utils.sh diff --git a/cfg_files.sh b/scripts/cfg_files.sh similarity index 100% rename from cfg_files.sh rename to scripts/cfg_files.sh From bbbad0dce469299e4168a54a939facc4c1be7c60 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 20 Feb 2023 10:07:03 +0100 Subject: [PATCH 080/106] improved handling of container cache + explicit pull of image cherry-picked via commit bfb1b29103dc309a12573f3ec9c247d432f08f73 Note, only part for eessi_container.sh included here --- eessi_container.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 6d3fbc0adc..cdd2c017fb 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -310,10 +310,23 @@ mkdir -p ${EESSI_TMPDIR} [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" # configure Singularity -export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache -mkdir -p ${SINGULARITY_CACHEDIR} +if [[ -z ${SINGULARITY_CACHEDIR} ]]; then + export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache + mkdir -p ${SINGULARITY_CACHEDIR} +fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" +# pull & convert image and reset CONTAINER +CONTAINER_URL_FMT=".*://(.*)" +if [[ ${CONTAINER} == ${CONTAINER_URL_FMT} ]]; then + CONTAINER_IMG=${BASH_REMATCH[1]//[:-]/_}.sif + singularity pull ${CONTAINER_IMG} ${CONTAINER} + if [[ -x ${CONTAINER_IMG} ]]; then + CONTAINER="${PWD}/${CONTAINER_IMG}" + fi +fi +[[ ${INFO} -eq 1 ]] && echo "CONTAINER='${CONTAINER}'" + # set env vars and create directories for CernVM-FS EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} EESSI_CVMFS_VAR_RUN=${EESSI_TMPDIR}/${CVMFS_VAR_RUN} @@ -326,8 +339,8 @@ mkdir -p ${EESSI_CVMFS_VAR_RUN} if [[ -z ${SINGULARITY_HOME} ]]; then export SINGULARITY_HOME="${EESSI_TMPDIR}/home:/home/${USER}" mkdir -p ${EESSI_TMPDIR}/home - [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" fi +[[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_HOME=${SINGULARITY_HOME}" # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs" From 15ae58df240b1b1396611ee6fb5b5dbd50d4e793 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 20 Feb 2023 10:30:17 +0100 Subject: [PATCH 081/106] use VERBOSE instead of INFO --- .../workflows/test_eessi_container_script.yml | 22 +++++++++++++++++++ eessi_container.sh | 2 +- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 7678aedc64..5a44bae155 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -11,6 +11,8 @@ jobs: matrix: SCRIPT_TEST: - help + - listrepos_default + - listrepos_custom - run - shell - container @@ -39,6 +41,26 @@ jobs: if [[ ${{matrix.SCRIPT_TEST}} == 'help' ]]; then ./eessi_container.sh --help + # test use of --list-repos without custom repos.cfg + elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then + outfile=out_listrepos.txt + ./eessi_container.sh --verbose --list-repos | tee ${outfile} + grep "EESSI-pilot" ${outfile} + + # test use of --list-repos with custom repos.cfg + elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then + outfile=out_listrepos.txt + outfile2=out_listrepos_2.txt + mkdir -p ${PWD}/cfg + echo "[EESSI/2021.12]" > cfg/repos.cfg + echo "[EESSI/2023.02]" >> cfg/repos.cfg + ./eessi_container.sh --verbose --list-repos | tee ${outfile} + grep "EESSI-pilot" ${outfile} + + export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg + ./eessi_container.sh --verbose --list-repos | tee ${outfile2} + grep "[EESSI/2023.02]" ${outfile2} + # test use of --mode run elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then outfile=out_run.txt diff --git a/eessi_container.sh b/eessi_container.sh index cdd2c017fb..76a4d34143 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -325,7 +325,7 @@ if [[ ${CONTAINER} == ${CONTAINER_URL_FMT} ]]; then CONTAINER="${PWD}/${CONTAINER_IMG}" fi fi -[[ ${INFO} -eq 1 ]] && echo "CONTAINER='${CONTAINER}'" +[[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}" # set env vars and create directories for CernVM-FS EESSI_CVMFS_VAR_LIB=${EESSI_TMPDIR}/${CVMFS_VAR_LIB} From 764e7138b0f8dfd82c81c1647bf78d04da1aec68 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Mon, 20 Feb 2023 10:48:38 +0100 Subject: [PATCH 082/106] fix test for --list-repos; improve output for --list-repos --- .../workflows/test_eessi_container_script.yml | 2 ++ eessi_container.sh | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 5a44bae155..74cfd1fb10 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -53,7 +53,9 @@ jobs: outfile2=out_listrepos_2.txt mkdir -p ${PWD}/cfg echo "[EESSI/2021.12]" > cfg/repos.cfg + echo "repo_version = 2021.12" >> cfg/repos.cfg echo "[EESSI/2023.02]" >> cfg/repos.cfg + echo "repo_version = 2023.02" >> cfg/repos.cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile} grep "EESSI-pilot" ${outfile} diff --git a/eessi_container.sh b/eessi_container.sh index 76a4d34143..dec082ce04 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -186,15 +186,18 @@ done set -- "${POSITIONAL_ARGS[@]}" + if [[ ${LIST_REPOS} -eq 1 ]]; then - echo "Repositories defined in the config file '${EESSI_REPOS_CFG_FILE}':" + echo "Listing available repositories with format 'name [source]':" echo " EESSI-pilot [default]" - cfg_load ${EESSI_REPOS_CFG_FILE} - sections=$(cfg_sections) - while IFS= read -r repo_id - do - echo " ${repo_id}" - done <<< "${sections}" + if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then + cfg_load ${EESSI_REPOS_CFG_FILE} + sections=$(cfg_sections) + while IFS= read -r repo_id + do + echo " ${repo_id} [${EESSI_REPOS_CFG_FILE}]" + done <<< "${sections}" + fi exit 0 fi From 47cc1c6be44896675949567d88f5ae85a998b64f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Tue, 21 Feb 2023 21:27:56 +0100 Subject: [PATCH 083/106] implemented requested changes --- .../workflows/test_eessi_container_script.yml | 8 +-- eessi_container.sh | 59 ++++++++++++++----- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index 74cfd1fb10..d4a9dfd53e 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -52,10 +52,10 @@ jobs: outfile=out_listrepos.txt outfile2=out_listrepos_2.txt mkdir -p ${PWD}/cfg - echo "[EESSI/2021.12]" > cfg/repos.cfg - echo "repo_version = 2021.12" >> cfg/repos.cfg - echo "[EESSI/2023.02]" >> cfg/repos.cfg - echo "repo_version = 2023.02" >> cfg/repos.cfg + echo "[EESSI/20AB.CD]" > cfg/repos.cfg + echo "repo_version = 20AB.CD" >> cfg/repos.cfg + echo "[EESSI/20HT.TP]" >> cfg/repos.cfg + echo "repo_version = 20HT.TP" >> cfg/repos.cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile} grep "EESSI-pilot" ${outfile} diff --git a/eessi_container.sh b/eessi_container.sh index dec082ce04..86a510d56e 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -54,12 +54,11 @@ CVMFS_VAR_RUN="var-run-cvmfs" # directory for tmp used inside container export TMP_IN_CONTAINER=/tmp -# repository cfg file, default name (default location: $PWD) -# can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE -export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}/repos.cfg" -# other repository cfg files in directory, default location: $PWD -# can be overwritten by setting env var EESSI_REPOS_CFG_DIR_OVERRIDE +# repository cfg directory and file +# directory: default $PWD or EESSI_REPOS_CFG_DIR_OVERRIDE if set +# file: directory + '/repos.cfg' export EESSI_REPOS_CFG_DIR="${EESSI_REPOS_CFG_DIR_OVERRIDE:=${PWD}}" +export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg" # 0. parse args @@ -288,7 +287,6 @@ else EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." fi -echo "RESUME_FROM_DIR ${EESSI_HOST_STORAGE}" # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} if [[ ! -z ${RESUME} && -f ${RESUME} ]]; then @@ -312,21 +310,52 @@ EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" -# configure Singularity +# configure Singularity: if SINGULARITY_CACHEDIR is already defined, use that +# a global SINGULARITY_CACHEDIR would ensure that we don't consume +# storage space again and again for the container & also speed-up +# launch times across different sessions if [[ -z ${SINGULARITY_CACHEDIR} ]]; then export SINGULARITY_CACHEDIR=${EESSI_TMPDIR}/singularity_cache mkdir -p ${SINGULARITY_CACHEDIR} fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" -# pull & convert image and reset CONTAINER +# we try our best to make sure that we retain access to the container image in +# a subsequent session ("best effort" only because pulling or copying operations +# can fail ... in those cases the script may still succeed, but it is not +# guaranteed that we have access to the same container when resuming later on) +# - if CONTAINER references an image in a registry, pull & convert image +# and store it in ${EESSI_TMPDIR} +# + however, only pull image if there is no matching image in ${EESSI_TMPDIR} yet +# - if CONTAINER references an image file, copy it to ${EESSI_TMPDIR} +# + however, only copy it if its base name does not yet exist in ${EESSI_TMPDIR} +# - if the image file created (pulled or copied) or resumed exists in +# ${EESSI_TMPDIR}, let CONTAINER point to it +# + thus subsequent singularity commands in this script would just use the +# image file in EESSI_TMPDIR or the originally given source (some URL or +# path to an image file) +CONTAINER_IMG= CONTAINER_URL_FMT=".*://(.*)" -if [[ ${CONTAINER} == ${CONTAINER_URL_FMT} ]]; then +if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then + # replace : and - with _ in match (everything after ://) and append .sif CONTAINER_IMG=${BASH_REMATCH[1]//[:-]/_}.sif - singularity pull ${CONTAINER_IMG} ${CONTAINER} - if [[ -x ${CONTAINER_IMG} ]]; then - CONTAINER="${PWD}/${CONTAINER_IMG}" + # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when + # resuming from a previous session) + if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then + singularity pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER} fi +else + # determine file name as basename of CONTAINER + CONTAINER_IMG=$(basename ${CONTAINER}) + # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when + # resuming from a previous session) + if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then + cp -a ${CONTAINER} ${EESSI_TMPDIR}/. + fi +fi +# let CONTAINER point to the pulled, copied or resumed image file +if [[ -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then + CONTAINER="${EESSI_TMPDIR}/${CONTAINER_IMG}" fi [[ ${VERBOSE} -eq 1 ]] && echo "CONTAINER=${CONTAINER}" @@ -373,7 +402,7 @@ else cfg_load ${EESSI_REPOS_CFG_FILE} # copy repos.cfg to job directory --> makes it easier to inspect the job - cp ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. + cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. # cfg file should include: repo_name, repo_version, config_bundle, # map { local_filepath -> container_filepath } @@ -453,7 +482,8 @@ if [[ ! -z ${http_proxy} ]]; then [[ ${VERBOSE} -eq 1 ]] && echo "HTTP_PROXY_IPV4='${HTTP_PROXY_IPV4}'" echo "CVMFS_HTTP_PROXY=\"${http_proxy}|http://${HTTP_PROXY_IPV4}:${PROXY_PORT}\"" \ >> ${EESSI_TMPDIR}/repos_cfg/default.local - cat ${EESSI_TMPDIR}/repos_cfg/default.local + [[ ${VERBOSE} -eq 1 ]] && echo "contents of default.local" + [[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local # if default.local is not BIND mounted into container, add it to BIND_PATHS if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then @@ -537,7 +567,6 @@ if [[ ! -z ${SAVE} ]]; then fi tar cf ${TGZ} -C ${EESSI_TMPDIR} . echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')" - echo "RESUME_FROM_TGZ ${TGZ}" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) From 712c40e567842da1c50b8c196fad0c9c5d30a69a Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 22 Feb 2023 22:54:43 +0100 Subject: [PATCH 084/106] improvements to resuming job environment cherry-picked via 23e773ce0d8c6504e1440629a4178318265b73d2 Note only applied part for bot/build.sh --- bot/build.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 8565511bfa..2c15b904f3 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -141,7 +141,7 @@ REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi -mkdir -p previous_tmp +mkdir -p previous_tmp/{build_step,tarball_step} build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" echo "./eessi_container.sh --access rw" @@ -151,7 +151,7 @@ echo " ${HTTPS_PROXY_OPT}" echo " --info" echo " --mode run" echo " ${REPOSITORY_OPT}" -echo " --save ${PWD}/previous_tmp" +echo " --save ${PWD}/previous_tmp/build_step" echo " --storage ${STORAGE}" echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg @@ -163,7 +163,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg --info \ --mode run \ ${REPOSITORY_OPT} \ - --save ${PWD}/previous_tmp \ + --save ${PWD}/previous_tmp/build_step \ --storage ${STORAGE} \ ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr} @@ -190,7 +190,7 @@ echo " --info" echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --resume ${BUILD_TMPDIR}" -echo " --save ${PWD}/previous_tmp" +echo " --save ${PWD}/previous_tmp/tarball_step" echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" ./eessi_container.sh --access rw \ ${CONTAINER_OPT} \ @@ -200,7 +200,7 @@ echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT --mode run \ ${REPOSITORY_OPT} \ --resume ${BUILD_TMPDIR} \ - --save ${PWD}/previous_tmp \ + --save ${PWD}/previous_tmp/tarball_step \ ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 From 16d98572d4148049a3a9f6a4258467eaf785d9f4 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 22 Feb 2023 23:00:42 +0100 Subject: [PATCH 085/106] improved handling of container cache + explicit pull of image cherry-picked via bfb1b29103dc309a12573f3ec9c247d432f08f73 Note, only applied part for bot/build.sh --- bot/build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 2c15b904f3..06503e59a0 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -56,6 +56,12 @@ LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) echo "LOCAL_TMP='${LOCAL_TMP}'" # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided +SINGULARITY_CACHEDIR=$(${YQ} '.site_config.container_cachedir // ""' ${JOB_CFG_FILE}) +echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" +if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then + export SINGULARITY_CACHEDIR +fi + echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " # replace any env variable in ${LOCAL_TMP} with its # current value (e.g., a value that is local to the job) From 9e8ca62cab90b013c6fb1b55180e37592aea02b5 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 22 Feb 2023 23:02:35 +0100 Subject: [PATCH 086/106] changed --info -> --verbose --- bot/build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 06503e59a0..77aa2d9077 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -154,7 +154,7 @@ echo "./eessi_container.sh --access rw" echo " ${CONTAINER_OPT}" echo " ${HTTP_PROXY_OPT}" echo " ${HTTPS_PROXY_OPT}" -echo " --info" +echo " --verbose" echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --save ${PWD}/previous_tmp/build_step" @@ -166,7 +166,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ ${HTTPS_PROXY_OPT} \ - --info \ + --verbose \ --mode run \ ${REPOSITORY_OPT} \ --save ${PWD}/previous_tmp/build_step \ @@ -192,7 +192,7 @@ echo "./eessi_container.sh --access rw" echo " ${CONTAINER_OPT}" echo " ${HTTP_PROXY_OPT}" echo " ${HTTPS_PROXY_OPT}" -echo " --info" +echo " --verbose" echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --resume ${BUILD_TMPDIR}" @@ -202,7 +202,7 @@ echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT ${CONTAINER_OPT} \ ${HTTP_PROXY_OPT} \ ${HTTPS_PROXY_OPT} \ - --info \ + --verbose \ --mode run \ ${REPOSITORY_OPT} \ --resume ${BUILD_TMPDIR} \ From c0350a9da92f6632c9ba52a57a5a28e03cb9c040 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 22 Feb 2023 23:14:38 +0100 Subject: [PATCH 087/106] improved messages for resume info and adjusted parsing in bot/build.sh --- bot/build.sh | 2 +- eessi_container.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 77aa2d9077..f9d8c9cede 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -174,7 +174,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr} # determine temporary directory to resume from -BUILD_TMPDIR=$(grep 'RESUME_FROM_DIR' ${build_outerr} | sed -e "s/^RESUME_FROM_DIR //") +BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) tar_outerr=$(mktemp tar.outerr.XXXX) timestamp=$(date +%s) diff --git a/eessi_container.sh b/eessi_container.sh index f1c755938c..060b003f33 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -284,7 +284,7 @@ else [[ ${VERBOSE} -eq 1 ]] && echo "skipping sanity checks for /tmp" fi EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) - echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." + echo "Using ${EESSI_HOST_STORAGE} as tmp directory (to resume session add '--resume ${EESSI_HOST_STORAGE}')." fi # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} @@ -565,7 +565,7 @@ if [[ ! -z ${SAVE} ]]; then TGZ=${SAVE} fi tar cf ${TGZ} -C ${EESSI_TMPDIR} . - echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')" + echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TGZ}' (to resume session add '--resume ${TGZ}')" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) From ef9a552aa1b0a96a7b13c4319b501e0a79459766 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 22 Feb 2023 23:25:27 +0100 Subject: [PATCH 088/106] delete one of two tmp storage tarballs created by bot/build.sh --- bot/build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index f9d8c9cede..5926f005d5 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -209,4 +209,10 @@ echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT --save ${PWD}/previous_tmp/tarball_step \ ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} +# if two tarballs have been generated, only keep the one from tarball step +NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l) +if [[ ${NUM_TARBALLS} -eq 2 ]]; then + rm -f previous_tmp/build_step/*.tgz +fi + exit 0 From 6974e4b80e3a59643d4eac0f229f50f8f547d430 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 Feb 2023 13:33:54 +0100 Subject: [PATCH 089/106] add verbose messages on pulling/copying/reusing of container image --- eessi_container.sh | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 86a510d56e..01c69bdd92 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -320,6 +320,13 @@ if [[ -z ${SINGULARITY_CACHEDIR} ]]; then fi [[ ${VERBOSE} -eq 1 ]] && echo "SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}" +# if VERBOSE is set to 0 (no arg --verbose), add argument '-q' +if [[ ${VERBOSE} -eq 0 ]]; then + RUN_QUIET='-q' +else + RUN_QUIET='' +fi + # we try our best to make sure that we retain access to the container image in # a subsequent session ("best effort" only because pulling or copying operations # can fail ... in those cases the script may still succeed, but it is not @@ -342,7 +349,10 @@ if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when # resuming from a previous session) if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then - singularity pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER} + [[ ${VERBOSE} -eq 1 ]] && echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}" + singularity ${RUN_QUIET} pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER} + else + [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}" fi else # determine file name as basename of CONTAINER @@ -350,7 +360,10 @@ else # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when # resuming from a previous session) if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then + [[ ${VERBOSE} -eq 1 ]] && echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}" cp -a ${CONTAINER} ${EESSI_TMPDIR}/. + else + [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}" fi fi # let CONTAINER point to the pulled, copied or resumed image file @@ -538,13 +551,6 @@ if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi -# if VERBOSE is set to 0 (no arg --verbose), add argument '-q' -if [[ ${VERBOSE} -eq 0 ]]; then - RUN_QUIET='-q' -else - RUN_QUIET='' -fi - echo "Launching container with command (next line):" echo "singularity ${RUN_QUIET} ${MODE} ${EESSI_FUSE_MOUNTS[@]} ${CONTAINER} $@" singularity ${RUN_QUIET} ${MODE} "${EESSI_FUSE_MOUNTS[@]}" ${CONTAINER} "$@" From 54bdea52e86d853a923d4884c4b62682440c5b09 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 Feb 2023 13:37:47 +0100 Subject: [PATCH 090/106] don't require --verbose for info message on pulling/copying/reusing container image --- eessi_container.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 01c69bdd92..e98cc53493 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -349,10 +349,10 @@ if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when # resuming from a previous session) if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then - [[ ${VERBOSE} -eq 1 ]] && echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}" + echo "Pulling container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}" singularity ${RUN_QUIET} pull ${EESSI_TMPDIR}/${CONTAINER_IMG} ${CONTAINER} else - [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}" + echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}" fi else # determine file name as basename of CONTAINER @@ -360,10 +360,10 @@ else # copy image file to ${EESSI_TMPDIR} if it is not there yet (i.e. when # resuming from a previous session) if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then - [[ ${VERBOSE} -eq 1 ]] && echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}" + echo "Copying container image from ${CONTAINER} to ${EESSI_TMPDIR}/${CONTAINER_IMG}" cp -a ${CONTAINER} ${EESSI_TMPDIR}/. else - [[ ${VERBOSE} -eq 1 ]] && echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}" + echo "Reusing existing container image ${EESSI_TMPDIR}/${CONTAINER_IMG}" fi fi # let CONTAINER point to the pulled, copied or resumed image file From a4cea9afb12242340360c9ad922e13c8492c9d57 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 Feb 2023 13:53:06 +0100 Subject: [PATCH 091/106] add check to make sure that container image exists in tmpdir --- .github/workflows/test_eessi_container_script.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index d4a9dfd53e..929fb22cec 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -108,6 +108,9 @@ jobs: tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g") rm -f ${outfile} + # make sure that container image exists + test -f ${tmpdir}/ghcr.io_eessi_build_node_debian11.sif || (echo "Container image not found in ${tmpdir}" >&2 && ls ${tmpdir} && exit 1) + ./eessi_container.sh --verbose --resume ${tmpdir} --mode shell <<< "${test_cmd}" > ${outfile} cat ${outfile} grep "Resuming from previous run using temporary storage at ${tmpdir}" ${outfile} From c9758326512d824ae11e790f836c4db36dc3c527 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 24 Feb 2023 15:46:52 +0100 Subject: [PATCH 092/106] fix determining filename from container URL --- eessi_container.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index e98cc53493..1d76360735 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -344,8 +344,8 @@ fi CONTAINER_IMG= CONTAINER_URL_FMT=".*://(.*)" if [[ ${CONTAINER} =~ ${CONTAINER_URL_FMT} ]]; then - # replace : and - with _ in match (everything after ://) and append .sif - CONTAINER_IMG=${BASH_REMATCH[1]//[:-]/_}.sif + # replace ':', '-', '/' with '_' in match (everything after ://) and append .sif + CONTAINER_IMG="$(echo ${BASH_REMATCH[1]} | sed 's/[:\/-]/_/g').sif" # pull container to ${EESSI_TMPDIR} if it is not there yet (i.e. when # resuming from a previous session) if [[ ! -x ${EESSI_TMPDIR}/${CONTAINER_IMG} ]]; then From 7cc65750c50bae0a2ea3f0cb9d96137fb21aa452 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 25 Feb 2023 02:55:37 +0100 Subject: [PATCH 093/106] support cmd line arg flag terminator --- eessi_container.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eessi_container.sh b/eessi_container.sh index b1f8922d61..b9553a7ea1 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -173,6 +173,11 @@ while [[ $# -gt 0 ]]; do export https_proxy=${HTTPS_PROXY} shift 2 ;; + --) + shift + POSITIONAL_ARGS+=("$@") # save positional args + break + ;; -*|--*) fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" ;; From 0def4e91ce9d52281e0e8fc3d30c96f095c75bd5 Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 24 Feb 2023 22:07:02 +0100 Subject: [PATCH 094/106] just print contents of cfg/job.cfg --- bot/build.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 5926f005d5..906f1bfd7c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -23,6 +23,10 @@ # defaults export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" +echo "bot/build.sh: Showing job.cfg from software-layer side" +cat cfg/job.cfg +exit 0 + # source utils.sh source utils.sh From ef608d15f20bea94bae116b7ece71a21586c7e3c Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 24 Feb 2023 23:55:44 +0100 Subject: [PATCH 095/106] switching from json (YQ) to ini (cfg_files.sh) --- bot/build.sh | 43 ++++++++++++++++--------------------------- 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 906f1bfd7c..c621b9524f 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -17,50 +17,39 @@ # - the working directory contains a directory 'cfg' where the main config # file 'job.cfg' has been deposited # - the directory may contain any additional files references in job.cfg -# - the tool 'yq' for working with json files is available via the PATH or -# the environment variable BOT_YQ (see https://github.com/mikefarah/yq) # defaults export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" echo "bot/build.sh: Showing job.cfg from software-layer side" cat cfg/job.cfg -exit 0 - -# source utils.sh -source utils.sh -# check setup / define key variables -# get path for 'yq' (if not found, an empty string is returned) -YQ=$(get_path_for_tool "yq" "BOT_YQ") -exit_code=$? -if [[ ${exit_code} -ne 0 ]]; then - fatal_error "could not find path to 'yq'; exiting" -else - echo_green "found yq (${YQ})" -fi +# source utils.sh and cfg_files.sh +source scripts/utils.sh +source scripts/cfg_files.sh # check if './cfg/job.cfg' exists if [[ ! -r "${JOB_CFG_FILE}" ]]; then fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" fi echo "obtaining configuration settings from '${JOB_CFG_FILE}'" +cfg_load ${JOB_CFG_FILE} # if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy -HTTP_PROXY=$(${YQ} '.site_config.http_proxy // ""' ${JOB_CFG_FILE}) +HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy") HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} echo "HTTP_PROXY='${HTTP_PROXY}'" # if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy -HTTPS_PROXY=$(${YQ} '.site_config.https_proxy // ""' ${JOB_CFG_FILE}) +HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy") HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} echo "HTTPS_PROXY='${HTTPS_PROXY}'" -LOCAL_TMP=$(${YQ} '.site_config.local_tmp // ""' ${JOB_CFG_FILE}) +LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") echo "LOCAL_TMP='${LOCAL_TMP}'" # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided -SINGULARITY_CACHEDIR=$(${YQ} '.site_config.container_cachedir // ""' ${JOB_CFG_FILE}) +SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then export SINGULARITY_CACHEDIR @@ -73,11 +62,11 @@ STORAGE=$(envsubst <<< ${LOCAL_TMP}) echo "'${STORAGE}'" # obtain list of modules to be loaded -LOAD_MODULES=$(${YQ} '.site_config.load_modules // ""' ${JOB_CFG_FILE}) +LOAD_MODULES=$(cfg_get_value "site_config" "load_modules") echo "LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND -CONTAINER=$(${YQ} '.repository.container // ""' ${JOB_CFG_FILE}) +CONTAINER=$(cfg_get_value "repository" "container") export SINGULARITY_HOME="$(pwd):/eessi_bot_job" export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} @@ -94,20 +83,20 @@ else fi # determine repository to be used from entry .repository in cfg/job.cfg -REPOSITORY=$(${YQ} '.repository.repo_id // ""' ${JOB_CFG_FILE}) -EESSI_REPOS_CFG_DIR_OVERRIDE=$(${YQ} '.repository.repos_cfg_dir // ""' ${JOB_CFG_FILE}) +REPOSITORY=$(cfg_get_value "repository" "repo_id") +EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} # determine pilot version to be used from .repository.repo_version in cfg/job.cfg # here, just set & export EESSI_PILOT_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) -export EESSI_PILOT_VERSION_OVERRIDE=$(${YQ} '.repository.repo_version // ""' ${JOB_CFG_FILE}) +export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") # determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG_FILE}) +export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") # determine architecture to be used from entry .architecture in cfg/job.cfg @@ -115,12 +104,12 @@ export EESSI_CVMFS_REPO_OVERRIDE=$(${YQ} '.repository.repo_name // ""' ${JOB_CFG if [[ ! -z "${CPU_TARGET}" ]]; then EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET} else - EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(${YQ} '.architecture.software_subdir // ""' ${JOB_CFG_FILE}) + EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir") fi export EESSI_SOFTWARE_SUBDIR_OVERRIDE # get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux) -EESSI_OS_TYPE=$(${YQ} '.architecture.os_type // ""' ${JOB_CFG_FILE}) +EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} # TODO From 429d5a03ad0da75e8e8c52fb8897b341d1c70511 Mon Sep 17 00:00:00 2001 From: trz42 Date: Thu, 23 Feb 2023 23:56:28 +0100 Subject: [PATCH 096/106] add --generic arg when running install script --- bot/build.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index c621b9524f..387a9c95c9 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -140,6 +140,11 @@ REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi +GENERIC_OPT= +if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ ".*/generic" ]]; then + GENERIC_OPT="--generic" +fi + mkdir -p previous_tmp/{build_step,tarball_step} build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" @@ -152,7 +157,7 @@ echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --save ${PWD}/previous_tmp/build_step" echo " --storage ${STORAGE}" -echo " ./install_software_layer.sh \"$@\" 2>&1 | tee -a ${build_outerr}" +echo " ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --access rw \ @@ -164,7 +169,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ${REPOSITORY_OPT} \ --save ${PWD}/previous_tmp/build_step \ --storage ${STORAGE} \ - ./install_software_layer.sh "$@" 2>&1 | tee -a ${build_outerr} + ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} # determine temporary directory to resume from BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) From bd31faa6b6e4e3945447118b6121667d01b4af7d Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 24 Feb 2023 00:17:57 +0100 Subject: [PATCH 097/106] fix error in regex --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index 387a9c95c9..860f888a46 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -141,7 +141,7 @@ if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" fi GENERIC_OPT= -if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ ".*/generic" ]]; then +if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then GENERIC_OPT="--generic" fi From 1bafb7889fa8f49b183e048d63a8fbdabee041f1 Mon Sep 17 00:00:00 2001 From: trz42 Date: Fri, 24 Feb 2023 00:43:23 +0100 Subject: [PATCH 098/106] use command separator -- --- bot/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 860f888a46..cdd59d6081 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -157,7 +157,7 @@ echo " --mode run" echo " ${REPOSITORY_OPT}" echo " --save ${PWD}/previous_tmp/build_step" echo " --storage ${STORAGE}" -echo " ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" +echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --access rw \ @@ -169,7 +169,7 @@ export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ${REPOSITORY_OPT} \ --save ${PWD}/previous_tmp/build_step \ --storage ${STORAGE} \ - ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} + -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} # determine temporary directory to resume from BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) From c9041e34604ec881e6a5aeb96fbf31da91bf63a1 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 25 Feb 2023 00:29:25 +0100 Subject: [PATCH 099/106] try fixing arg issue --- bot/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/build.sh b/bot/build.sh index cdd59d6081..f56f456dcf 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -161,7 +161,7 @@ echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --access rw \ - ${CONTAINER_OPT} \ + "${CONTAINER_OPT}" \ ${HTTP_PROXY_OPT} \ ${HTTPS_PROXY_OPT} \ --verbose \ From 9d46a22892a4462f1b87c52c679d90cfed6256bb Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 25 Feb 2023 00:41:36 +0100 Subject: [PATCH 100/106] put build args into an array --- bot/build.sh | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index f56f456dcf..8720c137a5 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -124,21 +124,30 @@ export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} # files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg') # prepare options and directories for calling eessi_container.sh +declare -a BUILD_STEP_ARGS=() +BUILD_STEP_ARGS+=("--access" "rw") +BUILD_STEP_ARGS+=("--mode" "run") +BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step") +BUILD_STEP_ARGS+=("--storage" "${STORAGE}") CONTAINER_OPT= if [[ ! -z ${CONTAINER} ]]; then CONTAINER_OPT="--container ${CONTAINER}" + BUILD_STEP_ARGS+=("--container" "${CONTAINER}") fi HTTP_PROXY_OPT= if [[ ! -z ${HTTP_PROXY} ]]; then HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}" + BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}") fi HTTPS_PROXY_OPT= if [[ ! -z ${HTTPS_PROXY} ]]; then HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}" + BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}") fi REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" + BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}") fi GENERIC_OPT= if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then @@ -148,27 +157,13 @@ fi mkdir -p previous_tmp/{build_step,tarball_step} build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" -echo "./eessi_container.sh --access rw" -echo " ${CONTAINER_OPT}" -echo " ${HTTP_PROXY_OPT}" -echo " ${HTTPS_PROXY_OPT}" +echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}" echo " --verbose" -echo " --mode run" -echo " ${REPOSITORY_OPT}" -echo " --save ${PWD}/previous_tmp/build_step" -echo " --storage ${STORAGE}" echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg -./eessi_container.sh --access rw \ - "${CONTAINER_OPT}" \ - ${HTTP_PROXY_OPT} \ - ${HTTPS_PROXY_OPT} \ +./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \ --verbose \ - --mode run \ - ${REPOSITORY_OPT} \ - --save ${PWD}/previous_tmp/build_step \ - --storage ${STORAGE} \ -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} # determine temporary directory to resume from From ff5045f0940730951c25cfb5260148655fbf933d Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 25 Feb 2023 01:19:25 +0100 Subject: [PATCH 101/106] use array to define args for build and tarball step --- bot/build.sh | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 8720c137a5..bb1f4319a1 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -125,29 +125,39 @@ export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} # prepare options and directories for calling eessi_container.sh declare -a BUILD_STEP_ARGS=() +BUILD_STEP_ARGS+=("--verbose") BUILD_STEP_ARGS+=("--access" "rw") BUILD_STEP_ARGS+=("--mode" "run") BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step") BUILD_STEP_ARGS+=("--storage" "${STORAGE}") +declare -a TARBALL_STEP_ARGS=() +TARBALL_STEP_ARGS+=("--verbose") +TARBALL_STEP_ARGS+=("--access" "rw") +TARBALL_STEP_ARGS+=("--mode" "run") +TARBALL_STEP_ARGS+=("--save" "${PWD}/previous_tmp/tarball_step") CONTAINER_OPT= if [[ ! -z ${CONTAINER} ]]; then CONTAINER_OPT="--container ${CONTAINER}" BUILD_STEP_ARGS+=("--container" "${CONTAINER}") + TARBALL_STEP_ARGS+=("--container" "${CONTAINER}") fi HTTP_PROXY_OPT= if [[ ! -z ${HTTP_PROXY} ]]; then HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}" BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}") + TARBALL_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}") fi HTTPS_PROXY_OPT= if [[ ! -z ${HTTPS_PROXY} ]]; then HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}" BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}") + TARBALL_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}") fi REPOSITORY_OPT= if [[ ! -z ${REPOSITORY} ]]; then REPOSITORY_OPT="--repository ${REPOSITORY}" BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}") + TARBALL_STEP_ARGS+=("--repository" "${REPOSITORY}") fi GENERIC_OPT= if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then @@ -158,16 +168,15 @@ mkdir -p previous_tmp/{build_step,tarball_step} build_outerr=$(mktemp build.outerr.XXXX) echo "Executing command to build software:" echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}" -echo " --verbose" echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" # set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \ - --verbose \ -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} # determine temporary directory to resume from BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) +TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") tar_outerr=$(mktemp tar.outerr.XXXX) timestamp=$(date +%s) @@ -181,26 +190,10 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $ # /tmp as default? TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" -echo "./eessi_container.sh --access rw" -echo " ${CONTAINER_OPT}" -echo " ${HTTP_PROXY_OPT}" -echo " ${HTTPS_PROXY_OPT}" -echo " --verbose" -echo " --mode run" -echo " ${REPOSITORY_OPT}" -echo " --resume ${BUILD_TMPDIR}" -echo " --save ${PWD}/previous_tmp/tarball_step" -echo " ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" -./eessi_container.sh --access rw \ - ${CONTAINER_OPT} \ - ${HTTP_PROXY_OPT} \ - ${HTTPS_PROXY_OPT} \ - --verbose \ - --mode run \ - ${REPOSITORY_OPT} \ - --resume ${BUILD_TMPDIR} \ - --save ${PWD}/previous_tmp/tarball_step \ - ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} +echo "./eessi_container.sh ${TARBALL_STEP_ARGS[@]}" +echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +./eessi_container.sh "${TARBALL_STEP_ARGS[@]}" \ + -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} # if two tarballs have been generated, only keep the one from tarball step NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l) From d08bd211ed6c23c5b917b01bd01ef75a5cb7b1b9 Mon Sep 17 00:00:00 2001 From: trz42 Date: Tue, 7 Mar 2023 14:43:52 +0100 Subject: [PATCH 102/106] addressed comments and changes requested by first review --- bot/build.sh | 172 ++++++++++++++++++++++++--------------------------- 1 file changed, 82 insertions(+), 90 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index bb1f4319a1..004e2d4881 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -16,41 +16,44 @@ # pull request (OR by some other means) # - the working directory contains a directory 'cfg' where the main config # file 'job.cfg' has been deposited -# - the directory may contain any additional files references in job.cfg +# - the directory may contain any additional files referenced in job.cfg -# defaults -export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" - -echo "bot/build.sh: Showing job.cfg from software-layer side" -cat cfg/job.cfg +# stop as soon as something fails +set -e # source utils.sh and cfg_files.sh source scripts/utils.sh source scripts/cfg_files.sh -# check if './cfg/job.cfg' exists +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" + +# check if ${JOB_CFG_FILE} exists if [[ ! -r "${JOB_CFG_FILE}" ]]; then fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" fi -echo "obtaining configuration settings from '${JOB_CFG_FILE}'" +echo "bot/build.sh: showing ${JOB_CFG_FILE} from software-layer side" +cat ${JOB_CFG_FILE} + +echo "bot/build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" cfg_load ${JOB_CFG_FILE} -# if http_proxy is in cfg/job.cfg use it, if not use env var $http_proxy +# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy") HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} -echo "HTTP_PROXY='${HTTP_PROXY}'" +echo "bot/build.sh: HTTP_PROXY='${HTTP_PROXY}'" -# if https_proxy is in cfg/job.cfg use it, if not use env var $https_proxy +# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy") HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} -echo "HTTPS_PROXY='${HTTPS_PROXY}'" +echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'" LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") -echo "LOCAL_TMP='${LOCAL_TMP}'" +echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" # TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") -echo "SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" +echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then export SINGULARITY_CACHEDIR fi @@ -61,14 +64,21 @@ echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " STORAGE=$(envsubst <<< ${LOCAL_TMP}) echo "'${STORAGE}'" +# make sure ${STORAGE} exists +mkdir -p ${STORAGE} + +# make sure the base tmp storage is unique +JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX) +echo "bot/build.sh: created unique base tmp storage directory at ${JOB_STORAGE}" + # obtain list of modules to be loaded LOAD_MODULES=$(cfg_get_value "site_config" "load_modules") -echo "LOAD_MODULES='${LOAD_MODULES}'" +echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND CONTAINER=$(cfg_get_value "repository" "container") -export SINGULARITY_HOME="$(pwd):/eessi_bot_job" -export SINGULARITY_TMPDIR="$(pwd)/singularity_tmpdir" +export SINGULARITY_HOME="${pwd}:/eessi_bot_job" +export SINGULARITY_TMPDIR="${pwd}/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} # load modules if LOAD_MODULES is not empty @@ -82,103 +92,91 @@ else echo "bot/build.sh: no modules to be loaded" fi -# determine repository to be used from entry .repository in cfg/job.cfg +# determine repository to be used from entry .repository in ${JOB_CFG_FILE} REPOSITORY=$(cfg_get_value "repository" "repo_id") EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} +echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" -# determine pilot version to be used from .repository.repo_version in cfg/job.cfg +# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} # here, just set & export EESSI_PILOT_VERSION_OVERRIDE # next script (eessi_container.sh) makes use of it via sourcing init scripts # (e.g., init/eessi_defaults or init/minimal_eessi_env) export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" -# determine CVMFS repo to be used from .repository.repo_name in cfg/job.cfg +# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") - - -# determine architecture to be used from entry .architecture in cfg/job.cfg -# default: leave empty to let downstream script(s) determine subdir to be used -if [[ ! -z "${CPU_TARGET}" ]]; then - EESSI_SOFTWARE_SUBDIR_OVERRIDE=${CPU_TARGET} -else - EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir") -fi +echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" + +# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} +# fallbacks: +# - ${CPU_TARGET} handed over from bot +# - left empty to let downstream script(s) determine subdir to be used +EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir") +EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} export EESSI_SOFTWARE_SUBDIR_OVERRIDE +echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" -# get EESSI_OS_TYPE from .architecture.os_type in cfg/job.cfg (default: linux) +# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux) EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} - -# TODO -# - CODED add handling of EESSI_SOFTWARE_SUBDIR_OVERRIDE to eessi_container.sh -# TODO ensure that the bot makes use of that. (currently sets env var -# CPU_TARGET & adds --export=ALL,CPU_TARGET=val to sbatch command ... also -# add it to cfg/job.cfg - .architecture.software_subdir) -# - CODED add handling of http(s)_proxy to eessi_container.sh, in there needs the -# CVMFS_HTTP_PROXY added to /etc/cvmfs/default.local (this needs a robust -# way to determine the IP address of a proxy) -# - bot needs to make repos.cfg and cfg_bundle available to job (likely, by copying -# files into './cfg/.' and defining '.repository.repos_cfg_dir' in './cfg/job.cfg') - -# prepare options and directories for calling eessi_container.sh +echo "bot/build.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'" + +# prepare arguments to eessi_container.sh common to build and tarball steps +declare -a COMMON_ARGS=() +COMMON_ARGS+=("--verbose") +COMMON_ARGS+=("--access" "rw") +COMMON_ARGS+=("--mode" "run") +[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}") +[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}") +[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") +[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}") + +# make sure to use the same parent dir for storing tarballs of tmp +PREVIOUS_TMP_DIR=${PWD}/previous_tmp + +# prepare directory to store tarball of tmp for build step +TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step +mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} + +# prepare arguments to eessi_container.sh specific to build step declare -a BUILD_STEP_ARGS=() -BUILD_STEP_ARGS+=("--verbose") -BUILD_STEP_ARGS+=("--access" "rw") -BUILD_STEP_ARGS+=("--mode" "run") -BUILD_STEP_ARGS+=("--save" "${PWD}/previous_tmp/build_step") +BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") BUILD_STEP_ARGS+=("--storage" "${STORAGE}") -declare -a TARBALL_STEP_ARGS=() -TARBALL_STEP_ARGS+=("--verbose") -TARBALL_STEP_ARGS+=("--access" "rw") -TARBALL_STEP_ARGS+=("--mode" "run") -TARBALL_STEP_ARGS+=("--save" "${PWD}/previous_tmp/tarball_step") -CONTAINER_OPT= -if [[ ! -z ${CONTAINER} ]]; then - CONTAINER_OPT="--container ${CONTAINER}" - BUILD_STEP_ARGS+=("--container" "${CONTAINER}") - TARBALL_STEP_ARGS+=("--container" "${CONTAINER}") -fi -HTTP_PROXY_OPT= -if [[ ! -z ${HTTP_PROXY} ]]; then - HTTP_PROXY_OPT="--http-proxy ${HTTP_PROXY}" - BUILD_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}") - TARBALL_STEP_ARGS+=("--http-proxy" "${HTTP_PROXY}") -fi -HTTPS_PROXY_OPT= -if [[ ! -z ${HTTPS_PROXY} ]]; then - HTTPS_PROXY_OPT="--https-proxy ${HTTPS_PROXY}" - BUILD_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}") - TARBALL_STEP_ARGS+=("--https-proxy" "${HTTPS_PROXY}") -fi -REPOSITORY_OPT= -if [[ ! -z ${REPOSITORY} ]]; then - REPOSITORY_OPT="--repository ${REPOSITORY}" - BUILD_STEP_ARGS+=("--repository" "${REPOSITORY}") - TARBALL_STEP_ARGS+=("--repository" "${REPOSITORY}") -fi + +# prepare arguments to install_software_layer.sh (specific to build step) GENERIC_OPT= if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then GENERIC_OPT="--generic" fi -mkdir -p previous_tmp/{build_step,tarball_step} +# create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) + echo "Executing command to build software:" -echo "./eessi_container.sh ${BUILD_STEP_ARGS[@]}" +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" -# set EESSI_REPOS_CFG_DIR_OVERRIDE to ./cfg -export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg -./eessi_container.sh "${BUILD_STEP_ARGS[@]}" \ +./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} +# prepare directory to store tarball of tmp for tarball step +TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step +mkdir -p ${TARBALL_TMP_TARBALL_STEP_DIR} + +# create tmp file for output of tarball step +tar_outerr=$(mktemp tar.outerr.XXXX) + +# prepare arguments to eessi_container.sh specific to tarball step +declare -a TARBALL_STEP_ARGS=() +TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}") + # determine temporary directory to resume from BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") -tar_outerr=$(mktemp tar.outerr.XXXX) timestamp=$(date +%s) # to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now source init/eessi_defaults @@ -190,15 +188,9 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $ # /tmp as default? TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" -echo "./eessi_container.sh ${TARBALL_STEP_ARGS[@]}" +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" -./eessi_container.sh "${TARBALL_STEP_ARGS[@]}" \ +./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} -# if two tarballs have been generated, only keep the one from tarball step -NUM_TARBALLS=$(find ${PWD}/previous_tmp -type f -name "*tgz" | wc -l) -if [[ ${NUM_TARBALLS} -eq 2 ]]; then - rm -f previous_tmp/build_step/*.tgz -fi - exit 0 From 54d2a21f04f7e10ee8577c674be7b2d411da5026 Mon Sep 17 00:00:00 2001 From: trz42 Date: Wed, 8 Mar 2023 13:31:25 +0100 Subject: [PATCH 103/106] improved usage information --- eessi_container.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index b9553a7ea1..48c4653ba9 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -67,7 +67,7 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg" # https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash display_help() { - echo "usage: $0 [OPTIONS] [SCRIPT]" + echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]" echo " OPTIONS:" echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" echo " -c | --container IMG - image file or URL defining the container to use" @@ -77,7 +77,7 @@ display_help() { echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" echo " -l | --list-repos - list available repository identifiers [default: false]" echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" - echo " MODE==run (run a script) [default: shell]" + echo " MODE==run (run a script or command) [default: shell]" echo " -r | --repository CFG - configuration file or identifier defining the" echo " repository to use [default: EESSI-pilot via" echo " default container, see --container]" @@ -98,7 +98,9 @@ display_help() { echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" echo " [default: not set]; uses env var \$https_proxy if set" echo - echo " If value for --mode is 'run', the SCRIPT provided is executed." + echo " If value for --mode is 'run', the SCRIPT/COMMAND provided is executed. If" + echo " arguments to the script/command start with '-' or '--', use the flag terminator" + echo " '--' to let eessi_container.sh stop parsing arguments." } # set defaults for command line arguments From 64d1866638288e315de91a41ad37a40fdaf53736 Mon Sep 17 00:00:00 2001 From: trz42 Date: Sat, 11 Mar 2023 01:33:50 +0100 Subject: [PATCH 104/106] fix bug, env var PWD is uppercase --- bot/build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bot/build.sh b/bot/build.sh index 004e2d4881..20334501ed 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -77,8 +77,8 @@ echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'" # singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND CONTAINER=$(cfg_get_value "repository" "container") -export SINGULARITY_HOME="${pwd}:/eessi_bot_job" -export SINGULARITY_TMPDIR="${pwd}/singularity_tmpdir" +export SINGULARITY_HOME="${PWD}:/eessi_bot_job" +export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir" mkdir -p ${SINGULARITY_TMPDIR} # load modules if LOAD_MODULES is not empty From ff78c62df541075ecfb58c37e8a14bc4a0e0d80f Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 15 Mar 2023 07:04:02 +0100 Subject: [PATCH 105/106] make sure that CPU arch specific directories are used as container cache dir --- bot/build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bot/build.sh b/bot/build.sh index 20334501ed..c8def2cdd3 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -27,6 +27,7 @@ source scripts/cfg_files.sh # defaults export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" +HOST_ARCH=$(uname -m) # check if ${JOB_CFG_FILE} exists if [[ ! -r "${JOB_CFG_FILE}" ]]; then @@ -55,6 +56,8 @@ echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then + # make sure that separate directories are used for different CPU families + SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH} export SINGULARITY_CACHEDIR fi From 6299c9b4a84a5ced2afde69e9c668c4de9752331 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Fri, 17 Mar 2023 19:36:00 +0100 Subject: [PATCH 106/106] add configuration file for production build-and-deploy bot in AWS CitC cluster --- bot/bot-eessi-aws-citc.cfg | 151 +++++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 bot/bot-eessi-aws-citc.cfg diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg new file mode 100644 index 0000000000..5b3ad34612 --- /dev/null +++ b/bot/bot-eessi-aws-citc.cfg @@ -0,0 +1,151 @@ +# Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5 + +[github] +# replace '123456' with the ID of your GitHub App +app_id = 281041 + +# a short (!) name for your app instance that can be used for example +# when adding/updating a comment to a PR +# (!) a short yet descriptive name is preferred because it appears in +# comments to the PR +# for example, the name could include the name of the cluster the bot +# runs on and the username which runs the bot +# NOTE avoid putting an actual username here as it will be visible on +# potentially publicly accessible GitHub pages. +app_name = eessi-bot-citc-aws + +# replace '12345678' with the ID of the installation of your GitHub App +# (can be derived by creating an event and then checking for the list +# of sent events and its payload either via the Smee channel's web page +# or via the Advanced section of your GitHub App on github.com) +installation_id = 33078935 + +# path to the private key that was generated when the GitHub App was registered +private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws-private-key.pem + + +[buildenv] +# name of the job script used for building an EESSI stack +build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm + +# The container_cachedir may be used to reuse downloaded container image files +# across jobs. Thus, jobs can more quickly launch containers. +container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir + +# it may happen that we need to customize some CVMFS configuration +# the value of cvmfs_customizations is a dictionary which maps a file +# name to an entry that needs to be added to that file +cvmfs_customizations = {} + +# if compute nodes have no internet connection, we need to set http(s)_proxy +# or commands such as pip3 cannot download software from package repositories +# for example, the temporary EasyBuild is installed via pip3 first +# http_proxy = http://PROXY_DNS:3128/ +# https_proxy = http://PROXY_DNS:3128/ + +# directory under which the bot prepares directories per job +# structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR +jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs + +# configure environment +# list of comma-separated modules to be loaded by build_job_script +# useful/needed if some tool is not provided as system-wide package +# (read by bot and handed over to build_job_script via parameter +# --load-modules) +load_modules = + +# PATH to temporary directory on build node ... ends up being used for +# for example, EESSI_TMPDIR --> /tmp/$USER/EESSI +# escaping variables with '\' delays expansion to the start of the +# build_job_script; this can be used for referencing environment +# variables that are only set inside a Slurm job +local_tmp = /tmp/$USER/EESSI + +# parameters to be added to all job submissions +# NOTE do not quote parameter string. Quotes are retained when reading in config and +# then the whole 'string' is recognised as a single parameter. +# NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs +# to be initialised as if it is for a login shell. +# note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed +slurm_params = --hold --time=24:0:0 + +# full path to the job submission command +submit_command = /usr/bin/sbatch + +# which GH account has the permission to trigger the build (by setting +# the label 'bot:build' (apparently this cannot be restricted on GitHub) +# if value is left/empty everyone can trigger the build +# value can be a space delimited list of GH accounts +build_permission = boegel trz42 bedroge + +[architecturetargets] +# defines both for which architectures the bot will build +# and what submission parameters shall be used +# medium instances (8 cores, 16GB RAM) +#arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} +# larger instances (16 cores, 32GB RAM) +arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} + +[repo_targets] +# defines for which repository a arch_target should be build for +# +# only building for repository EESSI-pilot +repo_target_map = { "linux/x86_64/generic" : ["EESSI-pilot"], "linux/x86_64/intel/haswell" : ["EESSI-pilot"], "linux/x86_64/intel/skylake_avx512" : ["EESSI-pilot"], "linux/x86_64/amd/zen2": ["EESSI-pilot"], "linux/x86_64/amd/zen3" : ["EESSI-pilot"], "linux/aarch64/generic" : ["EESSI-pilot"], "linux/aarch64/graviton2" : ["EESSI-pilot"], "linux/aarch64/graviton3" : ["EESSI-pilot"]} + +# points to definition of repositories (default EESSI-pilot defined by build container) +repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/cfg-bundles + +# configuration for event handler which receives events from a GitHub repository. +[event_handler] +# path to the log file to log messages for event handler +log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_event_handler.log + + +[job_manager] +# path to the log file to log messages for job manager +log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_job_manager.log + +# directory where job manager stores information about jobs to be tracked +# e.g. as symbolic link JOBID -> directory to job +job_ids_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs + +# full path to the job status checking command +poll_command = /usr/bin/squeue + +# polling interval in seconds +poll_interval = 60 + +# full path to the command for manipulating existing jobs +scontrol_command = /usr/bin/scontrol + +[deploycfg] +# script for uploading built software packages +tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging + +# URL to S3/minio bucket +# if attribute is set, bucket_base will be constructed as follows +# bucket_base=${endpoint_url}/${bucket_name} +# otherwise, bucket_base will be constructed as follows +# bucket_base=https://${bucket_name}.s3.amazonaws.com +# - The former variant is used for non AWS S3 services, eg, minio, or when +# the bucket name is not provided in the hostname (see latter case). +# - The latter variant is used for AWS S3 services. +#endpoint_url = URL_TO_S3_SERVER + +# bucket name +bucket_name = eessi-staging + +# upload policy: defines what policy is used for uploading built artefacts +# to an S3 bucket +# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible) +# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH) +# only upload the latest built artefact +# 'once' : only once upload any built artefact for the build target +# 'none' : do not upload any built artefacts +upload_policy = once + +# which GH account has the permission to trigger the deployment (by setting +# the label 'bot:deploy' (apparently this cannot be restricted on GitHub) +# if value is left/empty everyone can trigger the deployment +# value can be a space delimited list of GH accounts +deploy_permission = boegel trz42 bedroge