diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 92b1f71cad..04195dd619 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -40,4 +40,33 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort + echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" ./check_missing_installations.sh + + - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) + run: | + source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash + module load EasyBuild + eb --version + export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}} + export EESSI_OS_TYPE=linux + export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} + env | grep ^EESSI | sort + echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)" + echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml + echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml + tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml + # note, check_missing_installations.sh exits 1 if a package was + # missing, which is intepreted as false (exit code based, not + # boolean logic), hence when the script exits 0 if no package was + # missing it is interpreted as true, thus the test did not capture + # the missing package + if ./check_missing_installations.sh; then + echo "did NOT capture missing package; test FAILED" + exit 1 + else + echo "captured missing package; test PASSED" + exit 0 + fi diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 2830754b29..b7a4c536e6 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -6,6 +6,8 @@ # https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop # https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash +echo "test run for bot PR 172" + display_help() { echo "usage: $0 [OPTIONS]" echo " -g | --generic - instructs script to build for generic architecture target" @@ -165,10 +167,6 @@ else eb --install-latest-eb-release &> ${eb_install_out} check_exit_code $? "${ok_msg}" "${fail_msg}" - # restore origin $PATH and $PYTHONPATH values - export PATH=${ORIG_PATH} - export PYTHONPATH=${ORIG_PYTHONPATH} - eb --search EasyBuild-${REQ_EB_VERSION}.eb | grep EasyBuild-${REQ_EB_VERSION}.eb > /dev/null if [[ $? -eq 0 ]]; then ok_msg="EasyBuild v${REQ_EB_VERSION} installed, alright!" @@ -177,6 +175,10 @@ else check_exit_code $? "${ok_msg}" "${fail_msg}" fi + # restore origin $PATH and $PYTHONPATH values + export PATH=${ORIG_PATH} + export PYTHONPATH=${ORIG_PYTHONPATH} + module avail easybuild/${REQ_EB_VERSION} &> ${ml_av_easybuild_out} if [[ $? -eq 0 ]]; then echo_green ">> EasyBuild module installed!" diff --git a/bot/bot-eessi-aws-citc.cfg b/bot/bot-eessi-aws-citc.cfg new file mode 100644 index 0000000000..5b3ad34612 --- /dev/null +++ b/bot/bot-eessi-aws-citc.cfg @@ -0,0 +1,151 @@ +# Also see documentation at https://github.com/EESSI/eessi-bot-software-layer/blob/main/README.md#step5.5 + +[github] +# replace '123456' with the ID of your GitHub App +app_id = 281041 + +# a short (!) name for your app instance that can be used for example +# when adding/updating a comment to a PR +# (!) a short yet descriptive name is preferred because it appears in +# comments to the PR +# for example, the name could include the name of the cluster the bot +# runs on and the username which runs the bot +# NOTE avoid putting an actual username here as it will be visible on +# potentially publicly accessible GitHub pages. +app_name = eessi-bot-citc-aws + +# replace '12345678' with the ID of the installation of your GitHub App +# (can be derived by creating an event and then checking for the list +# of sent events and its payload either via the Smee channel's web page +# or via the Advanced section of your GitHub App on github.com) +installation_id = 33078935 + +# path to the private key that was generated when the GitHub App was registered +private_key = /mnt/shared/home/bot/eessi-bot-software-layer/eessi-bot-citc-aws-private-key.pem + + +[buildenv] +# name of the job script used for building an EESSI stack +build_job_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/bot-build.slurm + +# The container_cachedir may be used to reuse downloaded container image files +# across jobs. Thus, jobs can more quickly launch containers. +container_cachedir = /mnt/shared/home/bot/eessi-bot-software-layer/containers-cache-dir + +# it may happen that we need to customize some CVMFS configuration +# the value of cvmfs_customizations is a dictionary which maps a file +# name to an entry that needs to be added to that file +cvmfs_customizations = {} + +# if compute nodes have no internet connection, we need to set http(s)_proxy +# or commands such as pip3 cannot download software from package repositories +# for example, the temporary EasyBuild is installed via pip3 first +# http_proxy = http://PROXY_DNS:3128/ +# https_proxy = http://PROXY_DNS:3128/ + +# directory under which the bot prepares directories per job +# structure created is as follows: YYYY.MM/pr_PR_NUMBER/event_EVENT_ID/run_RUN_NUMBER/OS+SUBDIR +jobs_base_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs + +# configure environment +# list of comma-separated modules to be loaded by build_job_script +# useful/needed if some tool is not provided as system-wide package +# (read by bot and handed over to build_job_script via parameter +# --load-modules) +load_modules = + +# PATH to temporary directory on build node ... ends up being used for +# for example, EESSI_TMPDIR --> /tmp/$USER/EESSI +# escaping variables with '\' delays expansion to the start of the +# build_job_script; this can be used for referencing environment +# variables that are only set inside a Slurm job +local_tmp = /tmp/$USER/EESSI + +# parameters to be added to all job submissions +# NOTE do not quote parameter string. Quotes are retained when reading in config and +# then the whole 'string' is recognised as a single parameter. +# NOTE 2 '--get-user-env' may be needed on systems where the job's environment needs +# to be initialised as if it is for a login shell. +# note: hardcoded 24h time limit until https://github.com/EESSI/eessi-bot-software-layer/issues/146 is fixed +slurm_params = --hold --time=24:0:0 + +# full path to the job submission command +submit_command = /usr/bin/sbatch + +# which GH account has the permission to trigger the build (by setting +# the label 'bot:build' (apparently this cannot be restricted on GitHub) +# if value is left/empty everyone can trigger the build +# value can be a space delimited list of GH accounts +build_permission = boegel trz42 bedroge + +[architecturetargets] +# defines both for which architectures the bot will build +# and what submission parameters shall be used +# medium instances (8 cores, 16GB RAM) +#arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} +# larger instances (16 cores, 32GB RAM) +arch_target_map = { "linux/x86_64/generic" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/haswell" : "--constraint shape=c4.4xlarge", "linux/x86_64/intel/skylake_avx512" : "--constraint shape=c5.4xlarge", "linux/x86_64/amd/zen2": "--constraint shape=c5a.4xlarge", "linux/x86_64/amd/zen3" : "--constraint shape=c6a.4xlarge", "linux/aarch64/generic" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton2" : "--constraint shape=c6g.4xlarge", "linux/aarch64/graviton3" : "--constraint shape=c7g.4xlarge"} + +[repo_targets] +# defines for which repository a arch_target should be build for +# +# only building for repository EESSI-pilot +repo_target_map = { "linux/x86_64/generic" : ["EESSI-pilot"], "linux/x86_64/intel/haswell" : ["EESSI-pilot"], "linux/x86_64/intel/skylake_avx512" : ["EESSI-pilot"], "linux/x86_64/amd/zen2": ["EESSI-pilot"], "linux/x86_64/amd/zen3" : ["EESSI-pilot"], "linux/aarch64/generic" : ["EESSI-pilot"], "linux/aarch64/graviton2" : ["EESSI-pilot"], "linux/aarch64/graviton3" : ["EESSI-pilot"]} + +# points to definition of repositories (default EESSI-pilot defined by build container) +repos_cfg_dir = /mnt/shared/home/bot/eessi-bot-software-layer/cfg-bundles + +# configuration for event handler which receives events from a GitHub repository. +[event_handler] +# path to the log file to log messages for event handler +log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_event_handler.log + + +[job_manager] +# path to the log file to log messages for job manager +log_path = /mnt/shared/home/bot/eessi-bot-software-layer/eessi_bot_job_manager.log + +# directory where job manager stores information about jobs to be tracked +# e.g. as symbolic link JOBID -> directory to job +job_ids_dir = /mnt/shared/home/bot/eessi-bot-software-layer/jobs + +# full path to the job status checking command +poll_command = /usr/bin/squeue + +# polling interval in seconds +poll_interval = 60 + +# full path to the command for manipulating existing jobs +scontrol_command = /usr/bin/scontrol + +[deploycfg] +# script for uploading built software packages +tarball_upload_script = /mnt/shared/home/bot/eessi-bot-software-layer/scripts/eessi-upload-to-staging + +# URL to S3/minio bucket +# if attribute is set, bucket_base will be constructed as follows +# bucket_base=${endpoint_url}/${bucket_name} +# otherwise, bucket_base will be constructed as follows +# bucket_base=https://${bucket_name}.s3.amazonaws.com +# - The former variant is used for non AWS S3 services, eg, minio, or when +# the bucket name is not provided in the hostname (see latter case). +# - The latter variant is used for AWS S3 services. +#endpoint_url = URL_TO_S3_SERVER + +# bucket name +bucket_name = eessi-staging + +# upload policy: defines what policy is used for uploading built artefacts +# to an S3 bucket +# 'all' ..: upload all artefacts (mulitple uploads of the same artefact possible) +# 'latest': for each build target (eessi-VERSION-{software,init,compat}-OS-ARCH) +# only upload the latest built artefact +# 'once' : only once upload any built artefact for the build target +# 'none' : do not upload any built artefacts +upload_policy = once + +# which GH account has the permission to trigger the deployment (by setting +# the label 'bot:deploy' (apparently this cannot be restricted on GitHub) +# if value is left/empty everyone can trigger the deployment +# value can be a space delimited list of GH accounts +deploy_permission = boegel trz42 bedroge diff --git a/bot/build.sh b/bot/build.sh new file mode 100755 index 0000000000..c8def2cdd3 --- /dev/null +++ b/bot/build.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# +# script to build the EESSI software layer. Intended use is that it is called +# by a (batch) job running on a compute node. +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# ASSUMPTIONs: +# - working directory has been prepared by the bot with a checkout of a +# pull request (OR by some other means) +# - the working directory contains a directory 'cfg' where the main config +# file 'job.cfg' has been deposited +# - the directory may contain any additional files referenced in job.cfg + +# stop as soon as something fails +set -e + +# source utils.sh and cfg_files.sh +source scripts/utils.sh +source scripts/cfg_files.sh + +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" +HOST_ARCH=$(uname -m) + +# check if ${JOB_CFG_FILE} exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +fi +echo "bot/build.sh: showing ${JOB_CFG_FILE} from software-layer side" +cat ${JOB_CFG_FILE} + +echo "bot/build.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" +cfg_load ${JOB_CFG_FILE} + +# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy +HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy") +HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} +echo "bot/build.sh: HTTP_PROXY='${HTTP_PROXY}'" + +# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy +HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy") +HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} +echo "bot/build.sh: HTTPS_PROXY='${HTTPS_PROXY}'" + +LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp") +echo "bot/build.sh: LOCAL_TMP='${LOCAL_TMP}'" +# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided + +SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir") +echo "bot/build.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'" +if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then + # make sure that separate directories are used for different CPU families + SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH} + export SINGULARITY_CACHEDIR +fi + +echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> " +# replace any env variable in ${LOCAL_TMP} with its +# current value (e.g., a value that is local to the job) +STORAGE=$(envsubst <<< ${LOCAL_TMP}) +echo "'${STORAGE}'" + +# make sure ${STORAGE} exists +mkdir -p ${STORAGE} + +# make sure the base tmp storage is unique +JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX) +echo "bot/build.sh: created unique base tmp storage directory at ${JOB_STORAGE}" + +# obtain list of modules to be loaded +LOAD_MODULES=$(cfg_get_value "site_config" "load_modules") +echo "bot/build.sh: LOAD_MODULES='${LOAD_MODULES}'" + +# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND +CONTAINER=$(cfg_get_value "repository" "container") +export SINGULARITY_HOME="${PWD}:/eessi_bot_job" +export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir" +mkdir -p ${SINGULARITY_TMPDIR} + +# load modules if LOAD_MODULES is not empty +if [[ ! -z ${LOAD_MODULES} ]]; then + for mod in $(echo ${LOAD_MODULES} | tr ',' '\n') + do + echo "bot/build.sh: loading module '${mod}'" + module load ${mod} + done +else + echo "bot/build.sh: no modules to be loaded" +fi + +# determine repository to be used from entry .repository in ${JOB_CFG_FILE} +REPOSITORY=$(cfg_get_value "repository" "repo_id") +EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir") +export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg} +echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'" + +# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE} +# here, just set & export EESSI_PILOT_VERSION_OVERRIDE +# next script (eessi_container.sh) makes use of it via sourcing init scripts +# (e.g., init/eessi_defaults or init/minimal_eessi_env) +export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version") +echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'" + +# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} +# here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down +# "source init/eessi_defaults" via sourcing init/minimal_eessi_env +export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" + +# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} +# fallbacks: +# - ${CPU_TARGET} handed over from bot +# - left empty to let downstream script(s) determine subdir to be used +EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir") +EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} +export EESSI_SOFTWARE_SUBDIR_OVERRIDE +echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" + +# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux) +EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") +export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} +echo "bot/build.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'" + +# prepare arguments to eessi_container.sh common to build and tarball steps +declare -a COMMON_ARGS=() +COMMON_ARGS+=("--verbose") +COMMON_ARGS+=("--access" "rw") +COMMON_ARGS+=("--mode" "run") +[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}") +[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}") +[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}") +[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}") + +# make sure to use the same parent dir for storing tarballs of tmp +PREVIOUS_TMP_DIR=${PWD}/previous_tmp + +# prepare directory to store tarball of tmp for build step +TARBALL_TMP_BUILD_STEP_DIR=${PREVIOUS_TMP_DIR}/build_step +mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR} + +# prepare arguments to eessi_container.sh specific to build step +declare -a BUILD_STEP_ARGS=() +BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}") +BUILD_STEP_ARGS+=("--storage" "${STORAGE}") + +# prepare arguments to install_software_layer.sh (specific to build step) +GENERIC_OPT= +if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then + GENERIC_OPT="--generic" +fi + +# create tmp file for output of build step +build_outerr=$(mktemp build.outerr.XXXX) + +echo "Executing command to build software:" +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${BUILD_STEP_ARGS[@]}" +echo " -- ./install_software_layer.sh ${GENERIC_OPT} \"$@\" 2>&1 | tee -a ${build_outerr}" +./eessi_container.sh "${COMMON_ARGS[@]}" "${BUILD_STEP_ARGS[@]}" \ + -- ./install_software_layer.sh ${GENERIC_OPT} "$@" 2>&1 | tee -a ${build_outerr} + +# prepare directory to store tarball of tmp for tarball step +TARBALL_TMP_TARBALL_STEP_DIR=${PREVIOUS_TMP_DIR}/tarball_step +mkdir -p ${TARBALL_TMP_TARBALL_STEP_DIR} + +# create tmp file for output of tarball step +tar_outerr=$(mktemp tar.outerr.XXXX) + +# prepare arguments to eessi_container.sh specific to tarball step +declare -a TARBALL_STEP_ARGS=() +TARBALL_STEP_ARGS+=("--save" "${TARBALL_TMP_TARBALL_STEP_DIR}") + +# determine temporary directory to resume from +BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2) +TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}") + +timestamp=$(date +%s) +# to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now +source init/eessi_defaults +export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp}) + +# value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be +# synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh +# TODO should we make this a configurable parameter of eessi_container.sh using +# /tmp as default? +TMP_IN_CONTAINER=/tmp +echo "Executing command to create tarball:" +echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" +echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ + -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + +exit 0 diff --git a/check_missing_installations.sh b/check_missing_installations.sh index e4c75aebd8..926f475903 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -27,5 +27,18 @@ eb_missing_out=$LOCAL_TMPDIR/eb_missing.out # we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used # PR #16531: Nextflow-22.10.1.eb ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out} -grep "No missing modules" ${eb_missing_out} > /dev/null -check_exit_code $? "${ok_msg}" "${fail_msg}" + +# the above assesses the installed software for each easyconfig provided in +# the easystack file and then print messages such as +# `No missing modules!` +# or +# `2 out of 3 required modules missing:` +# depending on the result of the assessment. Hence, we need to check if the +# output does not contain any line with ` required modules missing:` + +grep " required modules missing:" ${eb_missing_out} > /dev/null + +# if grep returns 1 (` required modules missing:` was NOT found), we set +# MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1 +[[ $? -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1 +check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}" diff --git a/configure_easybuild b/configure_easybuild index 19b2d7454b..245553f342 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -13,6 +13,7 @@ export EASYBUILD_ZIP_LOGS=bzip2 export EASYBUILD_RPATH=1 export EASYBUILD_FILTER_ENV_VARS=LD_LIBRARY_PATH +export EASYBUILD_READ_ONLY_INSTALLDIR=1 # assume that eb_hooks.py is located in same directory as this script (configure_easybuild) TOPDIR=$(dirname $(realpath $BASH_SOURCE)) diff --git a/create_directory_tarballs.sh b/create_directory_tarballs.sh index be282463eb..70e666f871 100755 --- a/create_directory_tarballs.sh +++ b/create_directory_tarballs.sh @@ -26,7 +26,7 @@ fi tartmp=$(mktemp -t -d init.XXXXX) mkdir "${tartmp}/${version}" tarname="eessi-${version}-init-$(date +%s).tar.gz" -curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards */init/ +curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --no-wildcards-match-slash --wildcards '*/init/' source "${tartmp}/${version}/init/minimal_eessi_env" if [ "${EESSI_PILOT_VERSION}" != "${version}" ] then @@ -42,7 +42,7 @@ echo_green "Done! Created tarball ${tarname}." tartmp=$(mktemp -t -d scripts.XXXXX) mkdir "${tartmp}/${version}" tarname="eessi-${version}-scripts-$(date +%s).tar.gz" -curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --wildcards '*/scripts/' +curl -Ls ${SOFTWARE_LAYER_TARBALL_URL} | tar xzf - -C "${tartmp}/${version}" --strip-components=1 --no-wildcards-match-slash --wildcards '*/scripts/' tar czf "${tarname}" -C "${tartmp}" "${version}" rm -rf "${tartmp}" diff --git a/create_tarball.sh b/create_tarball.sh index 56ac8ab7ad..b6c72b341d 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -38,6 +38,7 @@ cd ${overlay_upper_dir}/versions/ echo ">> Collecting list of files/directories to include in tarball via ${PWD}..." files_list=${tmpdir}/files.list.txt +module_files_list=${tmpdir}/module_files.list.txt if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then # include Lmod cache and configuration file (lmodrc.lua), @@ -49,12 +50,31 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} # module symlinks find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + # module files and symlinks + find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ + | grep -v '/\.wh\.' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ + >> ${module_files_list} fi -if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then - # installation directories - ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list} +if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then + # installation directories but only those for which module files were created + # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' + # using EasyBuild's standard module naming scheme) match the name of the + # software installation directory (expected to be 'PACKAGE_NAME/VERSION/'). + # If either side changes (module naming scheme or naming of software + # installation directories), the procedure will likely not work. + for package_version in $(cat ${module_files_list}); do + echo "handling ${package_version}" + ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ + | grep -v '/\.wh\.' >> ${files_list} + done fi +# add a bit debug output +echo "wrote file list to ${files_list}" +[ -r ${files_list} ] && cat ${files_list} +echo "wrote module file list to ${module_files_list}" +[ -r ${module_files_list} ] && cat ${module_files_list} + topdir=${cvmfs_repo}/versions/ echo ">> Creating tarball ${target_tgz} from ${topdir}..." diff --git a/eb_hooks.py b/eb_hooks.py index df7742f999..777d583c7b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -3,8 +3,11 @@ import os import re +from easybuild.easyblocks.generic.configuremake import obtain_config_guess from easybuild.tools.build_log import EasyBuildError, print_msg from easybuild.tools.config import build_option, update_build_option +from easybuild.tools.filetools import apply_regex_substitutions, copy_file, which +from easybuild.tools.run import run_cmd from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC @@ -89,6 +92,36 @@ def pre_prepare_hook(self, *args, **kwargs): mpi_family, rpath_override_dirs) +def gcc_postprepare(self, *args, **kwargs): + """ + Post-configure hook for GCCcore: + - copy RPATH wrapper script for linker commands to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu' + """ + if self.name == 'GCCcore': + config_guess = obtain_config_guess() + system_type, _ = run_cmd(config_guess, log_all=True) + cmd_prefix = '%s-' % system_type.strip() + for cmd in ('ld', 'ld.gold', 'ld.bfd'): + wrapper = which(cmd) + self.log.info("Path to %s wrapper: %s" % (cmd, wrapper)) + wrapper_dir = os.path.dirname(wrapper) + prefix_wrapper = os.path.join(wrapper_dir, cmd_prefix + cmd) + copy_file(wrapper, prefix_wrapper) + self.log.info("Path to %s wrapper with '%s' prefix: %s" % (cmd, cmd_prefix, which(prefix_wrapper))) + + # we need to tweak the copied wrapper script, so that: + regex_subs = [ + # - CMD in the script is set to the command name without prefix, because EasyBuild's rpath_args.py + # script that is used by the wrapper script only checks for 'ld', 'ld.gold', etc. + # when checking whether or not to use -Wl + ('^CMD=.*', 'CMD=%s' % cmd), + # - the path to the correct actual binary is logged and called + ('/%s ' % cmd, '/%s ' % (cmd_prefix + cmd)), + ] + apply_regex_substitutions(prefix_wrapper, regex_subs) + else: + raise EasyBuildError("GCCcore-specific hook triggered for non-GCCcore easyconfig?!") + def post_prepare_hook(self, *args, **kwargs): """Main post-prepare hook: trigger custom functions.""" @@ -98,6 +131,9 @@ def post_prepare_hook(self, *args, **kwargs): print_msg("Resetting rpath_override_dirs to original value: %s", getattr(self, EESSI_RPATH_OVERRIDE_ATTR)) delattr(self, EESSI_RPATH_OVERRIDE_ATTR) + if self.name in POST_PREPARE_HOOKS: + POST_PREPARE_HOOKS[self.name](self, *args, **kwargs) + def cgal_toolchainopts_precise(ec, eprefix): """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER.""" @@ -187,6 +223,10 @@ def wrf_preconfigure(self, *args, **kwargs): 'UCX': ucx_eprefix, } +POST_PREPARE_HOOKS = { + 'GCCcore': gcc_postprepare, +} + PRE_CONFIGURE_HOOKS = { 'libfabric': libfabric_disable_psm3_x86_64_generic, 'MetaBAT': metabat_preconfigure, diff --git a/eessi_container.sh b/eessi_container.sh index 1d76360735..48c4653ba9 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -67,7 +67,7 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg" # https://stackoverflow.com/questions/192249/how-do-i-parse-command-line-arguments-in-bash display_help() { - echo "usage: $0 [OPTIONS] [SCRIPT]" + echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]" echo " OPTIONS:" echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" echo " -c | --container IMG - image file or URL defining the container to use" @@ -77,10 +77,10 @@ display_help() { echo " temporary data) [default: 1. TMPDIR, 2. /tmp]" echo " -l | --list-repos - list available repository identifiers [default: false]" echo " -m | --mode MODE - with MODE==shell (launch interactive shell) or" - echo " MODE==run (run a script) [default: shell]" + echo " MODE==run (run a script or command) [default: shell]" echo " -r | --repository CFG - configuration file or identifier defining the" echo " repository to use [default: EESSI-pilot via" - echo " container configuration]" + echo " default container, see --container]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," echo " where DIR points to a previously used tmp directory" echo " (check for output 'Using DIR as tmp ...' of a previous" @@ -98,7 +98,9 @@ display_help() { echo " -y | --https-proxy URL - provides URL for the env variable https_proxy" echo " [default: not set]; uses env var \$https_proxy if set" echo - echo " If value for --mode is 'run', the SCRIPT provided is executed." + echo " If value for --mode is 'run', the SCRIPT/COMMAND provided is executed. If" + echo " arguments to the script/command start with '-' or '--', use the flag terminator" + echo " '--' to let eessi_container.sh stop parsing arguments." } # set defaults for command line arguments @@ -173,6 +175,11 @@ while [[ $# -gt 0 ]]; do export https_proxy=${HTTPS_PROXY} shift 2 ;; + --) + shift + POSITIONAL_ARGS+=("$@") # save positional args + break + ;; -*|--*) fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" ;; @@ -185,7 +192,6 @@ done set -- "${POSITIONAL_ARGS[@]}" - if [[ ${LIST_REPOS} -eq 1 ]]; then echo "Listing available repositories with format 'name [source]':" echo " EESSI-pilot [default]" @@ -285,7 +291,7 @@ else [[ ${VERBOSE} -eq 1 ]] && echo "skipping sanity checks for /tmp" fi EESSI_HOST_STORAGE=$(mktemp -d --tmpdir eessi.XXXXXXXXXX) - echo "Using ${EESSI_HOST_STORAGE} as tmp storage (add '--resume ${EESSI_HOST_STORAGE}' to resume where this session ended)." + echo "Using ${EESSI_HOST_STORAGE} as tmp directory (to resume session add '--resume ${EESSI_HOST_STORAGE}')." fi # if ${RESUME} is a file (assume a tgz), unpack it into ${EESSI_HOST_STORAGE} @@ -572,7 +578,7 @@ if [[ ! -z ${SAVE} ]]; then TGZ=${SAVE} fi tar cf ${TGZ} -C ${EESSI_TMPDIR} . - echo "Saved contents of '${EESSI_TMPDIR}' to '${TGZ}' (to resume, add '--resume ${TGZ}')" + echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TGZ}' (to resume session add '--resume ${TGZ}')" fi # TODO clean up tmp by default? only retain if another option provided (--retain-tmp) diff --git a/init/eessi_software_subdir_for_host.py b/init/eessi_software_subdir_for_host.py index b09b4711e4..58e9cfd2e6 100755 --- a/init/eessi_software_subdir_for_host.py +++ b/init/eessi_software_subdir_for_host.py @@ -101,8 +101,6 @@ def find_best_target(eessi_prefix): continue if uarch in KNOWN_CPU_UARCHS: target_uarchs.append(KNOWN_CPU_UARCHS[uarch]) - else: - warning('Ignoring unknown target "%s"' % uarch) host_uarch = KNOWN_CPU_UARCHS[host_cpu_name] compat_target_uarchs = sorted([x for x in target_uarchs if x <= host_uarch]) diff --git a/init/test.py b/init/test.py index 0ed09abd6f..f10be5e66e 100644 --- a/init/test.py +++ b/init/test.py @@ -60,12 +60,12 @@ def broadwell_host_triple(): prep_tmpdir(tmpdir, ['x86_64/intel/ivybridge']) assert find_best_target(tmpdir) == 'x86_64/intel/ivybridge' - # unknown targets don't cause trouble (only warning) + # unknown targets don't cause trouble prep_tmpdir(tmpdir, ['x86_64/intel/no_such_intel_cpu']) assert find_best_target(tmpdir) == 'x86_64/intel/ivybridge' captured = capsys.readouterr() assert captured.out == '' - assert captured.err == 'WARNING: Ignoring unknown target "no_such_intel_cpu"\n' + assert captured.err == '' # older targets have to no impact on best target (sandybridge < ivybridge) prep_tmpdir(tmpdir, ['x86_64/intel/sandybridge']) diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index be01bcd201..c70077bf15 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -17,6 +17,18 @@ INPUT=$(echo "$@") if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" fi +if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then + INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" +fi +if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then + INPUT="export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}; ${INPUT}" +fi +if [ ! -z ${http_proxy} ]; then + INPUT="export http_proxy=${http_proxy}; ${INPUT}" +fi +if [ ! -z ${https_proxy} ]; then + INPUT="export https_proxy=${https_proxy}; ${INPUT}" +fi -echo "Running '${INPUT}' in EESSI ${EESSI_PILOT_VERSION} compatibility layer environment..." +echo "Running '${INPUT}' in EESSI (${EESSI_CVMFS_REPO}) ${EESSI_PILOT_VERSION} compatibility layer environment..." ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< "${INPUT}" diff --git a/versions/2021.06/init/Magic_Castle/bash b/versions/2021.06/init/Magic_Castle/bash new file mode 100644 index 0000000000..5f149c817f --- /dev/null +++ b/versions/2021.06/init/Magic_Castle/bash @@ -0,0 +1,3 @@ +/cvmfs/pilot.eessi-hpc.org/versions/2021.06/init/print_deprecation_warning.sh + +source /cvmfs/pilot.eessi-hpc.org/versions/2021.12/init/Magic_Castle/bash diff --git a/versions/2021.06/init/bash b/versions/2021.06/init/bash new file mode 100644 index 0000000000..82a078849a --- /dev/null +++ b/versions/2021.06/init/bash @@ -0,0 +1,3 @@ +/cvmfs/pilot.eessi-hpc.org/versions/2021.06/init/print_deprecation_warning.sh + +source /cvmfs/pilot.eessi-hpc.org/versions/2021.12/init/bash diff --git a/versions/2021.06/init/print_deprecation_warning.sh b/versions/2021.06/init/print_deprecation_warning.sh new file mode 100755 index 0000000000..b721ed2f71 --- /dev/null +++ b/versions/2021.06/init/print_deprecation_warning.sh @@ -0,0 +1,19 @@ +#!/bin/bash +function echo_yellow_stderr() { + echo -e "\e[33m${1}\e[0m" >&2 +} + +echo_yellow_stderr +echo_yellow_stderr "WARNING: Version 2021.06 of the EESSI pilot repository has been removed since 16 May 2023." +echo_yellow_stderr +echo_yellow_stderr "Version 2021.12 of the EESSI pilot repository can be used as a drop-in replacement, " +echo_yellow_stderr "so we have prepared your environment to use that instead." +echo_yellow_stderr +echo_yellow_stderr "In the future, please run" +echo_yellow_stderr +echo_yellow_stderr " source /cvmfs/pilot.eessi-hpc.org/latest/init/bash" +echo_yellow_stderr +echo_yellow_stderr "to prepare your start using the EESSI pilot repository." +echo_yellow_stderr +echo_yellow_stderr "See also https://eessi.github.io/docs/using_eessi/setting_up_environment ." +echo_yellow_stderr