diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 2fc071da8b..9d31f2aa90 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -23,6 +23,7 @@ jobs: - eessi-2023.06-eb-4.7.2-2021b.yml - eessi-2023.06-eb-4.7.2-2022a.yml - eessi-2023.06-eb-4.7.2-2022b.yml + - eessi-2023.06-eb-4.7.2-system.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 979156830a..c6c51e7abc 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -8,10 +8,43 @@ display_help() { echo "usage: $0 [OPTIONS]" + echo " --build-logs-dir - location to copy EasyBuild logs to for failed builds" echo " -g | --generic - instructs script to build for generic architecture target" echo " -h | --help - display this usage information" echo " -x | --http-proxy URL - provides URL for the environment variable http_proxy" echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy" + echo " --shared-fs-path - path to directory on shared filesystem that can be used" +} + +function copy_build_log() { + # copy specified build log to specified directory, with some context added + build_log=${1} + build_logs_dir=${2} + + # also copy to build logs directory, if specified + if [ ! -z "${build_logs_dir}" ]; then + log_filename="$(basename ${build_log})" + if [ ! -z "${SLURM_JOB_ID}" ]; then + # use subdirectory for build log in context of a Slurm job + build_log_path="${build_logs_dir}/jobs/${SLURM_JOB_ID}/${log_filename}" + else + build_log_path="${build_logs_dir}/non-jobs/${log_filename}" + fi + mkdir -p $(dirname ${build_log_path}) + cp -a ${build_log} ${build_log_path} + chmod 0644 ${build_log_path} + + # add context to end of copied log file + echo >> ${build_log_path} + echo "Context from which build log was copied:" >> ${build_log_path} + echo "- original path of build log: ${build_log}" >> ${build_log_path} + echo "- working directory: ${PWD}" >> ${build_log_path} + echo "- Slurm job ID: ${SLURM_OUT}" >> ${build_log_path} + echo "- EasyBuild version: ${eb_version}" >> ${build_log_path} + echo "- easystack file: ${easystack_file}" >> ${build_log_path} + + echo "EasyBuild log file ${build_log} copied to ${build_log_path} (with context appended)" + fi } POSITIONAL_ARGS=() @@ -35,6 +68,14 @@ while [[ $# -gt 0 ]]; do export https_proxy="$2" shift 2 ;; + --build-logs-dir) + export build_logs_dir="${2}" + shift 2 + ;; + --shared-fs-path) + export shared_fs_path="${2}" + shift 2 + ;; -*|--*) echo "Error: Unknown option: $1" >&2 exit 1 @@ -99,7 +140,7 @@ else echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" fi -# Set all the EESSI environment variables (respecting ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) +# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) # $EESSI_SILENT - don't print any messages # $EESSI_BASIC_ENV - give a basic set of environment variables EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables @@ -125,6 +166,14 @@ fi echo ">> Configuring EasyBuild..." source $TOPDIR/configure_easybuild +if [ ! -z "${shared_fs_path}" ]; then + shared_eb_sourcepath=${shared_fs_path}/easybuild/sources + echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path" + export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH} +fi + +${EB} --show-config + echo ">> Setting up \$MODULEPATH..." # make sure no modules are loaded module --force purge @@ -137,25 +186,42 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -for eb_version in '4.7.2'; do +# assume there's only one diff file that corresponds to the PR patch file +pr_diff=$(ls [0-9]*.diff | head -1) + +# use PR patch file to determine in which easystack files stuff was added +for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$'); do + + echo -e "Processing easystack file ${easystack_file}...\n\n" + + # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file + eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g') # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." - for es in $(ls eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-*.yml); do - - if [ -f ${es} ]; then - echo_green "Feeding easystack file ${es} to EasyBuild..." + if [ -f ${easystack_file} ]; then + echo_green "Feeding easystack file ${easystack_file} to EasyBuild..." - ${EB} --easystack ${TOPDIR}/${es} --robot + ${EB} --easystack ${TOPDIR}/${easystack_file} --robot + ec=$? - $TOPDIR/check_missing_installations.sh ${TOPDIR}/${es} - else - fatal_error "Easystack file ${es} not found!" + # copy EasyBuild log file if EasyBuild exited with an error + if [ ${ec} -ne 0 ]; then + eb_last_log=$(unset EB_VERBOSE; eb --last-log) + # copy to current working directory + cp -a ${eb_last_log} . + echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}" + # copy to build logs dir (with context added) + copy_build_log "${eb_last_log}" "${build_logs_dir}" fi - done + + $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} + else + fatal_error "Easystack file ${easystack_file} not found!" + fi done diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml index 49214fc84a..28612c5566 100644 --- a/eessi-2023.06-eb-4.7.2-2021a.yml +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -12,10 +12,11 @@ easyconfigs: - Rust-1.52.1-GCCcore-10.3.0.eb - foss-2021a.eb - QuantumESPRESSO-6.7-foss-2021a.eb - - GROMACS-2021.3-foss-2021a.eb: - options: - download-timeout: 1000 + - GROMACS-2021.3-foss-2021a.eb: + options: + download-timeout: 1000 - libGLU-9.0.1-GCCcore-10.3.0.eb + # since the BAGEL ec file is new we have to keep the from-pr or eb fails - BAGEL-1.2.2-foss-2021a.eb: options: from-pr: 18446 diff --git a/eessi-2023.06-eb-4.7.2-2021b.yml b/eessi-2023.06-eb-4.7.2-2021b.yml index 45776823ea..d838e157b7 100644 --- a/eessi-2023.06-eb-4.7.2-2021b.yml +++ b/eessi-2023.06-eb-4.7.2-2021b.yml @@ -9,6 +9,8 @@ easyconfigs: - FFTW-3.3.10-gompi-2021b.eb - BLIS-0.8.1-GCC-11.2.0.eb - foss-2021b.eb + # note, the below CMake has never been installed, just put here to use the + # same version as easy does # - CMake-3.22.1-GCCcore-11.2.0.eb: # options: # include-easyblocks-from-pr: 2248 diff --git a/eessi-2023.06-eb-4.7.2-system.yml b/eessi-2023.06-eb-4.7.2-system.yml new file mode 100644 index 0000000000..a5f84c7cd5 --- /dev/null +++ b/eessi-2023.06-eb-4.7.2-system.yml @@ -0,0 +1,4 @@ +easyconfigs: + - EasyBuild-4.8.0.eb: + options: + from-pr: 18282