Skip to content

Commit

Permalink
Merge branch 'nessi.no-2023.06' of https://github.com/NorESSI/softwar…
Browse files Browse the repository at this point in the history
…e-layer into nessi.no-2023.06-Yambo/5.1.1-foss/2021a
  • Loading branch information
Richard Top committed Oct 5, 2023
2 parents b35f7f6 + 2db782f commit f6c383b
Show file tree
Hide file tree
Showing 5 changed files with 88 additions and 14 deletions.
1 change: 1 addition & 0 deletions .github/workflows/test_eessi.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ jobs:
- eessi-2023.06-eb-4.7.2-2021b.yml
- eessi-2023.06-eb-4.7.2-2022a.yml
- eessi-2023.06-eb-4.7.2-2022b.yml
- eessi-2023.06-eb-4.7.2-system.yml
steps:
- name: Check out software-layer repository
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
Expand Down
88 changes: 77 additions & 11 deletions EESSI-pilot-install-software.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,43 @@

display_help() {
echo "usage: $0 [OPTIONS]"
echo " --build-logs-dir - location to copy EasyBuild logs to for failed builds"
echo " -g | --generic - instructs script to build for generic architecture target"
echo " -h | --help - display this usage information"
echo " -x | --http-proxy URL - provides URL for the environment variable http_proxy"
echo " -y | --https-proxy URL - provides URL for the environment variable https_proxy"
echo " --shared-fs-path - path to directory on shared filesystem that can be used"
}

function copy_build_log() {
# copy specified build log to specified directory, with some context added
build_log=${1}
build_logs_dir=${2}

# also copy to build logs directory, if specified
if [ ! -z "${build_logs_dir}" ]; then
log_filename="$(basename ${build_log})"
if [ ! -z "${SLURM_JOB_ID}" ]; then
# use subdirectory for build log in context of a Slurm job
build_log_path="${build_logs_dir}/jobs/${SLURM_JOB_ID}/${log_filename}"
else
build_log_path="${build_logs_dir}/non-jobs/${log_filename}"
fi
mkdir -p $(dirname ${build_log_path})
cp -a ${build_log} ${build_log_path}
chmod 0644 ${build_log_path}

# add context to end of copied log file
echo >> ${build_log_path}
echo "Context from which build log was copied:" >> ${build_log_path}
echo "- original path of build log: ${build_log}" >> ${build_log_path}
echo "- working directory: ${PWD}" >> ${build_log_path}
echo "- Slurm job ID: ${SLURM_OUT}" >> ${build_log_path}
echo "- EasyBuild version: ${eb_version}" >> ${build_log_path}
echo "- easystack file: ${easystack_file}" >> ${build_log_path}

echo "EasyBuild log file ${build_log} copied to ${build_log_path} (with context appended)"
fi
}

POSITIONAL_ARGS=()
Expand All @@ -35,6 +68,14 @@ while [[ $# -gt 0 ]]; do
export https_proxy="$2"
shift 2
;;
--build-logs-dir)
export build_logs_dir="${2}"
shift 2
;;
--shared-fs-path)
export shared_fs_path="${2}"
shift 2
;;
-*|--*)
echo "Error: Unknown option: $1" >&2
exit 1
Expand Down Expand Up @@ -99,7 +140,7 @@ else
echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
fi

# Set all the EESSI environment variables (respecting ${EESSI_SOFTWARE_SUBDIR_OVERRIDE})
# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
# $EESSI_SILENT - don't print any messages
# $EESSI_BASIC_ENV - give a basic set of environment variables
EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
Expand All @@ -125,6 +166,14 @@ fi
echo ">> Configuring EasyBuild..."
source $TOPDIR/configure_easybuild

if [ ! -z "${shared_fs_path}" ]; then
shared_eb_sourcepath=${shared_fs_path}/easybuild/sources
echo ">> Using ${shared_eb_sourcepath} as shared EasyBuild source path"
export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH}
fi

${EB} --show-config

echo ">> Setting up \$MODULEPATH..."
# make sure no modules are loaded
module --force purge
Expand All @@ -137,25 +186,42 @@ else
echo_green ">> MODULEPATH set up: ${MODULEPATH}"
fi

for eb_version in '4.7.2'; do
# assume there's only one diff file that corresponds to the PR patch file
pr_diff=$(ls [0-9]*.diff | head -1)

# use PR patch file to determine in which easystack files stuff was added
for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^eessi.*yml$'); do

echo -e "Processing easystack file ${easystack_file}...\n\n"

# determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')

# load EasyBuild module (will be installed if it's not available yet)
source ${TOPDIR}/load_easybuild_module.sh ${eb_version}

echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."

for es in $(ls eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-*.yml); do

if [ -f ${es} ]; then
echo_green "Feeding easystack file ${es} to EasyBuild..."
if [ -f ${easystack_file} ]; then
echo_green "Feeding easystack file ${easystack_file} to EasyBuild..."

${EB} --easystack ${TOPDIR}/${es} --robot
${EB} --easystack ${TOPDIR}/${easystack_file} --robot
ec=$?

$TOPDIR/check_missing_installations.sh ${TOPDIR}/${es}
else
fatal_error "Easystack file ${es} not found!"
# copy EasyBuild log file if EasyBuild exited with an error
if [ ${ec} -ne 0 ]; then
eb_last_log=$(unset EB_VERBOSE; eb --last-log)
# copy to current working directory
cp -a ${eb_last_log} .
echo "Last EasyBuild log file copied from ${eb_last_log} to ${PWD}"
# copy to build logs dir (with context added)
copy_build_log "${eb_last_log}" "${build_logs_dir}"
fi
done

$TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file}
else
fatal_error "Easystack file ${easystack_file} not found!"
fi

done

Expand Down
7 changes: 4 additions & 3 deletions eessi-2023.06-eb-4.7.2-2021a.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,11 @@ easyconfigs:
- Rust-1.52.1-GCCcore-10.3.0.eb
- foss-2021a.eb
- QuantumESPRESSO-6.7-foss-2021a.eb
- GROMACS-2021.3-foss-2021a.eb:
options:
download-timeout: 1000
- GROMACS-2021.3-foss-2021a.eb:
options:
download-timeout: 1000
- libGLU-9.0.1-GCCcore-10.3.0.eb
# since the BAGEL ec file is new we have to keep the from-pr or eb fails
- BAGEL-1.2.2-foss-2021a.eb:
options:
from-pr: 18446
Expand Down
2 changes: 2 additions & 0 deletions eessi-2023.06-eb-4.7.2-2021b.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ easyconfigs:
- FFTW-3.3.10-gompi-2021b.eb
- BLIS-0.8.1-GCC-11.2.0.eb
- foss-2021b.eb
# note, the below CMake has never been installed, just put here to use the
# same version as easy does
# - CMake-3.22.1-GCCcore-11.2.0.eb:
# options:
# include-easyblocks-from-pr: 2248
Expand Down
4 changes: 4 additions & 0 deletions eessi-2023.06-eb-4.7.2-system.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
easyconfigs:
- EasyBuild-4.8.0.eb:
options:
from-pr: 18282

0 comments on commit f6c383b

Please sign in to comment.