Skip to content

Commit

Permalink
flesh out load_easybuild_module.sh script from EESSI-pilot-install-so…
Browse files Browse the repository at this point in the history
…ftware.sh
  • Loading branch information
boegel committed Jun 3, 2023
1 parent b0998c5 commit bb37c31
Show file tree
Hide file tree
Showing 3 changed files with 158 additions and 66 deletions.
50 changes: 48 additions & 2 deletions .github/workflows/tests_scripts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,22 +4,26 @@ on:
push:
paths:
- build_container.sh
- create_directory_tarballs.sh
- EESSI-pilot-install-software.sh
- install_software_layer.sh
- load_easybuild_module.sh
- run_in_compat_layer_env.sh
- scripts/utils.sh
- update_lmod_cache.sh
- create_directory_tarballs.sh

pull_request:
branches:
- main
paths:
- build_container.sh
- create_directory_tarballs.sh
- EESSI-pilot-install-software.sh
- install_software_layer.sh
- load_easybuild_module.sh
- run_in_compat_layer_env.sh
- scripts/utils.sh
- update_lmod_cache.sh
- create_directory_tarballs.sh
permissions:
contents: read # to fetch code (actions/checkout)
jobs:
Expand All @@ -33,6 +37,48 @@ jobs:
run: |
./install_apptainer_ubuntu.sh
- name: test load_easybuild_module.sh script
run: |
export TMPDIR=$(mktemp -d)
# run tests from /tmp where scripts were copied into,
# since scripts must be accessible from within build container
cp -a * /tmp/
cd /tmp
for EB_VERSION in '4.5.0' '4.5.1' '4.7.2'; do
# Create script that uses load_easybuild_module.sh which we can run in compat layer environment
# note: Be careful with single vs double quotes below!
# ${TMPDIR} and ${EB_VERSION} should be expanded, so use double quotes;
# For statements using variables that are only defined in the script, like ${EASYBUILD_INSTALLPATH},
# use single quotes to avoid expansion while creating the script.
test_script="${TMPDIR}/eb-${EB_VERSION}.sh"
echo '#!/bin/bash' > ${test_script}
# both $EB and $TMPDIR environment must be set, required by load_easybuild_module.sh script
echo 'export EB="eb"' >> ${test_script}
echo "export TMPDIR=${TMPDIR}" >> ${test_script}
# set up environment to have utility functions in place that load_easybuild_module.sh script relies on,
# along with $EESSI_* environment variables, and Lmod
echo 'source /tmp/scripts/utils.sh' >> ${test_script}
echo 'source /tmp/init/eessi_environment_variables' >> ${test_script}
echo 'source ${EPREFIX}/usr/share/Lmod/init/bash' >> ${test_script}
# minimal configuration for EasyBuild so we can test installation aspect of load_easybuild_module.sh script
echo "export EASYBUILD_INSTALLPATH=${TMPDIR}" >> ${test_script}
echo 'module use ${EASYBUILD_INSTALLPATH}/modules/all' >> ${test_script}
echo '' >> ${test_script}
echo "source /tmp/load_easybuild_module.sh ${EB_VERSION}" >> ${test_script}
echo 'module list' >> ${test_script}
echo 'eb --version' >> ${test_script}
chmod u+x ${test_script}
# run wrapper script + capture & check output
out="${TMPDIR}/eb-${EB_VERSION}.out"
./build_container.sh run /tmp/$USER/EESSI /tmp/run_in_compat_layer_env.sh ${test_script} 2>&1 | tee ${out}
pattern="^This is EasyBuild ${EB_VERSION} "
grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1)
done
- name: test install_software_layer.sh script
run: |
# scripts need to be copied to /tmp,
Expand Down
66 changes: 2 additions & 64 deletions EESSI-pilot-install-software.sh
Original file line number Diff line number Diff line change
Expand Up @@ -139,70 +139,8 @@ fi

REQ_EB_VERSION='4.5.0'

echo ">> Checking for EasyBuild module..."
ml_av_easybuild_out=$TMPDIR/ml_av_easybuild.out
module avail 2>&1 | grep -i easybuild/${REQ_EB_VERSION} &> ${ml_av_easybuild_out}
if [[ $? -eq 0 ]]; then
echo_green ">> EasyBuild module found!"
else
echo_yellow ">> No EasyBuild module yet, installing it..."

EB_TMPDIR=${TMPDIR}/ebtmp
echo ">> Temporary installation (in ${EB_TMPDIR})..."
pip_install_out=${TMPDIR}/pip_install.out
pip3 install --prefix $EB_TMPDIR easybuild &> ${pip_install_out}

# keep track of original $PATH and $PYTHONPATH values, so we can restore them
ORIG_PATH=$PATH
ORIG_PYTHONPATH=$PYTHONPATH

echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..."
export PATH=${EB_TMPDIR}/bin:$PATH
export PYTHONPATH=$(ls -d ${EB_TMPDIR}/lib/python*/site-packages):$PYTHONPATH
eb_install_out=${TMPDIR}/eb_install.out
ok_msg="Latest EasyBuild release installed, let's go!"
fail_msg="Installing latest EasyBuild release failed, that's not good... (output: ${eb_install_out})"
eb --install-latest-eb-release &> ${eb_install_out}
check_exit_code $? "${ok_msg}" "${fail_msg}"

eb --search EasyBuild-${REQ_EB_VERSION}.eb | grep EasyBuild-${REQ_EB_VERSION}.eb > /dev/null
if [[ $? -eq 0 ]]; then
ok_msg="EasyBuild v${REQ_EB_VERSION} installed, alright!"
fail_msg="Installing EasyBuild v${REQ_EB_VERSION}, yikes! (output: ${eb_install_out})"
eb EasyBuild-${REQ_EB_VERSION}.eb >> ${eb_install_out} 2>&1
check_exit_code $? "${ok_msg}" "${fail_msg}"
fi

# restore origin $PATH and $PYTHONPATH values
export PATH=${ORIG_PATH}
export PYTHONPATH=${ORIG_PYTHONPATH}

module avail easybuild/${REQ_EB_VERSION} &> ${ml_av_easybuild_out}
if [[ $? -eq 0 ]]; then
echo_green ">> EasyBuild module installed!"
else
fatal_error "EasyBuild/${REQ_EB_VERSION} module failed to install?! (output of 'pip install' in ${pip_install_out}, output of 'eb' in ${eb_install_out}, output of 'ml av easybuild' in ${ml_av_easybuild_out})"
fi
fi

echo ">> Loading EasyBuild module..."
module load EasyBuild/$REQ_EB_VERSION
eb_show_system_info_out=${TMPDIR}/eb_show_system_info.out
$EB --show-system-info > ${eb_show_system_info_out}
if [[ $? -eq 0 ]]; then
echo_green ">> EasyBuild seems to be working!"
$EB --version | grep "${REQ_EB_VERSION}"
if [[ $? -eq 0 ]]; then
echo_green "Found EasyBuild version ${REQ_EB_VERSION}, looking good!"
else
$EB --version
fatal_error "Expected to find EasyBuild version ${REQ_EB_VERSION}, giving up here..."
fi
$EB --show-config
else
cat ${eb_show_system_info_out}
fatal_error "EasyBuild not working?!"
fi
# load EasyBuild module (will be installed if it's not available yet)
source ${TOPDIR}/load_easybuild_module.sh ${REQ_EB_VERSION}

echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..."

Expand Down
108 changes: 108 additions & 0 deletions load_easybuild_module.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
# Script to load the environment module for a specific version of EasyBuild.
# If that module is not available yet, the current latest EasyBuild version of EasyBuild will be installed,
# and used to install the specific EasyBuild version being specified.
#
# This script must be sourced, since it makes changes in the current environment, like loading an EasyBuild module.

set -o pipefail

if [ $# -ne 1 ]; then
echo "Usage: $0 <EasyBuild version>" >&2
exit 1
fi

# don't use $EB_VERSION, since that enables always running 'eb --version'
EB_VERSION=${1}

# make sure that environment variables that we expect to be set are indeed set
if [ -z "${TMPDIR}" ]; then
echo "\$TMPDIR is not set" >&2
exit 2
fi

if [ -z "${EB}" ]; then
echo "\$EB is not set" >&2
exit 2
fi

# make sure that utility functions are defined (cfr. scripts/utils.sh script in EESSI/software-layer repo)
type check_exit_code
if [ $? -ne 0 ]; then
echo "check_exit_code function is not defined" >&2
exit 3
fi

echo ">> Checking for EasyBuild module..."

ml_av_easybuild_out=${TMPDIR}/ml_av_easybuild.out
module avail 2>&1 | grep -i easybuild/${EB_VERSION} &> ${ml_av_easybuild_out}

if [[ $? -eq 0 ]]; then
echo_green ">> Module for EasyBuild v${EB_VERSION} found!"
else
echo_yellow ">> No module yet for EasyBuild v${EB_VERSION}, installing it..."

EB_TMPDIR=${TMPDIR}/ebtmp
echo ">> Temporary installation (in ${EB_TMPDIR})..."
pip_install_out=${TMPDIR}/pip_install.out
pip3 install --prefix ${EB_TMPDIR} easybuild &> ${pip_install_out}

# keep track of original $PATH and $PYTHONPATH values, so we can restore them
ORIG_PATH=${PATH}
ORIG_PYTHONPATH=${PYTHONPATH}

echo ">> Final installation in ${EASYBUILD_INSTALLPATH}..."
export PATH=${EB_TMPDIR}/bin:${PATH}
export PYTHONPATH=$(ls -d ${EB_TMPDIR}/lib/python*/site-packages):${PYTHONPATH}
eb_install_out=${TMPDIR}/eb_install.out
ok_msg="Latest EasyBuild release installed, let's go!"
fail_msg="Installing latest EasyBuild release failed, that's not good... (output: ${eb_install_out})"
${EB} --install-latest-eb-release 2>&1 | tee ${eb_install_out}
check_exit_code $? "${ok_msg}" "${fail_msg}"

eb_ec=EasyBuild-${EB_VERSION}.eb
${EB} --search ${eb_ec} | grep ${eb_ec} > /dev/null
if [[ $? -eq 0 ]]; then
echo "Easyconfig found for EasyBuild v${EB_VERSION}, so installing it..."
ok_msg="EasyBuild v${EB_VERSION} installed, alright!"
fail_msg="Installing EasyBuild v${EB_VERSION}, yikes! (output: ${eb_install_out})"
#${EB} EasyBuild-${EB_VERSION}.eb >> ${eb_install_out} 2>&1
${EB} EasyBuild-${EB_VERSION}.eb 2>&1 | tee -a ${eb_install_out}
check_exit_code $? "${ok_msg}" "${fail_msg}"
else
echo "No easyconfig found for EasyBuild v${EB_VERSION}"
fi

# restore origin $PATH and $PYTHONPATH values, and clean up environment variables that are no longer needed
export PATH=${ORIG_PATH}
export PYTHONPATH=${ORIG_PYTHONPATH}
unset EB_TMPDIR ORIG_PATH ORIG_PYTHONPATH

module avail easybuild/${EB_VERSION} &> ${ml_av_easybuild_out}
if [[ $? -eq 0 ]]; then
echo_green ">> EasyBuild module installed!"
else
fatal_error "EasyBuild/${EB_VERSION} module failed to install?! (output of 'pip install' in ${pip_install_out}, output of 'eb' in ${eb_install_out}, output of 'ml av easybuild' in ${ml_av_easybuild_out})"
fi
fi

echo ">> Loading EasyBuild v${EB_VERSION} module..."
module load EasyBuild/${EB_VERSION}
eb_show_system_info_out=${TMPDIR}/eb_show_system_info.out
${EB} --show-system-info > ${eb_show_system_info_out}
if [[ $? -eq 0 ]]; then
echo_green ">> EasyBuild seems to be working!"
${EB} --version | grep "${EB_VERSION}"
if [[ $? -eq 0 ]]; then
echo_green "Found EasyBuild version ${EB_VERSION}, looking good!"
else
${EB} --version
fatal_error "Expected to find EasyBuild version ${EB_VERSION}, giving up here..."
fi
${EB} --show-config
else
cat ${eb_show_system_info_out}
fatal_error "EasyBuild not working?!"
fi

unset EB_VERSION

0 comments on commit bb37c31

Please sign in to comment.