diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml
index 92b1f71cad..5e496a6b7d 100644
--- a/.github/workflows/test_eessi.yml
+++ b/.github/workflows/test_eessi.yml
@@ -31,7 +31,19 @@ jobs:
cvmfs_http_proxy: DIRECT
cvmfs_repositories: pilot.eessi-hpc.org
- - name: Test check_missing_installations.sh script
+# - name: Test check_missing_installations.sh script
+# run: |
+# source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+# module load EasyBuild
+# eb --version
+# export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+# export EESSI_OS_TYPE=linux
+# export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
+# env | grep ^EESSI | sort
+# echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
+# ./check_missing_installations.sh
+
+ - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
run: |
source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
module load EasyBuild
@@ -40,4 +52,21 @@ jobs:
export EESSI_OS_TYPE=linux
export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
env | grep ^EESSI | sort
- ./check_missing_installations.sh
+ echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)"
+ echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+ echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+ echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml
+ echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml
+ tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml
+ # note, check_missing_installations.sh exits 1 if a package was
+ # missing, which is intepreted as false (exit code based, not
+ # boolean logic), hence when the script exits 0 if no package was
+ # missing it is interpreted as true, thus the test did not capture
+ # the missing package
+ if ./check_missing_installations.sh; then
+ echo "did NOT capture missing package; test FAILED"
+ exit 1
+ else
+ echo "captured missing package; test PASSED"
+ exit 0
+ fi
diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh
index 2830754b29..419ce586e8 100755
--- a/EESSI-pilot-install-software.sh
+++ b/EESSI-pilot-install-software.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-#
+
# Script to install EESSI pilot software stack (version set through init/eessi_defaults)
# see example parsing of command line arguments at
@@ -137,7 +137,11 @@ else
echo_green ">> MODULEPATH set up: ${MODULEPATH}"
fi
-REQ_EB_VERSION='4.5.0'
+echo "EESSI_CVMFS_REPO=${EESSI_CVMFS_REPO}"
+
+REQ_EB_VERSION='4.7.1'
+echo "REQ_EB_VERSION=${REQ_EB_VERSION}"
+module avail 2>&1 | grep -i easybuild
echo ">> Checking for EasyBuild module..."
ml_av_easybuild_out=$TMPDIR/ml_av_easybuild.out
@@ -165,10 +169,6 @@ else
eb --install-latest-eb-release &> ${eb_install_out}
check_exit_code $? "${ok_msg}" "${fail_msg}"
- # restore origin $PATH and $PYTHONPATH values
- export PATH=${ORIG_PATH}
- export PYTHONPATH=${ORIG_PYTHONPATH}
-
eb --search EasyBuild-${REQ_EB_VERSION}.eb | grep EasyBuild-${REQ_EB_VERSION}.eb > /dev/null
if [[ $? -eq 0 ]]; then
ok_msg="EasyBuild v${REQ_EB_VERSION} installed, alright!"
@@ -177,6 +177,10 @@ else
check_exit_code $? "${ok_msg}" "${fail_msg}"
fi
+ # restore origin $PATH and $PYTHONPATH values
+ export PATH=${ORIG_PATH}
+ export PYTHONPATH=${ORIG_PYTHONPATH}
+
module avail easybuild/${REQ_EB_VERSION} &> ${ml_av_easybuild_out}
if [[ $? -eq 0 ]]; then
echo_green ">> EasyBuild module installed!"
@@ -206,232 +210,27 @@ fi
echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..."
-# install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up,
-# see https://github.com/EESSI/software-layer/issues/123
-# and https://github.com/easybuilders/easybuild-easyblocks/pull/2557
-ok_msg="Java installed, off to a good (?) start!"
-fail_msg="Failed to install Java, woopsie..."
-$EB Java-11.eb --robot --include-easyblocks-from-pr 2557
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# install GCC for foss/2020a
-export GCC_EC="GCC-9.3.0.eb"
-echo ">> Starting slow with ${GCC_EC}..."
-ok_msg="${GCC_EC} installed, yippy! Off to a good start..."
-fail_msg="Installation of ${GCC_EC} failed!"
-# pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453,
-# which includes patch to fix build of GCC 9.3 when recent kernel headers are in place
-$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# install CMake with custom easyblock that patches CMake when --sysroot is used
-echo ">> Install CMake with fixed easyblock to take into account --sysroot"
-ok_msg="CMake installed!"
-fail_msg="Installation of CMake failed, what the ..."
-$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
+# add OpenBLAS: requires special handling for GENERIC CPU targets
# If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946
-echo ">> Installing OpenBLAS..."
-ok_msg="Done with OpenBLAS!"
-fail_msg="Installation of OpenBLAS failed!"
+# we need to specify --parallel 1 or run into weird linking issues supposedly
+# introduced by GNU make v4.4, see for example
+# https://github.com/xianyi/OpenBLAS/issues/3899 and https://github.com/xianyi/OpenBLAS/issues/3979
+# a fix may be provided with https://github.com/xianyi/OpenBLAS/pull/3983
+# the fix did not work, so only option seems to be --parallel 1
+# another option seems to be to use make v4.3 as build dependency
if [[ $GENERIC -eq 1 ]]; then
+ echo ">> Installing OpenBLAS..."
+ ok_msg="Done with OpenBLAS (GENERIC architecture)!"
+ fail_msg="Installation of OpenBLAS (GENERIC architecture) failed!"
echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS."
- openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946"
-else
- openblas_include_easyblocks_from_pr=''
-fi
-$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing OpenMPI..."
-ok_msg="OpenMPI installed, w00!"
-fail_msg="Installation of OpenMPI failed, that's not good..."
-$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# install Python
-echo ">> Install Python 2.7.18 and Python 3.8.2..."
-ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!"
-fail_msg="Installation of Python failed, oh no..."
-$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Perl..."
-ok_msg="Perl installed, making progress..."
-fail_msg="Installation of Perl failed, this never happens..."
-# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640
-# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...)
-$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Qt5..."
-ok_msg="Qt5 installed, phieuw, that was a big one!"
-fail_msg="Installation of Qt5 failed, that's frustrating..."
-$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# skip test step when installing SciPy-bundle on aarch64,
-# to dance around problem with broken numpy tests;
-# cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959
-echo ">> Installing SciPy-bundle"
-ok_msg="SciPy-bundle installed, yihaa!"
-fail_msg="SciPy-bundle installation failed, bummer..."
-SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb
-if [[ "$(uname -m)" == "aarch64" ]]; then
- $EB $SCIPY_EC --robot --skip-test-step
-else
- $EB $SCIPY_EC --robot
-fi
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing GROMACS..."
-ok_msg="GROMACS installed, wow!"
-fail_msg="Installation of GROMACS failed, damned..."
-$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-# note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)!
-# 32GB is sufficient to build with 16 cores
-echo ">> Installing OpenFOAM (twice!)..."
-ok_msg="OpenFOAM installed, now we're talking!"
-fail_msg="Installation of OpenFOAM failed, we were so close..."
-$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then
- echo ">> Installing QuantumESPRESSO..."
- ok_msg="QuantumESPRESSO installed, let's go quantum!"
- fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!"
- $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot
+ $EB --include-easyblocks easyblocks/o/openblas-pr1946-cc74e45.py OpenBLAS-0.3.15-GCC-10.3.0.eb --robot
check_exit_code $? "${ok_msg}" "${fail_msg}"
fi
-echo ">> Installing R 4.0.0 (better be patient)..."
-ok_msg="R installed, wow!"
-fail_msg="Installation of R failed, so sad..."
-$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Bioconductor 3.11 bundle..."
-ok_msg="Bioconductor installed, enjoy!"
-fail_msg="Installation of Bioconductor failed, that's annoying..."
-$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing TensorFlow 2.3.1..."
-ok_msg="TensorFlow 2.3.1 installed, w00!"
-fail_msg="Installation of TensorFlow failed, why am I not surprised..."
-$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Horovod 0.21.3..."
-ok_msg="Horovod installed! Go do some parallel training!"
-fail_msg="Horovod installation failed. There comes the headache..."
-$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then
-
- echo ">> Installing code-server 3.7.3..."
- ok_msg="code-server 3.7.3 installed, now you can use VS Code!"
- fail_msg="Installation of code-server failed, that's going to be hard to fix..."
- $EB code-server-3.7.3.eb --robot
- check_exit_code $? "${ok_msg}" "${fail_msg}"
-fi
+eb_install_easystack_out=${TMPDIR}/eb_install_easystack.out
+${EB:-eb} --robot --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental | tee ${eb_install_easystack_out}
-echo ">> Installing RStudio-Server 1.3.1093..."
-ok_msg="RStudio-Server installed, enjoy!"
-fail_msg="Installation of RStudio-Server failed, might be OS deps..."
-$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing OSU-Micro-Benchmarks 5.6.3..."
-ok_msg="OSU-Micro-Benchmarks installed, yihaa!"
-fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..."
-$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Spark 3.1.1..."
-ok_msg="Spark installed, set off the fireworks!"
-fail_msg="Installation of Spark failed, no fireworks this time..."
-$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing IPython 7.15.0..."
-ok_msg="IPython installed, launch your Jupyter Notebooks!"
-fail_msg="Installation of IPython failed, that's unexpected..."
-$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing WRF 3.9.1.1..."
-ok_msg="WRF installed, it's getting hot in here!"
-fail_msg="Installation of WRF failed, that's unexpected..."
-OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing R 4.1.0 (better be patient)..."
-ok_msg="R installed, wow!"
-fail_msg="Installation of R failed, so sad..."
-$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing Nextflow 22.10.1..."
-ok_msg="Nextflow installed, the work must flow..."
-fail_msg="Installation of Nextflow failed, that's unexpected..."
-$EB -r --from-pr 16531 Nextflow-22.10.1.eb
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..."
-ok_msg="OSU-Micro-Benchmarks installed, yihaa!"
-fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..."
-$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-echo ">> Installing EasyBuild 4.5.1..."
-ok_msg="EasyBuild v4.5.1 installed"
-fail_msg="EasyBuild v4.5.1 failed to install"
-$EB --from-pr 14545 --include-easyblocks-from-pr 2805
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1
-check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1"
-
-echo ">> Installing SciPy-bundle with foss/2021a..."
-ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age"
-fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..."
-# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454
-# which includes patch to fix installation with recent Linux kernel headers
-$EB --from-pr 14454 GCCcore-10.3.0.eb --robot
-# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640
-# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...)
-$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640
-# use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set
-# from https://github.com/easybuilders/easybuild-easyblocks/pull/2248
-$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248
-# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584
-# that includes patch to fix bootstrap problem when using alternate sysroot
-$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot
-# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885
-# which includes a patch to fix installation on POWER
-$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot
-# ignore failing FlexiBLAS tests when building on POWER;
-# some tests are failing due to a segmentation fault due to "invalid memory reference",
-# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476;
-# using -fstack-protector-strong -fstack-clash-protection should fix that,
-# but it doesn't for some reason when building for ppc64le/generic...
-if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then
- $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure
-else
- $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb
-fi
-
-$EB SciPy-bundle-2021.05-foss-2021a.eb --robot
-check_exit_code $? "${ok_msg}" "${fail_msg}"
-
-### add packages here
-
-echo ">> Creating/updating Lmod cache..."
+echo ">> Creating/updating Lmod cache on $(date) (nr 1) ..."
export LMOD_RC="${EASYBUILD_INSTALLPATH}/.lmod/lmodrc.lua"
if [ ! -f $LMOD_RC ]; then
python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH}
@@ -442,5 +241,9 @@ $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH}
$TOPDIR/check_missing_installations.sh
+# TODO do not clean up by default (or provide a cmd line arg and when run
+# from bot, do not clean up)
echo ">> Cleaning up ${TMPDIR}..."
rm -r ${TMPDIR}
+
+exit 0
diff --git a/OpenBLAS-0.3.15-GCC-10.3.0.eb b/OpenBLAS-0.3.15-GCC-10.3.0.eb
new file mode 100644
index 0000000000..13b731bc6d
--- /dev/null
+++ b/OpenBLAS-0.3.15-GCC-10.3.0.eb
@@ -0,0 +1,39 @@
+name = 'OpenBLAS'
+version = '0.3.15'
+
+homepage = 'https://xianyi.github.com/OpenBLAS/'
+description = "OpenBLAS is an optimized BLAS library based on GotoBLAS2 1.13 BSD version."
+
+toolchain = {'name': 'GCC', 'version': '10.3.0'}
+
+source_urls = [
+ # order matters, trying to download the large.tgz/timing.tgz LAPACK tarballs from GitHub causes trouble
+ 'https://www.netlib.org/lapack/timing/',
+ 'https://github.com/xianyi/OpenBLAS/archive/',
+]
+sources = ['v%(version)s.tar.gz']
+patches = [
+ ('large.tgz', '.'),
+ ('timing.tgz', '.'),
+ 'OpenBLAS-%(version)s_icelake-detection.patch',
+ 'OpenBLAS-%(version)s_fix-aarch64.patch',
+ 'OpenBLAS-%(version)s_workaround-gcc-miscompilation.patch',
+]
+checksums = [
+ '30a99dec977594b387a17f49904523e6bc8dd88bd247266e83485803759e4bbe', # v0.3.15.tar.gz
+ 'f328d88b7fa97722f271d7d0cfea1c220e0f8e5ed5ff01d8ef1eb51d6f4243a1', # large.tgz
+ '999c65f8ea8bd4eac7f1c7f3463d4946917afd20a997807300fe35d70122f3af', # timing.tgz
+ 'aa96e1c3b1532dbafe4b6e8591688ef200e99304a902851f342cb9d467219762', # OpenBLAS-0.3.15_icelake-detection.patch
+ 'ba7bd45af9fe0516f8c9b1cf047eec7b833996e481bc925d4fb2563865d3db38', # OpenBLAS-0.3.15_fix-aarch64.patch
+ # OpenBLAS-0.3.15_workaround-gcc-miscompilation.patch
+ 'e6b326fb8c4a8a6fd07741d9983c37a72c55c9ff9a4f74a80e1352ce5f975971',
+]
+
+builddependencies = [
+ ('make', '4.3'),
+]
+
+# extensive testing can be enabled by uncommenting the line below
+# runtest = 'PATH=.:$PATH lapack-timing'
+
+moduleclass = 'numlib'
diff --git a/bot/check-result.sh b/bot/check-result.sh
new file mode 100755
index 0000000000..8774d68f68
--- /dev/null
+++ b/bot/check-result.sh
@@ -0,0 +1,277 @@
+#!/bin/bash
+#
+# Script to check the result of building the EESSI software layer.
+# Intended use is that it is called by a (batch) job running on a compute
+# node.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# result cases
+
+# - SUCCESS (all of)
+# - working directory contains slurm-JOBID.out file
+# - working directory contains eessi*tar.gz
+# - no message ERROR
+# - no message FAILED
+# - no message ' required modules missing:'
+# - one or more of 'No missing modules!'
+# - message regarding created tarball
+# - FAILED (one of ... implemented as NOT SUCCESS)
+# - no slurm-JOBID.out file
+# - no tarball
+# - message with ERROR
+# - message with FAILED
+# - message with ' required modules missing:'
+# - no message regarding created tarball
+
+# stop as soon as something fails
+# set -e
+
+TOPDIR=$(dirname $(realpath $0))
+
+source ${TOPDIR}/../scripts/utils.sh
+source ${TOPDIR}/../scripts/cfg_files.sh
+
+display_help() {
+ echo "usage: $0 [OPTIONS]"
+ echo " OPTIONS:"
+ echo " -h | --help - display this usage information [default: false]"
+ echo " -v | --verbose - display more information [default: false]"
+}
+
+# set defaults for command line arguments
+VERBOSE=0
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+ case $1 in
+ -h|--help)
+ display_help
+ exit 0
+ ;;
+ -v|--verbose)
+ VERBOSE=1
+ shift 1
+ ;;
+ --)
+ shift
+ POSITIONAL_ARGS+=("$@") # save positional args
+ break
+ ;;
+ -*|--*)
+ fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}"
+ ;;
+ *) # No more options
+ POSITIONAL_ARGS+=("$1") # save positional arg
+ shift
+ ;;
+ esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+job_dir=${PWD}
+
+[[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}"
+
+GP_slurm_out="slurm-${SLURM_JOB_ID}.out"
+job_out=$(ls ${job_dir} | grep "${GP_slurm_out}")
+[[ $? -eq 0 ]] && SLURM=1 || SLURM=0
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${GP_slurm_out}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'"
+
+GP_error='ERROR: '
+grep_out=$(grep "${GP_error}" ${job_dir}/${job_out})
+[[ $? -eq 0 ]] && ERROR=1 || ERROR=0
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+
+GP_failed='FAILED: '
+grep_out=$(grep "${GP_failed}" ${job_dir}/${job_out})
+[[ $? -eq 0 ]] && FAILED=1 || FAILED=0
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+
+GP_req_missing=' required modules missing:'
+grep_out=$(grep "${GP_req_missing}" ${job_dir}/${job_out})
+[[ $? -eq 0 ]] && MISSING=1 || MISSING=0
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+
+GP_no_missing='No missing modules!'
+grep_out=$(grep "${GP_no_missing}" ${job_dir}/${job_out})
+[[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+
+GP_tgz_created="tar.gz created!"
+TARBALL=
+grep_out=$(grep "${GP_tgz_created}" ${job_dir}/${job_out})
+if [[ $? -eq 0 ]]; then
+ TGZ=1
+ TARBALL=$(echo ${grep_out} | sed -e 's@^.*\(eessi[^/ ]*\) .*$@\1@')
+else
+ TGZ=0
+fi
+# have to be careful to not add searched for pattern into slurm out file
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'"
+[[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+
+[[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}"
+[[ ${VERBOSE} -ne 0 ]] && echo " test name : result (expected result)"
+[[ ${VERBOSE} -ne 0 ]] && echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)"
+[[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)"
+[[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)"
+[[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
+[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
+
+job_result_file=_bot_job${SLURM_JOB_ID}.result
+
+if [[ ${SLURM} -eq 1 ]] && \
+ [[ ${ERROR} -eq 0 ]] && \
+ [[ ${FAILED} -eq 0 ]] && \
+ [[ ${MISSING} -eq 0 ]] && \
+ [[ ${NO_MISSING} -eq 1 ]] && \
+ [[ ${TGZ} -eq 1 ]] && \
+ [[ ! -z ${TARBALL} ]]; then
+ # SUCCESS
+ summary=":grin: SUCCESS"
+ status="success"
+else
+ # FAILURE
+ summary=":cry: FAILURE"
+ status="failure"
+fi
+
+### Example details/descriptions
+#
+# :cry: FAILURE _(click triangle for detailed information)_
Details:
:heavy_check_mark: job output file slurm-470503.out
:heavy_multiplication_x: found message matching ERROR:
:heavy_multiplication_x: found message matching FAILED:
:heavy_multiplication_x: found message matching required modules missing:
:heavy_check_mark: found message(s) matching No missing modules!
:heavy_check_mark: found message matching tar.gz created!
Artefacts:eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz:grin: SUCCESS _(click triangle for detailed information)_
Details:
:heavy_check_mark: job output file slurm-470503.out
:heavy_check_mark: found message matching ERROR:
:heavy_check_mark: found message matching FAILED:
:heavy_check_mark: found message matching required modules missing:
:heavy_check_mark: found message(s) matching No missing modules!
:heavy_check_mark: found message matching tar.gz created!
Artefacts:eessi-2023.04-software-linux-x86_64-amd-zen2-1682384569.tar.gz__SUMMARY_FMT____DETAILS_FMT____ARTEFACTS_FMT__ "
+comment_summary_fmt="__SUMMARY__ _(click triangle for detailed information)_"
+comment_details_fmt="Details:
__DETAILS_LIST__"
+comment_artefacts_fmt="Artefacts:
__ARTEFACTS_LIST__"
+comment_success_item_fmt=":heavy_check_mark: __ITEM__"
+comment_failure_item_fmt=":heavy_multiplication_x: __ITEM__"
+
+function print_br_item() {
+ format="${1}"
+ item="${2}"
+ echo -n " ${format//__ITEM__/${item}}
"
+}
+
+function print_list_item() {
+ format="${1}"
+ item="${2}"
+ echo -n "${format//__ITEM__/${item}}"
+}
+
+function success() {
+ format="${comment_success_item_fmt}"
+ item="$1"
+ print_br_item "${format}" "${item}"
+}
+
+function failure() {
+ format="${comment_failure_item_fmt}"
+ item="$1"
+ print_br_item "${format}" "${item}"
+}
+
+function add_detail() {
+ actual=${1}
+ expected=${2}
+ success_msg="${3}"
+ failure_msg="${4}"
+ if [[ ${actual} -eq ${expected} ]]; then
+ success "${success_msg}"
+ else
+ failure "${failure_msg}"
+ fi
+}
+
+echo "[RESULT]" > ${job_result_file}
+echo -n "comment_description = " >> ${job_result_file}
+
+# construct values for placeholders in comment_template:
+# - __SUMMARY_FMT__ -> variable $comment_summary
+# - __DETAILS_FMT__ -> variable $comment_details
+# - __ARTEFACTS_FMT__ -> variable $comment_artefacts
+
+comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+
+# first construct comment_details_list, abbreviated CoDeList
+# then use it to set comment_details
+CoDeList=""
+
+success_msg="job output file ${job_out}"
+failure_msg="no job output file matching ${GP_slurm_out}"
+CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching ${GP_error}"
+failure_msg="found message matching ${GP_error}"
+CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching ${GP_failed}"
+failure_msg="found message matching ${GP_failed}"
+CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching ${GP_req_missing}"
+failure_msg="found message matching ${GP_req_missing}"
+CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="found message(s) matching ${GP_no_missing}"
+failure_msg="no message matching ${GP_no_missing}"
+CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+
+success_msg="found message matching ${GP_tgz_created}"
+failure_msg="no message matching ${GP_tgz_created}"
+CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+
+
+# first construct comment_artefacts_list, abbreviated CoArList
+# then use it to set comment_artefacts
+CoArList=""
+
+# TARBALL should only contain a single tarball
+if [[ ! -z ${TARBALL} ]]; then
+ CoArList=${CoArList}$(print_list_item '__ITEM__' "${TARBALL}")
+else
+ CoArList=${CoArList}$(print_list_item 'No artefacts were created/found.' '')
+fi
+comment_artefacts=${comment_artefacts_fmt/__ARTEFACTS_LIST__/${CoArList}}
+
+# now put all pieces together creating comment_details from comment_template
+comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
+comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
+comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}}
+
+echo "${comment_description}" >> ${job_result_file}
+
+# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts
+# - this should make use of subsequent steps such as deploying a tarball more
+# efficient
+echo "status = ${status}" >> ${job_result_file}
+echo "artefacts = " >> ${job_result_file}
+echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file}
+
+exit 0
diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index e4c75aebd8..945bf32b19 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -3,6 +3,7 @@
# Script to check for missing installations in EESSI pilot software stack (version 2021.12)
#
# author: Kenneth Hoste (@boegel)
+# author: Thomas Roeblitz (@trz42)
#
# license: GPLv2
#
@@ -21,11 +22,34 @@ source $TOPDIR/scripts/utils.sh
source $TOPDIR/configure_easybuild
echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
-ok_msg="No missing installations, party time!"
-fail_msg="On no, some installations are still missing, how did that happen?!"
eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used
# PR #16531: Nextflow-22.10.1.eb
-${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing | tee ${eb_missing_out}
-grep "No missing modules" ${eb_missing_out} > /dev/null
-check_exit_code $? "${ok_msg}" "${fail_msg}"
+# ${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out}
+# PR 16531 not needed since we use EB v4.7.0
+# this, however, breaks the GHA https://github.com/NorESSI/software-layer/blob/main/.github/workflows/test_eessi.yml
+# because it uses the EESSI pilot which only provides EB 4.5.1, so adding it back
+${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out}
+exit_code=${PIPESTATUS[0]}
+
+ok_msg="Command 'eb --missing ...' succeeded, analysing output..."
+fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'"
+check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}"
+
+# the above assesses the installed software for each easyconfig provided in
+# the easystack file and then print messages such as
+# `No missing modules!`
+# or
+# `2 out of 3 required modules missing:`
+# depending on the result of the assessment. Hence, we need to check if the
+# output does not contain any line with ` required modules missing:`
+
+grep " required modules missing:" ${eb_missing_out} > /dev/null
+exit_code=$?
+
+# if grep returns 1 (` required modules missing:` was NOT found), we set
+# MODULES_MISSING to 0, otherwise (it was found or another error) we set it to 1
+[[ ${exit_code} -eq 1 ]] && MODULES_MISSING=0 || MODULES_MISSING=1
+ok_msg="No missing installations, party time!"
+fail_msg="On no, some installations are still missing, how did that happen?!"
+check_exit_code ${MODULES_MISSING} "${ok_msg}" "${fail_msg}"
diff --git a/create_tarball.sh b/create_tarball.sh
index 56ac8ab7ad..b6c72b341d 100755
--- a/create_tarball.sh
+++ b/create_tarball.sh
@@ -38,6 +38,7 @@ cd ${overlay_upper_dir}/versions/
echo ">> Collecting list of files/directories to include in tarball via ${PWD}..."
files_list=${tmpdir}/files.list.txt
+module_files_list=${tmpdir}/module_files.list.txt
if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
# include Lmod cache and configuration file (lmodrc.lua),
@@ -49,12 +50,31 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list}
# module symlinks
find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
+ # module files and symlinks
+ find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \
+ | grep -v '/\.wh\.' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \
+ >> ${module_files_list}
fi
-if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software ]; then
- # installation directories
- ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/*/* | grep -v '/\.wh\.' >> ${files_list}
+if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then
+ # installation directories but only those for which module files were created
+ # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua'
+ # using EasyBuild's standard module naming scheme) match the name of the
+ # software installation directory (expected to be 'PACKAGE_NAME/VERSION/').
+ # If either side changes (module naming scheme or naming of software
+ # installation directories), the procedure will likely not work.
+ for package_version in $(cat ${module_files_list}); do
+ echo "handling ${package_version}"
+ ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \
+ | grep -v '/\.wh\.' >> ${files_list}
+ done
fi
+# add a bit debug output
+echo "wrote file list to ${files_list}"
+[ -r ${files_list} ] && cat ${files_list}
+echo "wrote module file list to ${module_files_list}"
+[ -r ${module_files_list} ] && cat ${module_files_list}
+
topdir=${cvmfs_repo}/versions/
echo ">> Creating tarball ${target_tgz} from ${topdir}..."
diff --git a/easyblocks/o/openblas-pr1946-cc74e45.py b/easyblocks/o/openblas-pr1946-cc74e45.py
new file mode 100644
index 0000000000..defd5d0691
--- /dev/null
+++ b/easyblocks/o/openblas-pr1946-cc74e45.py
@@ -0,0 +1,299 @@
+"""
+EasyBuild support for building and installing OpenBLAS, implemented as an easyblock
+
+@author: Andrew Edmondson (University of Birmingham)
+@author: Alex Domingo (Vrije Universiteit Brussel)
+@author: Terje Kvernes (University of Oslo)
+"""
+import os
+import re
+from distutils.version import LooseVersion
+from easybuild.base import fancylogger
+from easybuild.easyblocks.generic.configuremake import ConfigureMake
+from easybuild.framework.easyconfig import CUSTOM
+from easybuild.tools.build_log import EasyBuildError, print_warning
+from easybuild.tools.config import ERROR, build_option
+from easybuild.tools.filetools import read_file
+from easybuild.tools.systemtools import POWER, get_cpu_architecture, get_shared_lib_ext
+from easybuild.tools.run import run_cmd, check_log_for_errors
+
+_log = fancylogger.getLogger('systemtools', fname=False)
+
+try:
+ from archspec import cpu as archspec_cpu
+ HAVE_ARCHSPEC = True
+except ImportError as err:
+ _log.debug("Failed to import 'archspec' Python module: %s", err)
+ HAVE_ARCHSPEC = False
+
+TARGET = 'TARGET'
+
+
+class EB_OpenBLAS(ConfigureMake):
+ """Support for building/installing OpenBLAS."""
+
+ @staticmethod
+ def extra_options():
+ """Custom easyconfig parameters for OpenBLAS"""
+ extra_vars = {
+ 'targetfile': ['TargetList.txt', "File containing OpenBLAS target list", CUSTOM],
+ }
+ return ConfigureMake.extra_options(extra_vars)
+
+ def configure_step(self):
+ """ set up some options - but no configure command to run"""
+
+ default_opts = {
+ 'BINARY': '64',
+ 'CC': os.getenv('CC'),
+ 'FC': os.getenv('FC'),
+ 'USE_OPENMP': '1',
+ 'USE_THREAD': '1',
+ }
+
+ # Handle the possibility of setting the target architecture as dynamic,
+ # where OpenBLAS will optimize the kernel at runtime.
+ self._dynamic_target = False
+
+ compiler_optarch = self._optarch_for_compiler(build_option('optarch'))
+
+ # Retain the (m)arch part of the optarch settings across the entire object.
+ self._optarch_architecture = compiler_optarch
+
+ if '%s=' % TARGET in self.cfg['buildopts']:
+ # Add any TARGET in buildopts to default_opts, so it is passed to testopts and installopts
+ for buildopt in self.cfg['buildopts'].split():
+ optpair = buildopt.split('=')
+ if optpair[0] == TARGET:
+ default_opts[optpair[0]] = optpair[1]
+ elif LooseVersion(self.version) < LooseVersion('0.3.6') and get_cpu_architecture() == POWER:
+ # There doesn't seem to be a POWER9 option yet, but POWER8 should work.
+ print_warning("OpenBLAS 0.3.5 and lower have known issues on POWER systems")
+ default_opts[TARGET] = 'POWER8'
+ elif compiler_optarch:
+ compiler_family = self.toolchain.comp_family()
+ self.log.info("EasyBuild full optarch requested for %s: %s" % (compiler_family, compiler_optarch))
+ optarch_as_target = self._parse_optarch(compiler_optarch)
+ mapped_target = None
+
+ if optarch_as_target:
+ # Note that _parse_optarch returns lowercased results, so GENERIC has become 'generic'.
+ if optarch_as_target == 'generic':
+ self._set_dynamic_architecture(default_opts)
+ mapped_target = 'generic'
+ else:
+ self.log.info("EasyBuild march: %s" % optarch_as_target)
+ openblas_targets = self._get_openblas_targets(self.cfg['targetfile'])
+ mapped_target = self._get_mapped_target(optarch_as_target, openblas_targets)
+ else:
+ self.log.info("Optarch specified for %s, but no march detected", compiler_family)
+
+ if mapped_target is None:
+ print_warning("optarch for %s given as '%s'\n"
+ "EasyBuild was unable to map this to an equivalent OpenBLAS target!\n"
+ "OpenBLAS will be built to optimize its kernel at runtime!\n"
+ % (self.toolchain.comp_family(), compiler_optarch))
+ self.log.warning("Unable to map %s to an OpenBLAS target, falling back to runtime optimization."
+ % compiler_optarch)
+ self._set_dynamic_architecture(default_opts)
+ elif mapped_target == 'generic':
+ self.log.info("Optarch 'GENERIC' requested, will enable runtime optimization.")
+ else:
+ mapped_target = mapped_target.upper()
+ self.log.info("Optarch mapped between EasyBuild and OpenBLAS to: " + mapped_target)
+ default_opts[TARGET] = mapped_target
+
+ for key in sorted(default_opts.keys()):
+ for opts_key in ['buildopts', 'testopts', 'installopts']:
+ if '%s=' % key not in self.cfg[opts_key]:
+ self.cfg.update(opts_key, "%s='%s'" % (key, default_opts[key]))
+
+ self.cfg.update('installopts', 'PREFIX=%s' % self.installdir)
+
+ def build_step(self):
+ """ Custom build step excluding the tests """
+
+ # Equivalent to `make all` without the tests
+ build_parts = ['libs', 'netlib']
+ for buildopt in self.cfg['buildopts'].split():
+ if 'BUILD_RELAPACK' in buildopt and '1' in buildopt:
+ build_parts += ['re_lapack']
+ build_parts += ['shared']
+
+ # If we're doing either a dynamic build or utilizing optarch,
+ # strip march from all environment variables except the EBVAR-prefixed ones.
+ # For dynamic builds we should ignore optarch completely and for optarch-set builds
+ # we need to adhere to TARGET and not march.
+ if self._dynamic_target is True or self._optarch_architecture is True:
+ self.log.info('Dynamic build requested, stripping march settings from environment variables')
+ for k in os.environ.keys():
+ optarch_to_strip = '-' + self._optarch_architecture
+ if 'EBVAR' not in k and self._optarch_architecture in os.environ[k]:
+ os.environ[k] = os.environ[k].replace(optarch_to_strip, '')
+
+ # Pass CFLAGS through command line to avoid redefinitions (issue xianyi/OpenBLAS#818)
+ cflags = 'CFLAGS'
+ if os.environ[cflags]:
+ self.cfg.update('buildopts', "%s='%s'" % (cflags, os.environ[cflags]))
+ del os.environ[cflags]
+ self.log.info("Environment variable %s unset and passed through command line" % cflags)
+
+ makecmd = 'make'
+ if self.cfg['parallel']:
+ makecmd += ' -j %s' % self.cfg['parallel']
+
+ cmd = ' '.join([self.cfg['prebuildopts'], makecmd, ' '.join(build_parts), self.cfg['buildopts']])
+ run_cmd(cmd, log_all=True, simple=True)
+
+ def test_step(self):
+ """ Mandatory test step plus optional runtest """
+
+ run_tests = ['tests']
+ if self.cfg['runtest']:
+ run_tests += [self.cfg['runtest']]
+
+ for runtest in run_tests:
+ cmd = "%s make %s %s" % (self.cfg['pretestopts'], runtest, self.cfg['testopts'])
+ (out, _) = run_cmd(cmd, log_all=True, simple=False, regexp=False)
+
+ # Raise an error if any test failed
+ check_log_for_errors(out, [('FATAL ERROR', ERROR)])
+
+ def sanity_check_step(self):
+ """ Custom sanity check for OpenBLAS """
+ custom_paths = {
+ 'files': ['include/cblas.h', 'include/f77blas.h', 'include/lapacke_config.h', 'include/lapacke.h',
+ 'include/lapacke_mangling.h', 'include/lapacke_utils.h', 'include/openblas_config.h',
+ 'lib/libopenblas.a', 'lib/libopenblas.%s' % get_shared_lib_ext()],
+ 'dirs': [],
+ }
+ super(EB_OpenBLAS, self).sanity_check_step(custom_paths=custom_paths)
+
+ def _optarch_for_compiler(self, optarch):
+ """
+ Extracts the appropriate optarch for the compiler currently being used.
+ If it is not compiler-specific it is returned as-is.
+ If no optarch is found, False is returned.
+ :param optarch: A complete optarch statement.
+ https://easybuild.readthedocs.io/en/latest/Controlling_compiler_optimization_flags.html
+ """
+ if optarch is False:
+ return False
+
+ compiler = self.toolchain.comp_family()
+ compiler_specific_optarch_string = ''
+
+ if type(optarch) == str:
+ compiler_specific_optarch_string = optarch
+ elif type(optarch) == dict:
+ if compiler in optarch:
+ compiler_specific_optarch_string = optarch[compiler]
+ else:
+ raise EasyBuildError("optarch in an unexpected format: '%s'" % type(optarch)).__class__.__name__
+
+ return compiler_specific_optarch_string
+
+ def _parse_optarch(self, compiler_optarch):
+ """
+ Pick the march out of a given optarch.
+ Note that the result is lowercased.
+ :param compiler_optarch: An optarch for a given compiler.
+ https://easybuild.readthedocs.io/en/latest/Controlling_compiler_optimization_flags.html
+ """
+
+ target_arch = ''
+ pieces = compiler_optarch.split()
+
+ for piece in pieces:
+ spec = piece.split('=')
+ if spec[0] == 'march' or spec[0] == '-march':
+ target_arch = spec[1]
+
+ return target_arch.lower()
+
+ def _get_openblas_targets(self, targetfile):
+ """
+ Parse the openblas target file and generate a list of targets.
+ :param targetfile: A file with OpenBLAS targets.
+ """
+ targets = []
+
+ if os.path.isfile(targetfile):
+ # Assumption, the OpenBLAS TargetList.txt has one target per line and that
+ # single words on a line is a target if they match a simple regexp...
+ re_target = re.compile(r'^[A-Z0-9_]+$')
+ for line in read_file(targetfile).splitlines():
+ match = re_target.match(line)
+ if match is not None:
+ targets.append(line.strip().lower())
+ else:
+ print_warning("Unable to find OpenBLAS targetfile '%s'" % os.path.realpath(targetfile))
+
+ return targets
+
+ def _set_dynamic_architecture(self, default_opts):
+ """
+ Sets the DYNAMIC_ARCH option for OpenBLAS, building a library that chooses
+ an optimized kernel at runtime. Also removes any previous TARGET setting, if any.
+ """
+ default_opts['DYNAMIC_ARCH'] = 1
+ default_opts.pop(TARGET, None)
+ self._dynamic_target = True
+
+ def _get_mapped_target(self, march, openblas_targets):
+ """
+ Attempts to match the given march in the list of openblas targets.
+ If archspec is installed, will try to match directly or follow ancestors for
+ architectures that will work.
+ Returns None if no target was found.
+ """
+
+ result = None
+
+ if HAVE_ARCHSPEC:
+ self.log.info("Using archspec to match optarch to openblas targets.")
+
+ openblas_arch = set(['alpha', 'arm', 'ia64', 'mips', 'mips64',
+ 'power', 'sparc', 'zarch'])
+ openblas_arch_map = {
+ 'amd64': 'x86_64',
+ 'powerpc64': 'power',
+ 'i386': 'x86',
+ 'aarch64': 'arm64',
+ }
+ openblas_arch.update(openblas_arch_map.keys())
+ openblas_arch.update(openblas_arch_map.values())
+
+ skylake = set(["skylake"])
+ available_targets = set(openblas_targets) | skylake | openblas_arch
+
+ try:
+ uarch = archspec_cpu.TARGETS[march]
+ except KeyError:
+ warning_string = "Archspec was asked to find '" + march + "' as an architecture, but failed!"
+ print_warning(warning_string)
+ self.log.warning(warning_string)
+ return None
+
+ if uarch.name in available_targets:
+ result = uarch.name
+ else:
+ self.log.info("No direct match for '" + march + "' between archspec and OpenBLAS, traversing ancestry.")
+ for uarch in uarch.ancestors:
+ if uarch.name in available_targets:
+ self.log.info("Ancestral match between '" + march + "' and '" + uarch.name + "'.")
+ result = uarch.name
+ break
+
+ # Skylake for OpenBLAS is called 'skylakex'. Handle this exception exceptionally.
+ if result == 'skylake':
+ result = 'skylakex'
+
+ else:
+ self.log.info("Unable to find archspec, optarch matching will be poor.")
+ if march == 'skylake':
+ result = 'skylakex'
+ elif march in openblas_targets:
+ result = march
+
+ return result
diff --git a/eb_hooks.py b/eb_hooks.py
index df7742f999..7f3f53a6c0 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -50,6 +50,10 @@ def parse_hook(ec, *args, **kwargs):
# determine path to Prefix installation in compat layer via $EPREFIX
eprefix = get_eessi_envvar('EPREFIX')
+
+ # always replace Rust/1.52.1 with Rust/1.60.0
+ Rust_ver_replace(ec, eprefix)
+
if ec.name in PARSE_HOOKS:
PARSE_HOOKS[ec.name](ec, eprefix)
@@ -181,6 +185,35 @@ def wrf_preconfigure(self, *args, **kwargs):
raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!")
+def Rust_ver_replace(ec, eprefix):
+ """When using the new compat layer, building Rust/1.52.1 fails while Rust/1.60.0 succeeds ,the goal is to replace
+ Rust/1.52.1 when found as dependency/hiddendependency/buildependency by Rust/1.60.0 while building software"""
+ for index in range(len(ec['dependencies'])):
+ dep = ec['dependencies'][index]
+ if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'):
+ print_msg("NOTE:Rust dependency version has been modified from Rust/1.52.1 --> Rust/1.60.0")
+ if isinstance(dep, list):
+ ec['dependencies'][index] = ["Rust", "1.60.0"]
+ else:
+ ec['dependencies'][index] = ("Rust", "1.60.0")
+
+ for index in range(len(ec['hiddendependencies'])):
+ dep = ec['hiddendependencies'][index]
+ if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'):
+ print_msg("NOTE:Rust hiddendependency version has been modified from Rust/1.52.1 --> Rust/1.60.0 ")
+ if isinstance(dep, list):
+ ec['hiddendependencies'][index] = ["Rust", "1.60.0"]
+ else:
+ ec['hiddendependencies'][index] = ("Rust", "1.60.0")
+ for index in range(len(ec['builddependencies'])):
+ dep = ec['builddependencies'][index]
+ if isinstance(dep, (list,tuple)) and (dep[0] == "Rust" and dep[1] == '1.52.1'):
+ print_msg("NOTE:Rust builddependency version has been modified from Rust/1.52.1 --> Rust/1.60.0")
+ if isinstance(dep, list):
+ ec['builddependencies'][index] = ["Rust", "1.60.0"]
+ else:
+ ec['builddependencies'][index] = ("Rust", "1.60.0")
+
PARSE_HOOKS = {
'CGAL': cgal_toolchainopts_precise,
'fontconfig': fontconfig_add_fonts,
diff --git a/eessi-2022.11.yml b/eessi-2022.11.yml
new file mode 100644
index 0000000000..13bf7989aa
--- /dev/null
+++ b/eessi-2022.11.yml
@@ -0,0 +1,26 @@
+easyconfigs:
+ - CMake-3.16.4-GCCcore-9.3.0.eb
+ - CMake-3.20.1-GCCcore-10.3.0.eb
+ - EasyBuild-4.6.2.eb
+ - GCC-9.3.0.eb
+ - GCC-10.3.0.eb
+ - ImageMagick-7.0.11-14-GCCcore-10.3.0.eb
+ - Java-11.eb
+ - Java-17.eb
+ - Nextflow-22.10.1.eb
+ - OpenBLAS-0.3.9-GCC-9.3.0.eb
+ - OpenMPI-4.0.3-GCC-9.3.0.eb
+ - OpenMPI-4.1.1-GCC-10.3.0.eb
+ - OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb
+ - OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb
+ - Perl-5.30.2-GCCcore-9.3.0.eb
+ - Python-2.7.18-GCCcore-9.3.0.eb
+ - Python-3.8.2-GCCcore-9.3.0.eb
+ - Python-3.9.5-GCCcore-10.3.0.eb
+ - Qt5-5.14.1-GCCcore-9.3.0.eb
+ - R-4.0.0-foss-2020a.eb
+ - SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb
+ - Spark-3.1.1-foss-2020a-Python-3.8.2.eb
+ - WRF-3.9.1.1-foss-2020a-dmpar.eb
+ - QuantumESPRESSO-6.6-foss-2020a.eb
+ - FlexiBLAS-3.0.4-GCC-10.3.0.eb
diff --git a/eessi-2023.04.yml b/eessi-2023.04.yml
new file mode 100644
index 0000000000..ff4904d01f
--- /dev/null
+++ b/eessi-2023.04.yml
@@ -0,0 +1,16 @@
+easyconfigs:
+# - GCC-9.3.0.eb
+ - GCC-10.3.0.eb:
+ options:
+ include-easyblocks-from-pr: 2921
+# - GCC-11.3.0.eb
+# - GCC-12.2.0.eb
+ - OpenSSL-1.1.eb:
+ options:
+ include-easyblocks-from-pr: 2922
+ - CMake-3.20.1-GCCcore-10.3.0.eb:
+ options:
+ include-easyblocks-from-pr: 2248
+ - Perl-5.32.1-GCCcore-10.3.0.eb
+ - OpenBLAS-0.3.15-GCC-10.3.0.eb
+ - OpenMPI-4.1.1-GCC-10.3.0.eb