From b81151ad8c03adbc4de023dbb908cacb87e2ff6f Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 17 Sep 2024 13:40:06 +0200 Subject: [PATCH 001/152] {2023.06}[2023a,zen4] LAMMPS 2Aug2023 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..df3d0dedaa --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb From 700f37abe0fc335f4b06fafe069862caa143ab0d Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 17 Sep 2024 12:07:52 +0000 Subject: [PATCH 002/152] {2023.06}[GCCcore/12.2.0-GCC/12.2.0-gompi/2022b-foss/2022b] bio-packages --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index a22b78718f..02b00e6173 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -6,4 +6,25 @@ easyconfigs: from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc - gnuplot-5.4.6-GCCcore-12.2.0.eb - h5py-3.8.0-foss-2022b.eb - - MDAnalysis-2.4.2-foss-2022b.eb + - MDAnalysis-2.4.2-foss-2022b.eb + - ncbi-vdb-3.0.5-gompi-2022b.eb + - Bio-DB-HTS-3.01-GCC-12.2.0.eb + - MAFFT-7.505-GCC-12.2.0-with-extensions.eb + - MetaEuk-6-GCC-12.2.0.eb + - BamTools-2.5.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - Mash-2.3-GCC-12.2.0.eb + - CapnProto-0.10.3-GCCcore-12.2.0.eb + - WhatsHap-2.1-foss-2022b.eb + - SAMtools-1.17-GCC-12.2.0.eb + - Bowtie2-2.5.1-GCC-12.2.0.eb + - CD-HIT-4.8.1-GCC-12.2.0.eb + - VCFtools-0.1.16-GCC-12.2.0.eb + - GenomeTools-1.6.2-GCC-12.2.0.eb + - Bio-SearchIO-hmmer-1.7.3-GCC-12.2.0.eb + - parallel-20230722-GCCcore-12.2.0.eb + - BCFtools-1.17-GCC-12.2.0.eb + - lpsolve-5.5.2.11-GCC-12.2.0.eb + - fastp-0.23.4-GCC-12.2.0.eb + - KronaTools-2.8.1-GCCcore-12.2.0.eb + - MultiQC-1.14-foss-2022b.eb From 9cced0ea972ef5570d47520006458def319ffc05 Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 17 Sep 2024 14:58:33 +0200 Subject: [PATCH 003/152] {2023.6}[2023a,a64fx] LAMMPS 2 Aug2023 for a64fx --- .../2023.06/a64fx/eessi-2023.06-eb-4.9.3-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..df3d0dedaa --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb From 48c01077a960623489c82167e0fe88edbec56daa Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 17 Sep 2024 15:34:16 +0200 Subject: [PATCH 004/152] Add ReFrame 4.6.2 --- .../2023.06/eessi-2023.06-eb-4.9.2-001-system.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml index 97c6031c79..66169b735d 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml @@ -3,3 +3,7 @@ easyconfigs: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21412 from-commit: 1cdd81524c974a29825e37bcf8ef3ccc291f5227 + - ReFrame-4.6.2.eb: + options: + from-commit: 0c4bd5c5a80f571a8932fbc38880d72455406816 + include-easyblocks-from-commit: efddeb02abe1a679324ac01ef19601dedbe79cc0 From 42df639223495e4d256ffd764b59cae8e8357857 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com> Date: Tue, 17 Sep 2024 16:21:06 +0200 Subject: [PATCH 005/152] Update easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bob Dröge --- .../2023.06/eessi-2023.06-eb-4.9.2-001-system.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml index 66169b735d..1b2343ec1f 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-001-system.yml @@ -5,5 +5,7 @@ easyconfigs: from-commit: 1cdd81524c974a29825e37bcf8ef3ccc291f5227 - ReFrame-4.6.2.eb: options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21307 from-commit: 0c4bd5c5a80f571a8932fbc38880d72455406816 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3431 include-easyblocks-from-commit: efddeb02abe1a679324ac01ef19601dedbe79cc0 From 8d0085ceaa5ea92fe75a8f3cc0140b5b5a6c4266 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Sep 2024 15:28:25 +0200 Subject: [PATCH 006/152] take into account accelerator target when configuring EasyBuild --- EESSI-install-software.sh | 12 ++++++++++++ bot/build.sh | 6 +++++- configure_easybuild | 19 +++++++++++++++++++ run_in_compat_layer_env.sh | 3 +++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 7d358e205a..27c96eba9f 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -207,7 +207,19 @@ echo ">> Setting up \$MODULEPATH..." module --force purge # ignore current $MODULEPATH entirely module unuse $MODULEPATH + +# if an accelerator target is specified, we need to make sure that the CPU-only modules are also still available +if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then + CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all + if [ -d ${CPU_ONLY_MODULES_PATH} ]; then + module use ${CPU_ONLY_MODULES_PATH} + else + fatal_error "Derived path to CPU-only modules does not exist: ${CPU_ONLY_MODULES_PATH}" + fi +fi + module use $EASYBUILD_INSTALLPATH/modules/all + if [[ -z ${MODULEPATH} ]]; then fatal_error "Failed to set up \$MODULEPATH?!" else diff --git a/bot/build.sh b/bot/build.sh index 145be740d3..7a07d09c4c 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -141,7 +141,7 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name") echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" -# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} +# determine CPU architecture to be used from entry .architecture in ${JOB_CFG_FILE} # fallbacks: # - ${CPU_TARGET} handed over from bot # - left empty to let downstream script(s) determine subdir to be used @@ -150,6 +150,10 @@ EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}} export EESSI_SOFTWARE_SUBDIR_OVERRIDE echo "bot/build.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'" +# determine accelerator target (if any) from .architecture in ${JOB_CFG_FILE} +export EESSI_ACCELERATOR_TARGET=$(cfg_get_value "architecture" "accelerator") +echo "bot/build.sh: EESSI_ACCELERATOR_TARGET='${EESSI_ACCELERATOR_TARGET}'" + # get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux) EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type") export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux} diff --git a/configure_easybuild b/configure_easybuild index ed3e651a4c..3b6d40cd96 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -1,7 +1,26 @@ +# if $WORKDIR is not defined, use a local temporary directory +if [ -z ${WORKDIR} ]; then + WORKDIR=$(mktemp -d) +fi + export EASYBUILD_PREFIX=${WORKDIR}/easybuild export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR} export EASYBUILD_SOURCEPATH=${WORKDIR}/easybuild/sources:${EESSI_SOURCEPATH} +# take into account accelerator target (if specified via $EESSI_ACCELERATOR_TARGET) +if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then + if [[ "${EESSI_ACCELERATOR_TARGET}" =~ ^nvidia/cc[0-9][0-9]$ ]]; then + # tweak path to installation directories used by EasyBuild + export EASYBUILD_INSTALLPATH=${EASYBUILD_INSTALLPATH}/accel/${EESSI_ACCELERATOR_TARGET} + # nvidia/cc80 should result in setting $EASYBUILD_CUDA_COMPUTE_CAPABILITIES to '8.0' + export EASYBUILD_CUDA_COMPUTE_CAPABILITIES=$(echo ${EESSI_ACCELERATOR_TARGET} | cut -f2 -d/ | sed 's/^cc\([0-9]\)\([0-9]\)/\1.\2/g') + else + fatal_error "Incorrect value for \$EESSI_ACCELERATOR_TARGET: ${EESSI_ACCELERATOR_TARGET}" + fi +else + echo_yellow "(configure_easybuild) \$EESSI_ACCELERATOR_TARGET not defined" +fi + # just ignore OS dependencies for now, see https://github.com/easybuilders/easybuild-framework/issues/3430 export EASYBUILD_IGNORE_OSDEPS=1 diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index b8e9cf979b..744e208ae0 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -20,6 +20,9 @@ fi if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then INPUT="export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}; ${INPUT}" fi +if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then + INPUT="export EESSI_ACCELERATOR_TARGET=${EESSI_ACCELERATOR_TARGET}; ${INPUT}" +fi if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then INPUT="export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}; ${INPUT}" fi From e8af7d829f961e0d0cb3e5529e8ecb72ba1b3c37 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 17 Sep 2024 18:42:57 +0200 Subject: [PATCH 007/152] update create_tarball.sh script to make it aware of installations in accel/ subdirectory --- EESSI-install-software.sh | 2 -- bot/build.sh | 4 +-- create_tarball.sh | 59 +++++++++++++++++++++------------------ 3 files changed, 34 insertions(+), 31 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 27c96eba9f..f9dd971a0d 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -309,8 +309,6 @@ else done fi -### add packages here - echo ">> Creating/updating Lmod RC file..." export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" diff --git a/bot/build.sh b/bot/build.sh index 7a07d09c4c..1c4032ecbc 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -282,8 +282,8 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI TMP_IN_CONTAINER=/tmp echo "Executing command to create tarball:" echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}" -echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" +echo " -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} \"${EESSI_ACCELERATOR_TARGET}\" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}" ./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \ - -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} + -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} "${EESSI_ACCELERATOR_TARGET}" /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr} exit 0 diff --git a/create_tarball.sh b/create_tarball.sh index 2dee665060..7b38da3b07 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -4,14 +4,15 @@ set -e base_dir=$(dirname $(realpath $0)) -if [ $# -ne 4 ]; then - echo "ERROR: Usage: $0 " >&2 +if [ $# -ne 5 ]; then + echo "ERROR: Usage: $0 " >&2 exit 1 fi eessi_tmpdir=$1 eessi_version=$2 cpu_arch_subdir=$3 -target_tgz=$4 +accel_subdir=$4 +target_tgz=$5 tmpdir=`mktemp -d` echo ">> tmpdir: $tmpdir" @@ -57,30 +58,34 @@ if [ -d ${eessi_version}/init ]; then find ${eessi_version}/init -type f | grep -v '/\.wh\.' >> ${files_list} fi -if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then - # module files - find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} - # module symlinks - find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} - # module files and symlinks - find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \ - | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ - >> ${module_files_list} -fi - -if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then - # installation directories but only those for which module files were created - # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' - # using EasyBuild's standard module naming scheme) match the name of the - # software installation directory (expected to be 'PACKAGE_NAME/VERSION/'). - # If either side changes (module naming scheme or naming of software - # installation directories), the procedure will likely not work. - for package_version in $(cat ${module_files_list}); do - echo "handling ${package_version}" - ls -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \ - | grep -v '/\.wh\.' >> ${files_list} - done -fi +# consider both CPU-only and accelerator subdirectories +for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do + + if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then + # module files + find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} + # module symlinks + find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + # module files and symlinks + find ${eessi_version}/software/${os}/${subdir}/modules/all -type f -o -type l \ + | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ + >> ${module_files_list} + fi + + if [ -d ${eessi_version}/software/${os}/${subdir}/software -a -r ${module_files_list} ]; then + # installation directories but only those for which module files were created + # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua' + # using EasyBuild's standard module naming scheme) match the name of the + # software installation directory (expected to be 'PACKAGE_NAME/VERSION/'). + # If either side changes (module naming scheme or naming of software + # installation directories), the procedure will likely not work. + for package_version in $(cat ${module_files_list}); do + echo "handling ${package_version}" + ls -d ${eessi_version}/software/${os}/${subdir}/software/${package_version} \ + | grep -v '/\.wh\.' >> ${files_list} + done + fi +done # add a bit debug output echo "wrote file list to ${files_list}" From 0eee4960d05d25f025b510984b572416e421a5d3 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 18 Sep 2024 09:57:57 +0200 Subject: [PATCH 008/152] add hook for zen4 builds to set --- eb_hooks.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 3e095129f9..0709389ffb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -26,6 +26,8 @@ CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' CPU_TARGET_A64FX = 'aarch64/a64fx' +CPU_TARGET_ZEN4 = 'x86_64/amd/zen4' + EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' SYSTEM = EASYCONFIG_CONSTANTS['SYSTEM'][0] @@ -514,6 +516,23 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") +def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): + """ + pre-configure hook for LAMMPS: + - set kokkos_arch on x86_64/amd/zen4 + """ + + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if self.name == 'LAMMPS': + if self.version == '2Aug2023': + if get_cpu_architecture() == X86_64: + if cpu_target == CPU_TARGET_ZEN4: + # There is no support for ZEN4 in LAMMPS yet so falling back to ZEN3 + self.cfg['kokkos'] = 'ZEN3' + else: + raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") + + def pre_test_hook(self,*args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: @@ -783,6 +802,7 @@ def inject_gpu_property(ec): 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, 'WRF': pre_configure_hook_wrf_aarch64, + 'LAMMPS': pre_configure_hook_LAMMPS_zen4, } PRE_TEST_HOOKS = { From eba92ca6c2b292c64fd47e6e5df6cfd97c5e8fdc Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:20:36 +0200 Subject: [PATCH 009/152] Set correct kokkos variable --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 0709389ffb..740f5e6cd8 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -528,7 +528,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): if get_cpu_architecture() == X86_64: if cpu_target == CPU_TARGET_ZEN4: # There is no support for ZEN4 in LAMMPS yet so falling back to ZEN3 - self.cfg['kokkos'] = 'ZEN3' + self.cfg['kokkos_arch'] = 'ZEN3' else: raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") From 1fca3170505a5ba32ccfcb97121405acd4bed5cc Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 18 Sep 2024 12:45:55 +0200 Subject: [PATCH 010/152] {2023.06}[2023a,GPU] LAMMPS 2Aug2023 CUDA 12.1.1 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml new file mode 100644 index 0000000000..91c4b1f472 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos-CUDA-12.1.1.eb From 54faa34196fe387fe9231677a07d83fc0021af36 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 18 Sep 2024 12:55:27 +0200 Subject: [PATCH 011/152] fix lammps version in hook --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 740f5e6cd8..b3e457cfe3 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -524,7 +524,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'LAMMPS': - if self.version == '2Aug2023': + if self.version == '2Aug2023_update2': if get_cpu_architecture() == X86_64: if cpu_target == CPU_TARGET_ZEN4: # There is no support for ZEN4 in LAMMPS yet so falling back to ZEN3 From 4794e09643bd122dbbfa2ae72cf219629aa6f77c Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 18 Sep 2024 14:59:02 +0200 Subject: [PATCH 012/152] {2023.06}[2023a,zen4] JupyterNotebook v7.0.2 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index df3d0dedaa..0818701bbb 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -1,2 +1,3 @@ easyconfigs: - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb + - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb From 028e57ea6110ce0347d8e33c4b6dd843fbb0f7ff Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 18 Sep 2024 15:23:26 +0200 Subject: [PATCH 013/152] use fixed prefix for tmp tarball and compress it --- eessi_container.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eessi_container.sh b/eessi_container.sh index 34fa833d99..fc97f9877c 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -846,14 +846,14 @@ if [[ ! -z ${SAVE} ]]; then # of these aspects to where the script is used if [[ -d ${SAVE} ]]; then # assume SAVE is name of a directory to which tarball shall be written to - # name format: {REPO_ID}-{TIMESTAMP}.tgz + # name format: tmp_storage-{TIMESTAMP}.tgz ts=$(date +%s) - TGZ=${SAVE}/${REPOSITORY}-${ts}.tgz + TGZ=${SAVE}/tmp_storage-${ts}.tgz else # assume SAVE is the full path to a tarball's name TGZ=${SAVE} fi - tar cf ${TGZ} -C ${EESSI_TMPDIR} . + tar czf ${TGZ} -C ${EESSI_TMPDIR} . echo "Saved contents of tmp directory '${EESSI_TMPDIR}' to tarball '${TGZ}' (to resume session add '--resume ${TGZ}')" fi From 03e0a7a2db36be52862a2a0db32898c1dc374157 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:05:17 +0200 Subject: [PATCH 014/152] Add CUDA-Samples for the accelerator prefix --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml new file mode 100644 index 0000000000..01a47bbc99 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb From f66e1608fbfbfcc6a2be668e6ede8a66a318b6f1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:10:44 +0200 Subject: [PATCH 015/152] Add OSU Microbenchmarks 7.2 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml new file mode 100644 index 0000000000..cccbfa6808 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb From b201c533c2271f6b4815661850570c26576e120c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:26:17 +0200 Subject: [PATCH 016/152] Added CUDA-12.1.1 --- .../2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml new file mode 100644 index 0000000000..630bf6aa25 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - CUDA-12.1.1.eb From 5c9fca2fd5ea3160ec4e45f0f47cb73a0cdc264c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:29:31 +0200 Subject: [PATCH 017/152] UCX cuda 1.14.1 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml new file mode 100644 index 0000000000..bfe4a4dc52 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - UCX-CUDA-1.14.1-GCCcore-12.3.0-CUDA-12.1.1.eb From e8f92f62f83c5c939467edb863c995e829a808d1 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:54:24 +0200 Subject: [PATCH 018/152] Rebuild CUDA in accelerator prefix --- .../2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml | 2 -- .../20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml | 7 +++++++ 2 files changed, 7 insertions(+), 2 deletions(-) delete mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml deleted file mode 100644 index 630bf6aa25..0000000000 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system-CUDA.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - CUDA-12.1.1.eb diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml new file mode 100644 index 0000000000..755bea096e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml @@ -0,0 +1,7 @@ +# 2024.09.18 +# We need to reinstall CUDA in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/720 +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA From 78fbc1e86412379e382e473f36bf7ae98b1613e2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 22:17:47 +0200 Subject: [PATCH 019/152] make sure removal script also takes software prefix into account --- EESSI-remove-software.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 446a156cb8..e435dd8088 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -79,6 +79,17 @@ echo ">> Setting up \$MODULEPATH..." module --force purge # ignore current $MODULEPATH entirely module unuse $MODULEPATH + +# if an accelerator target is specified, we need to make sure that the CPU-only modules are also still available +if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then + CPU_ONLY_MODULES_PATH=$(echo $EASYBUILD_INSTALLPATH | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@g")/modules/all + if [ -d ${CPU_ONLY_MODULES_PATH} ]; then + module use ${CPU_ONLY_MODULES_PATH} + else + fatal_error "Derived path to CPU-only modules does not exist: ${CPU_ONLY_MODULES_PATH}" + fi +fi + module use $EASYBUILD_INSTALLPATH/modules/all if [[ -z ${MODULEPATH} ]]; then fatal_error "Failed to set up \$MODULEPATH?!" From 20b65e67e633f68f49c9beb5129d03043aee6ad6 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 23:01:49 +0200 Subject: [PATCH 020/152] Make sure we remove from where the module is really located - and not just assume that it's in EASYBUILD_INSTALLPATH --- EESSI-remove-software.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index e435dd8088..25593b5a79 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -120,8 +120,12 @@ if [ $EUID -eq 0 ]; then # * [R] $CFGS/s/someapp/someapp-someversion.eb (module: someapp/someversion) rebuild_apps=$(eb --allow-use-as-root-and-accept-consequences --dry-run-short --rebuild --easystack ${easystack_file} | grep "^ \* \[R\]" | grep -o "module: .*[^)]" | awk '{print $2}') for app in ${rebuild_apps}; do - app_dir=${EASYBUILD_INSTALLPATH}/software/${app} - app_module=${EASYBUILD_INSTALLPATH}/modules/all/${app}.lua + # Returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all: + app_modulepath=$(module --terse av ${app} 2>&1 | head -n 1 | sed 's/://') + # Two dirname invocations, so returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2 + app_installprefix=$(dirname $(dirname app_modulpath)) + app_dir=${app_installprefix}/software/${app} + app_module=${app_installprefix}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." rm -rf ${app_dir} rm -rf ${app_module} From 4b3612dc807edc3988eb2d1727a812afcd751d7e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 23:11:35 +0200 Subject: [PATCH 021/152] Fix typo --- EESSI-remove-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 25593b5a79..05572257a5 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -123,7 +123,7 @@ if [ $EUID -eq 0 ]; then # Returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2/modules/all: app_modulepath=$(module --terse av ${app} 2>&1 | head -n 1 | sed 's/://') # Two dirname invocations, so returns e.g. /cvmfs/software.eessi.io/versions/2023.06/software/linux/x86_64/amd/zen2 - app_installprefix=$(dirname $(dirname app_modulpath)) + app_installprefix=$(dirname $(dirname ${app_modulepath})) app_dir=${app_installprefix}/software/${app} app_module=${app_installprefix}/modules/all/${app}.lua echo_yellow "Removing ${app_dir} and ${app_module}..." From 4f732776fc96e1a779ffb053bd0937137ef6c020 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 07:05:37 +0200 Subject: [PATCH 022/152] Limit this PR to only the EESSI-remove-software.sh change --- .../20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml deleted file mode 100644 index 755bea096e..0000000000 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml +++ /dev/null @@ -1,7 +0,0 @@ -# 2024.09.18 -# We need to reinstall CUDA in the accelerator prefixes -# See https://github.com/EESSI/software-layer/pull/720 -easyconfigs: - - CUDA-12.1.1.eb: - options: - accept-eula-for: CUDA From 53661bf07c9d240da40a12ae81bb02703c6bd4df Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 08:05:49 +0200 Subject: [PATCH 023/152] Fix issue with grep not returning anything on the CPU prefix --- create_tarball.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index e70a9b18d6..9c212681a5 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -64,9 +64,9 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then # module files - find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match # module symlinks - find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match # module files and symlinks find ${eessi_version}/software/${os}/${subdir}/modules/all -type f -o -type l \ | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ @@ -83,7 +83,7 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" ls -d ${eessi_version}/software/${os}/${subdir}/software/${package_version} \ - | grep -v '/\.wh\.' >> ${files_list} + | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match done fi done From ce09733386a5a9b99695c548731c4989bd0041c4 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 08:41:25 +0200 Subject: [PATCH 024/152] Readd cuda reinstall --- .../20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml new file mode 100644 index 0000000000..755bea096e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml @@ -0,0 +1,7 @@ +# 2024.09.18 +# We need to reinstall CUDA in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/720 +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA From 6d8ff958be445f3e46f74526614cc0c3c61981ee Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 08:43:42 +0200 Subject: [PATCH 025/152] Fix creation of tarball, since grep doesn't match (and thus returns non-zero exit code) for the CPU prefix if nothing got build there --- create_tarball.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index e70a9b18d6..9c212681a5 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -64,9 +64,9 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do if [ -d ${eessi_version}/software/${os}/${subdir}/modules ]; then # module files - find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match # module symlinks - find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} + find ${eessi_version}/software/${os}/${subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match # module files and symlinks find ${eessi_version}/software/${os}/${subdir}/modules/all -type f -o -type l \ | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \ @@ -83,7 +83,7 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do for package_version in $(cat ${module_files_list}); do echo "handling ${package_version}" ls -d ${eessi_version}/software/${os}/${subdir}/software/${package_version} \ - | grep -v '/\.wh\.' >> ${files_list} + | grep -v '/\.wh\.' >> ${files_list} || true # Make sure we don't exit because of set -e if grep doesn't return a match done fi done From 67fb8131f0aeb9af3d59205531ed440f4352d742 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 19 Sep 2024 09:24:30 +0200 Subject: [PATCH 026/152] {2023.06}[foss/2023a,zen4] PyQt5 v5.15.10 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 0818701bbb..ddf0ec4507 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -1,3 +1,4 @@ easyconfigs: - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb + - PyQt5-5.15.10-GCCcore-12.3.0.eb From 3d43781830578a34384ee54541662e8b698a9590 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 19 Sep 2024 09:31:14 +0200 Subject: [PATCH 027/152] {2023} --- .../2023.06/eessi-2023.06-eb-4.9.3-2023b.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml new file mode 100644 index 0000000000..382241b98e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml @@ -0,0 +1,5 @@ +easyconfigs: + - LAMMPS-29Aug2024-foss-2023b-kokkos.eb: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21436 + options: + from-pr: 21436 From 8adaaafbe93a89a60702e97c08afa17903c9452d Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 19 Sep 2024 09:49:44 +0200 Subject: [PATCH 028/152] update hook for v29Aug2024 --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index b3e457cfe3..f25aacfc9c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -301,7 +301,7 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): """ Remove x86_64 specific dependencies for the CI to pass on aarch64 """ - if ec.name == 'LAMMPS' and ec.version in ('2Aug2023_update2',): + if ec.name == 'LAMMPS' and ec.version in ('2Aug2023_update2', '29Aug2024'): if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': # ScaFaCoS and tbb are not compatible with aarch64/* CPU targets, # so remove them as dependencies for LAMMPS (they're optional); From 914d5c13572d78defa82e3bd13aa01c28b5f022a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:31:39 +0200 Subject: [PATCH 029/152] Move to rebuilds --- .../20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023a-CUDA.yml => rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml From 24879a80c16eec3da1b19ea409fc20331b19b407 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:34:39 +0200 Subject: [PATCH 030/152] Move to rebuilds --- .../20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023a-CUDA.yml => rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml From aef0999a54787988fd844ec7b07643efba74c28c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:44:42 +0200 Subject: [PATCH 031/152] Move into rebuilds --- .../20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023a-CUDA.yml => rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml From 5dcce7688a3af265da6b966eaceb13ff2b3de599 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:45:44 +0200 Subject: [PATCH 032/152] Add comment --- .../20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml index cccbfa6808..23801e0250 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml @@ -1,2 +1,5 @@ +# 2024.09.19 +# We need to reinstall OSU-Micro-Benchmarks in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/716 easyconfigs: - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb From 4805c09a06977c77177f700fa135954cde6a3859 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:49:20 +0200 Subject: [PATCH 033/152] Add comment --- .../20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml index 01a47bbc99..da2c06ae1e 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml @@ -1,2 +1,5 @@ +# 2024.09.19 +# We need to reinstall CUDA-Samples in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/715 easyconfigs: - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb From 9232ac91aa8b3c5ee734b3e72397aa90e16831e7 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:49:58 +0200 Subject: [PATCH 034/152] Add comment --- .../rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml index bfe4a4dc52..d347af335a 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml @@ -1,2 +1,5 @@ +# 2024.09.19 +# We need to reinstall UCX-CUDA in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/719 easyconfigs: - UCX-CUDA-1.14.1-GCCcore-12.3.0-CUDA-12.1.1.eb From 09bfb97a4d29f62b901316c9dd08dc6a422bc519 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 19 Sep 2024 11:09:34 +0200 Subject: [PATCH 035/152] update missing installations hook --- eb_hooks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index f25aacfc9c..d999c735c3 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -297,9 +297,9 @@ def parse_hook_ucx_eprefix(ec, eprefix): raise EasyBuildError("UCX-specific hook triggered for non-UCX easyconfig?!") -def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): +def parse_hook_lammps_remove_deps_for_aarch64(ec, *args, **kwargs): """ - Remove x86_64 specific dependencies for the CI to pass on aarch64 + Remove x86_64 specific dependencies for the CI and missing installations to pass on aarch64 """ if ec.name == 'LAMMPS' and ec.version in ('2Aug2023_update2', '29Aug2024'): if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': @@ -309,7 +309,7 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): # https://github.com/easybuilders/easybuild-easyconfigs/pull/19000; # we need this hook because we check for missing installations for all CPU targets # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true) - ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('ScaFaCoS', 'tbb')] + ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('ScaFaCoS', 'tbb',)] else: raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") @@ -776,7 +776,7 @@ def inject_gpu_property(ec): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'grpcio': parse_hook_grpcio_zlib, - 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, + 'LAMMPS': parse_hook_lammps_remove_deps_for_aarch64, 'CP2K': parse_hook_CP2K_remove_deps_for_aarch64, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, From db91170bf73112ff98e92c818a857cfa4b9a6c93 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 19 Sep 2024 12:01:44 +0200 Subject: [PATCH 036/152] use --from-commit --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml index 382241b98e..b3c903161a 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023b.yml @@ -2,4 +2,4 @@ easyconfigs: - LAMMPS-29Aug2024-foss-2023b-kokkos.eb: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21436 options: - from-pr: 21436 + from-commit: 9dc24e57880a8adb06ae10557c5315e66671a533 From 05be1a2a9ea800796e40c551e42f4639088916a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Thu, 19 Sep 2024 16:54:37 +0200 Subject: [PATCH 037/152] adjust the prefix variable for accelerator builds --- bot/check-build.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bot/check-build.sh b/bot/check-build.sh index d8246c67be..f185b18dda 100755 --- a/bot/check-build.sh +++ b/bot/check-build.sh @@ -457,8 +457,14 @@ if [[ ! -z ${TARBALL} ]]; then repo_version=$(cfg_get_value "repository" "repo_version") os_type=$(cfg_get_value "architecture" "os_type") software_subdir=$(cfg_get_value "architecture" "software_subdir") + accelerator=$(cfg_get_value "architecture" "accelerator") prefix="${repo_version}/software/${os_type}/${software_subdir}" + # if we build for an accelerator, the prefix is different + if [[ ! -z ${accelerator} ]]; then + prefix="${prefix}/accel/${accelerator}" + fi + # extract directories/entries from tarball content modules_entries=$(grep "${prefix}/modules" ${tmpfile}) software_entries=$(grep "${prefix}/software" ${tmpfile}) From 07572887b9ae6683ffe6eb700eb8a51aa0e9325a Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 20 Sep 2024 08:23:46 +0200 Subject: [PATCH 038/152] {2023.06}[foss/2023a] NCCL v2.18.3 w/ CUDA 12.1.1 --- .../20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml new file mode 100644 index 0000000000..6ff3b1bfbf --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml @@ -0,0 +1,5 @@ +# 2024.09.19 +# We need to reinstall NCCL in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/487 +easyconfigs: + - NCCL-2.18.3-GCCcore-12.3.0-CUDA-12.1.1.eb From 7475143a08d4d0da33304debd8a849d7a165a3bb Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 20 Sep 2024 08:31:57 +0200 Subject: [PATCH 039/152] Update eessi-2023.06-eb-4.9.3-2023a.yml --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index ddf0ec4507..c51a0b273a 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -2,3 +2,4 @@ easyconfigs: - LAMMPS-2Aug2023_update2-foss-2023a-kokkos.eb - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb - PyQt5-5.15.10-GCCcore-12.3.0.eb + - OrthoFinder-2.5.5-foss-2023a.eb From ac3b4dd80a1802c42b6e4684e9105c095a226b94 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 20 Sep 2024 12:14:43 +0200 Subject: [PATCH 040/152] {2023.06}[foss/2023a,zen4] snakemake v8.4.2 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index c51a0b273a..f5d5f6c16c 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -3,3 +3,4 @@ easyconfigs: - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb - PyQt5-5.15.10-GCCcore-12.3.0.eb - OrthoFinder-2.5.5-foss-2023a.eb + - snakemake-8.4.2-foss-2023a.eb From b963a4889d1a4c80c57d9bc72f5061c68520c200 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 20 Sep 2024 15:39:11 +0200 Subject: [PATCH 041/152] {2023.06}[2023a,zen4] Rivet 3.1.9 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index f5d5f6c16c..cc35cb7af5 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -4,3 +4,4 @@ easyconfigs: - PyQt5-5.15.10-GCCcore-12.3.0.eb - OrthoFinder-2.5.5-foss-2023a.eb - snakemake-8.4.2-foss-2023a.eb + - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb From b27229f0980c15dc61f2685ab350a975750f9184 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 20 Sep 2024 17:35:03 +0200 Subject: [PATCH 042/152] {2023.06}[foss/2023a,zen4] GATK v4.5.0.0 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index cc35cb7af5..6864fcf632 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -5,3 +5,4 @@ easyconfigs: - OrthoFinder-2.5.5-foss-2023a.eb - snakemake-8.4.2-foss-2023a.eb - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb + - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb From 193948cb4c906be2ef494e8b4fb82611b8664a29 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 23 Sep 2024 09:19:55 +0200 Subject: [PATCH 043/152] {2023.06}[gfbf/2023a,zen4] ipympl v0.9.3 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 6864fcf632..ded94d4cbe 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -6,3 +6,5 @@ easyconfigs: - snakemake-8.4.2-foss-2023a.eb - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb + - ipympl-0.9.3-foss-2023a.eb + - ipympl-0.9.3-gfbf-2023a.eb From 4ad4da28aab74e5076d4adfaba65d8cfc0980d62 Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 23 Sep 2024 11:26:23 +0200 Subject: [PATCH 044/152] remove ipympl-0.9.3-foss-2023a.eb --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index ded94d4cbe..cbacef6501 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -6,5 +6,4 @@ easyconfigs: - snakemake-8.4.2-foss-2023a.eb - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb - - ipympl-0.9.3-foss-2023a.eb - ipympl-0.9.3-gfbf-2023a.eb From d31d052eaaf607d5f9bc9798cf7eb435403d80c3 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 23 Sep 2024 11:39:39 +0000 Subject: [PATCH 045/152] {2023.06}[foss/2022b] math-packages --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 02b00e6173..5f63a7bc14 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -27,4 +27,8 @@ easyconfigs: - lpsolve-5.5.2.11-GCC-12.2.0.eb - fastp-0.23.4-GCC-12.2.0.eb - KronaTools-2.8.1-GCCcore-12.2.0.eb - - MultiQC-1.14-foss-2022b.eb + - MultiQC-1.14-foss-2022b.eb + - CGAL-5.5.2-GCCcore-12.2.0.eb + - KaHIP-3.14-gompi-2022b.eb + - MPC-1.3.1-GCCcore-12.2.0.eb + - MUMPS-5.6.1-foss-2022b-metis.eb From c14690cad1395fc2a12e08af6a0fc848edd904ca Mon Sep 17 00:00:00 2001 From: lara Date: Tue, 24 Sep 2024 11:42:46 +0200 Subject: [PATCH 046/152] {2023.06}[foss/2023a,zen4] LHAPDF v6.5.4 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index cbacef6501..29ab1609da 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -7,3 +7,4 @@ easyconfigs: - Rivet-3.1.9-gompi-2023a-HepMC3-3.2.6.eb - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb - ipympl-0.9.3-gfbf-2023a.eb + - LHAPDF-6.5.4-GCC-12.3.0.eb From 8fa60259ea3dda1bef61de33dad61f9c966595f5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 24 Sep 2024 12:51:22 +0200 Subject: [PATCH 047/152] Limit CUDA hook to EESSI installs only, and remove duplication when creating symlinks --- eb_hooks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index b3e457cfe3..e3c6c4faeb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -684,7 +684,8 @@ def post_sanitycheck_cuda(self, *args, **kwargs): Remove files from CUDA installation that we are not allowed to ship, and replace them with a symlink to a corresponding installation under host_injections. """ - if self.name == 'CUDA': + # Make sure we only do this for CUDA and only if we are doing a CVMFS installation + if self.name == 'CUDA' and self.installdir.startswith('/cvmfs/software.eessi.io/versions'): print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped @@ -733,6 +734,9 @@ def post_sanitycheck_cuda(self, *args, **kwargs): basename, full_path) # if it is not in the allowlist, delete the file and create a symlink to host_injections host_inj_path = full_path.replace('versions', 'host_injections') + # CUDA itself doesn't care about compute capability so remove this duplication from + # under host_injections + host_inj_path = re.sub(r"accel/nvidia/cc\d+/", '', host_inj_path) # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From c1764335999be619d5b2ae8629fd9552a1b87abc Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 24 Sep 2024 15:23:06 +0200 Subject: [PATCH 048/152] Add gmsh --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..f2d8a9380f --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - gmsh-4.12.2-foss-2023a.eb From 087b7d7d42436b07c035b3433d03c9318fd9ec4c Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 24 Sep 2024 15:46:08 +0200 Subject: [PATCH 049/152] Use EESSI_ACCELERATOR_TARGET rather than regex --- eb_hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index e3c6c4faeb..0a42a91d49 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -736,7 +736,9 @@ def post_sanitycheck_cuda(self, *args, **kwargs): host_inj_path = full_path.replace('versions', 'host_injections') # CUDA itself doesn't care about compute capability so remove this duplication from # under host_injections - host_inj_path = re.sub(r"accel/nvidia/cc\d+/", '', host_inj_path) + accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") + if accel_subdir: + host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, 'host_injections') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From 8594bb2abbbc01ec5c23e87fb95a6e8198b43284 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 24 Sep 2024 15:59:45 +0200 Subject: [PATCH 050/152] Also add basemap and geopandas --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml index f2d8a9380f..a34e5d768e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -1,2 +1,4 @@ easyconfigs: - gmsh-4.12.2-foss-2023a.eb + - basemap-1.3.9-foss-2023a.eb + - geopandas-0.14.2-foss-2023a.eb From b8555d1ee64330a0a43df838f4d1ec58a916a4b5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 24 Sep 2024 16:00:32 +0200 Subject: [PATCH 051/152] Ensure we are making an EESSI install when using the CUDA hook --- eb_hooks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 0a42a91d49..4d94357e8c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -685,7 +685,12 @@ def post_sanitycheck_cuda(self, *args, **kwargs): and replace them with a symlink to a corresponding installation under host_injections. """ # Make sure we only do this for CUDA and only if we are doing a CVMFS installation - if self.name == 'CUDA' and self.installdir.startswith('/cvmfs/software.eessi.io/versions'): + is_eessi_install = ( + self.installdir.startswith("/cvmfs/software.eessi.io/versions") + and not build_option("sanity_check_only") + and not build_option("module_only") + ) + if self.name == 'CUDA' and is_eessi_install: print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped From fb2c0858a6a5c42d6056b019c62e043b8172ad92 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 24 Sep 2024 16:07:16 +0200 Subject: [PATCH 052/152] Sloppy copy/paste --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4d94357e8c..96fa48e129 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -743,7 +743,7 @@ def post_sanitycheck_cuda(self, *args, **kwargs): # under host_injections accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: - host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, 'host_injections') + host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, '') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From cf63589f1c5a0f0f08a4eb44384ee0a920754848 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 24 Sep 2024 16:08:37 +0200 Subject: [PATCH 053/152] Update eb_hooks.py --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 96fa48e129..ed006bae1c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -743,7 +743,7 @@ def post_sanitycheck_cuda(self, *args, **kwargs): # under host_injections accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: - host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, '') + host_inj_path = host_inj_path.replace("/accel/%s" % accel_subdir, '') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From 4478fa89b26472c24824c053d67d35f040fc1bc2 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 24 Sep 2024 16:12:22 +0200 Subject: [PATCH 054/152] Add various tools --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..d3927ea3f1 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,6 @@ +easyconfigs: + - ccache-4.9-GCCcore-12.3.0.eb + - GDB-13.2-GCCcore-12.3.0.eb + - mold-1.11.0-GCCcore-12.3.0.eb + - tmux-3.3a-GCCcore-12.3.0.eb + - Vim-9.1.0004-GCCcore-12.3.0.eb From 2ca35fc51326cee97b21fe381c55421c047f87ae Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 09:32:10 +0200 Subject: [PATCH 055/152] Add fPIC through hooks for FreeImage on ARM, see if that resolves https://github.com/EESSI/software-layer/pull/736#issuecomment-2373261889 --- eb_hooks.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index b3e457cfe3..0987d6d683 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -297,6 +297,18 @@ def parse_hook_ucx_eprefix(ec, eprefix): raise EasyBuildError("UCX-specific hook triggered for non-UCX easyconfig?!") +def parse_hook_freeimage_aarch64(ec, *args, **kwargs): + """ + Make sure to build with -fPIC on ARM to avoid + https://github.com/EESSI/software-layer/pull/736#issuecomment-2373261889 + """ + if ec.name == 'FreeImage' and ec.version in ('3.18.0',): + if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': + if not hasattr(ec, 'toolchainopts'): + ec['toolchainopts'] = {} + ec['toolchainopts']['fPIC'] = True + print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) + def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): """ Remove x86_64 specific dependencies for the CI to pass on aarch64 From 3b9fbdc9bf4b1396388efd01647468ec7a36a2de Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 25 Sep 2024 09:32:47 +0200 Subject: [PATCH 056/152] Update eessi-2023.06-eb-4.9.3-2023a.yml --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 29ab1609da..1d1b2caa09 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -8,3 +8,4 @@ easyconfigs: - GATK-4.5.0.0-GCCcore-12.3.0-Java-17.eb - ipympl-0.9.3-gfbf-2023a.eb - LHAPDF-6.5.4-GCC-12.3.0.eb + - LoopTools-2.15-GCC-12.3.0.eb From e61617ce19307a16132c5154fbc9729abee9d434 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 25 Sep 2024 09:51:34 +0200 Subject: [PATCH 057/152] {2023.06}[system] EasyBuild v4.9.4 --- .../2023.06/eessi-2023.06-eb-4.9.3-001-system.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml new file mode 100644 index 0000000000..d9c6075561 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml @@ -0,0 +1,5 @@ +easyconfigs: + - EasyBuild-4.9.4.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + from-commit: 39cdebd7bd2cb4a9c170ee22439401316b2e7a25 From 07c5f0aab33b591c30101c01fc08f904a21d7335 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 25 Sep 2024 10:05:21 +0200 Subject: [PATCH 058/152] {2023.06}[foss/2023a] NCCL 2.18.3 w/ CUDA 12.1.1 --- ...ix.yml => 20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename easystacks/software.eessi.io/2023.06/rebuilds/{20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml => 20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml} (92%) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml similarity index 92% rename from easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml index 6ff3b1bfbf..d6667af9a1 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml @@ -1,4 +1,4 @@ -# 2024.09.19 +# 2024.09.25 # We need to reinstall NCCL in the accelerator prefixes # See https://github.com/EESSI/software-layer/pull/487 easyconfigs: From ae62bd31a55cac6dc789ebacae464a9b308c5b5d Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 25 Sep 2024 10:32:49 +0200 Subject: [PATCH 059/152] {2023.06}[2023a,zen4] ncdu 1.18 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 1d1b2caa09..7ced0d9218 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -9,3 +9,4 @@ easyconfigs: - ipympl-0.9.3-gfbf-2023a.eb - LHAPDF-6.5.4-GCC-12.3.0.eb - LoopTools-2.15-GCC-12.3.0.eb + - ncdu-1.18-GCC-12.3.0.eb From b4f6b60c2f5c582360a1148ceb5052077d7287fd Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 10:41:07 +0200 Subject: [PATCH 060/152] Actually use the hook... --- eb_hooks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/eb_hooks.py b/eb_hooks.py index 0987d6d683..15e51e688c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -787,6 +787,7 @@ def inject_gpu_property(ec): 'casacore': parse_hook_casacore_disable_vectorize, 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, + 'FreeImage': parse_hook_freeimage_aarch64, 'grpcio': parse_hook_grpcio_zlib, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, 'CP2K': parse_hook_CP2K_remove_deps_for_aarch64, From d37a958d008ba679af7872b4bb9038df74338842 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 10:54:02 +0200 Subject: [PATCH 061/152] Move CUDA hook to post-install, allow hook to trigger for any EESSI distributed repo (but always make symlinks to software.eessi.io) --- eb_hooks.py | 76 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4d94357e8c..788a0419ba 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -131,7 +131,8 @@ def pre_prepare_hook(self, *args, **kwargs): def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs): """ Post-configure hook for GCCcore: - - copy RPATH wrapper script for linker commands to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu' + - copy RPATH wrapper script for linker commands to also have a wrapper in + place with system type prefix like 'x86_64-pc-linux-gnu' """ if self.name == 'GCCcore': config_guess = obtain_config_guess() @@ -279,10 +280,10 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): Disable check for QtWebEngine in Qt5 as workaround for problem with determining glibc version. """ if ec.name == 'Qt5': - # workaround for glibc version being reported as "UNKNOWN" in Gentoo Prefix environment by EasyBuild v4.7.2, - # see also https://github.com/easybuilders/easybuild-framework/pull/4290 - ec['check_qtwebengine'] = False - print_msg("Checking for QtWebEgine in Qt5 installation has been disabled") + # workaround for glibc version being reported as "UNKNOWN" in Gentoo Prefix environment by EasyBuild v4.7.2, + # see also https://github.com/easybuilders/easybuild-framework/pull/4290 + ec['check_qtwebengine'] = False + print_msg("Checking for QtWebEgine in Qt5 installation has been disabled") else: raise EasyBuildError("Qt5-specific hook triggered for non-Qt5 easyconfig?!") @@ -341,7 +342,7 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg if self.name == 'Highway': tcname, tcversion = self.toolchain.name, self.toolchain.version cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - # note: keep condition in sync with the one used in + # note: keep condition in sync with the one used in # post_prepare_hook_highway_handle_test_compilation_issues if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': if cpu_target in [CPU_TARGET_A64FX, CPU_TARGET_NEOVERSE_V1]: @@ -360,12 +361,13 @@ def post_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwar if self.name == 'Highway': tcname, tcversion = self.toolchain.name, self.toolchain.version cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - # note: keep condition in sync with the one used in + # note: keep condition in sync with the one used in # pre_prepare_hook_highway_handle_test_compilation_issues if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': if cpu_target == CPU_TARGET_NEOVERSE_N1: update_build_option('optarch', self.orig_optarch) + def pre_configure_hook(self, *args, **kwargs): """Main pre-configure hook: trigger custom functions based on software name.""" if self.name in PRE_CONFIGURE_HOOKS: @@ -389,6 +391,7 @@ def pre_configure_hook_BLIS_a64fx(self, *args, **kwargs): else: raise EasyBuildError("BLIS-specific hook triggered for non-BLIS easyconfig?!") + def pre_configure_hook_extrae(self, *args, **kwargs): """ Pre-configure hook for Extrae @@ -414,7 +417,11 @@ def pre_configure_hook_extrae(self, *args, **kwargs): # replace use of 'which' with 'command -v', since 'which' is broken in EESSI build container; # this must be done *after* running configure script, because initial configuration re-writes configure script, # and problem due to use of which only pops up when running make ?! - self.cfg.update('prebuildopts', "cp config/mpi-macros.m4 config/mpi-macros.m4.orig && sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && ") + self.cfg.update( + 'prebuildopts', + "cp config/mpi-macros.m4 config/mpi-macros.m4.orig &&" + "sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && " + ) else: raise EasyBuildError("Extrae-specific hook triggered for non-Extrae easyconfig?!") @@ -445,7 +452,10 @@ def pre_configure_hook_gromacs(self, *args, **kwargs): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if LooseVersion(self.version) <= LooseVersion('2024.1') and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg.update('configopts', '-DGMX_SIMD=ARM_NEON_ASIMD') - print_msg("Avoiding use of SVE instructions for GROMACS %s by using ARM_NEON_ASIMD as GMX_SIMD value", self.version) + print_msg( + "Avoiding use of SVE instructions for GROMACS %s by using ARM_NEON_ASIMD as GMX_SIMD value", + self.version + ) else: raise EasyBuildError("GROMACS-specific hook triggered for non-GROMACS easyconfig?!") @@ -506,12 +516,12 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): pattern = "Linux x86_64 ppc64le, gfortran" repl = "Linux x86_64 aarch64 ppc64le, gfortran" if LooseVersion(self.version) <= LooseVersion('3.9.0'): - self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl)) - print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) + self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl)) + print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) if LooseVersion('4.0.0') <= LooseVersion(self.version) <= LooseVersion('4.2.1'): - self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl)) - print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) + self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl)) + print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) else: raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") @@ -533,7 +543,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") -def pre_test_hook(self,*args, **kwargs): +def pre_test_hook(self, *args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: PRE_TEST_HOOKS[self.name](self, *args, **kwargs) @@ -596,6 +606,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): elif cpu_target == CPU_TARGET_A64FX and self.version in scipy_bundle_versions_a64fx: self.cfg['testopts'] = "|| echo ignoring failing tests" + def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): """ Pre-test hook for netCDF: skip failing tests for selected netCDF versions on neoverse_v1 @@ -609,6 +620,7 @@ def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): if self.name == 'netCDF' and self.version == '4.9.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" + def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs): """ Pre-test hook for PyTorch: increase max failing tests for ARM for PyTorch 2.1.2 @@ -673,24 +685,24 @@ def pre_single_extension_testthat(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && " -def post_sanitycheck_hook(self, *args, **kwargs): - """Main post-sanity-check hook: trigger custom functions based on software name.""" - if self.name in POST_SANITYCHECK_HOOKS: - POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) +def post_postproc_hook(self, *args, **kwargs): + """Main post-postprocessing hook: trigger custom functions based on software name.""" + if self.name in POST_POSTPROC_HOOKS: + POST_POSTPROC_HOOKS[self.name](self, *args, **kwargs) -def post_sanitycheck_cuda(self, *args, **kwargs): +def post_postproc_cuda(self, *args, **kwargs): """ Remove files from CUDA installation that we are not allowed to ship, and replace them with a symlink to a corresponding installation under host_injections. """ - # Make sure we only do this for CUDA and only if we are doing a CVMFS installation - is_eessi_install = ( - self.installdir.startswith("/cvmfs/software.eessi.io/versions") - and not build_option("sanity_check_only") - and not build_option("module_only") - ) - if self.name == 'CUDA' and is_eessi_install: + + # We need to check if we are doing an EESSI-distributed installation + eessi_pattern = r"^/cvmfs/[^/]*.eessi.io/versions/" + host_injections_location = "/cvmfs/software.eessi.io/host_injections/" + eessi_installation = bool(re.search(eessi_pattern, self.installdir)) + + if self.name == 'CUDA' and eessi_installation: print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped @@ -738,12 +750,14 @@ def post_sanitycheck_cuda(self, *args, **kwargs): self.log.debug("%s is not found in allowlist, so replacing it with symlink: %s", basename, full_path) # if it is not in the allowlist, delete the file and create a symlink to host_injections - host_inj_path = full_path.replace('versions', 'host_injections') + + # the host_injections path is under a fixed repo/location for CUDA + host_inj_path = re.sub(eessi_pattern, host_injections_location, full_path) # CUDA itself doesn't care about compute capability so remove this duplication from # under host_injections accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: - host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, 'host_injections') + host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, '') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " @@ -775,7 +789,7 @@ def inject_gpu_property(ec): ec_dict['builddependencies'].append(dep) value = '\n'.join([value, 'setenv("EESSICUDAVERSION","%s")' % cuda_version]) if key in ec_dict: - if not value in ec_dict[key]: + if value not in ec_dict[key]: ec[key] = '\n'.join([ec_dict[key], value]) else: ec[key] = value @@ -835,6 +849,6 @@ def inject_gpu_property(ec): 'numpy': post_single_extension_numpy, } -POST_SANITYCHECK_HOOKS = { - 'CUDA': post_sanitycheck_cuda, +POST_POSTPROC_HOOKS = { + 'CUDA': post_postproc_cuda, } From 1d446dcddcc68b09efa85429198435f7a3cdfe05 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 11:05:38 +0200 Subject: [PATCH 062/152] The eb toolchainopts is called pic, not fPIC --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 15e51e688c..2b503f4f9d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -306,7 +306,7 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': if not hasattr(ec, 'toolchainopts'): ec['toolchainopts'] = {} - ec['toolchainopts']['fPIC'] = True + ec['toolchainopts']['pic'] = True print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): From 4fe6194a76ba39f808b5b1cf65b46ca5701594e5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 11:10:51 +0200 Subject: [PATCH 063/152] Remove mold, it's a complicated case since it is a linker, and it's not urgent now --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml index d3927ea3f1..0a4a4e5554 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -1,6 +1,5 @@ easyconfigs: - ccache-4.9-GCCcore-12.3.0.eb - GDB-13.2-GCCcore-12.3.0.eb - - mold-1.11.0-GCCcore-12.3.0.eb - tmux-3.3a-GCCcore-12.3.0.eb - Vim-9.1.0004-GCCcore-12.3.0.eb From c58033861c94c4b01dca443ef7a672b63da274fa Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 11:14:29 +0200 Subject: [PATCH 064/152] Address review comments --- eb_hooks.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index f9ecefe1f6..9b0e9c8dcb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -32,6 +32,9 @@ SYSTEM = EASYCONFIG_CONSTANTS['SYSTEM'][0] +EESSI_INSTALLATION_REGEX = r"^/cvmfs/[^/]*.eessi.io/versions/" +HOST_INJECTIONS_LOCATION = "/cvmfs/software.eessi.io/host_injections/" + def get_eessi_envvar(eessi_envvar): """Get an EESSI environment variable from the environment""" @@ -419,7 +422,7 @@ def pre_configure_hook_extrae(self, *args, **kwargs): # and problem due to use of which only pops up when running make ?! self.cfg.update( 'prebuildopts', - "cp config/mpi-macros.m4 config/mpi-macros.m4.orig &&" + "cp config/mpi-macros.m4 config/mpi-macros.m4.orig && " "sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && " ) else: @@ -698,9 +701,7 @@ def post_postproc_cuda(self, *args, **kwargs): """ # We need to check if we are doing an EESSI-distributed installation - eessi_pattern = r"^/cvmfs/[^/]*.eessi.io/versions/" - host_injections_location = "/cvmfs/software.eessi.io/host_injections/" - eessi_installation = bool(re.search(eessi_pattern, self.installdir)) + eessi_installation = bool(re.search(EESSI_INSTALLATION_REGEX, self.installdir)) if self.name == 'CUDA' and eessi_installation: print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") @@ -752,12 +753,12 @@ def post_postproc_cuda(self, *args, **kwargs): # if it is not in the allowlist, delete the file and create a symlink to host_injections # the host_injections path is under a fixed repo/location for CUDA - host_inj_path = re.sub(eessi_pattern, host_injections_location, full_path) + host_inj_path = re.sub(EESSI_INSTALLATION_REGEX, HOST_INJECTIONS_LOCATION, full_path) # CUDA itself doesn't care about compute capability so remove this duplication from - # under host_injections + # under host_injections (symlink to a single CUDA installation for all compute + # capabilities) accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: - host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, '') host_inj_path = host_inj_path.replace("/accel/%s" % accel_subdir, '') # make sure source and target of symlink are not the same if full_path == host_inj_path: From e59156577044ee348de0ec415122c8a840453284 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 25 Sep 2024 12:09:36 +0200 Subject: [PATCH 065/152] {2023.06} Nextflow/23.10.0 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml new file mode 100644 index 0000000000..1e30631e57 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml @@ -0,0 +1,2 @@ +easyconfigs: + - Nextflow-23.10.0.eb From b235c7cbc7ecad2cf2f1709ece38db8cf64ddff8 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 12:20:52 +0200 Subject: [PATCH 066/152] Make sure Lmod site package and RC file do not appear in accelerator subdir --- create_lmodrc.py | 6 ++++++ create_lmodsitepackage.py | 17 ++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/create_lmodrc.py b/create_lmodrc.py index 28ad2a1915..e06f8ccfa4 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -33,6 +33,12 @@ def error(msg): error("Prefix directory %s does not exist!" % prefix) lmodrc_path = os.path.join(prefix, DOT_LMOD, 'lmodrc.lua') +# Lmod itself doesn't care about compute capability so remove this duplication from +# the install path (if it exists) +accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") +if accel_subdir: + lmodrc_path = lmodrc_path.replace("/accel/%s" % accel_subdir, '') + lmodrc_txt = TEMPLATE_LMOD_RC % { 'dot_lmod': DOT_LMOD, 'prefix': prefix, diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 62f073c9a6..11ca614be5 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -8,7 +8,7 @@ DOT_LMOD = '.lmod' -hook_txt ="""require("strict") +hook_txt = """require("strict") local hook = require("Hook") local open = io.open @@ -36,7 +36,7 @@ -- eessi_prefix_host_injections is the prefix with site-extensions (i.e. additional modules) -- to the official EESSI modules, e.g. /cvmfs/software.eessi.io/host_injections/2023.06 local eessi_prefix_host_injections = string.gsub(eessi_prefix, 'versions', 'host_injections') - + -- Check if the full modulepath starts with the eessi_prefix_* return string.find(t.fn, "^" .. eessi_prefix) ~= nil or string.find(t.fn, "^" .. eessi_prefix_host_injections) ~= nil end @@ -103,7 +103,7 @@ if isFile(archSitePackage) then dofile(archSitePackage) end - + end @@ -111,7 +111,7 @@ local frameStk = require("FrameStk"):singleton() local mt = frameStk:mt() local simpleName = string.match(t.modFullName, "(.-)/") - -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. + -- If we try to load CUDA itself, check if the full CUDA SDK was installed on the host in host_injections. -- This is required for end users to build additional CUDA software. If the full SDK isn't present, refuse -- to load the CUDA module and print an informative message on how to set up GPU support for EESSI local refer_to_docs = "For more information on how to do this, see https://www.eessi.io/docs/gpu/.\\n" @@ -207,6 +207,7 @@ load_site_specific_hooks() """ + def error(msg): sys.stderr.write("ERROR: %s\n" % msg) sys.exit(1) @@ -221,12 +222,18 @@ def error(msg): error("Prefix directory %s does not exist!" % prefix) sitepackage_path = os.path.join(prefix, DOT_LMOD, 'SitePackage.lua') + +# Lmod itself doesn't care about compute capability so remove this duplication from +# the install path (if it exists) +accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") +if accel_subdir: + sitepackage_path = sitepackage_path.replace("/accel/%s" % accel_subdir, '') try: os.makedirs(os.path.dirname(sitepackage_path), exist_ok=True) with open(sitepackage_path, 'w') as fp: fp.write(hook_txt) # Make sure that the created Lmod file has "read/write" for the user/group and "read" permissions for others - os.chmod(sitepackage_path, S_IREAD|S_IWRITE|S_IRGRP|S_IWGRP|S_IROTH) + os.chmod(sitepackage_path, S_IREAD | S_IWRITE | S_IRGRP | S_IWGRP | S_IROTH) except (IOError, OSError) as err: error("Failed to create %s: %s" % (sitepackage_path, err)) From 93dae0daa8b44150aa4bdd9f606eafd3a7742c1a Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 12:23:33 +0200 Subject: [PATCH 067/152] Tweak comment --- create_lmodrc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/create_lmodrc.py b/create_lmodrc.py index e06f8ccfa4..1720b762f0 100755 --- a/create_lmodrc.py +++ b/create_lmodrc.py @@ -33,8 +33,8 @@ def error(msg): error("Prefix directory %s does not exist!" % prefix) lmodrc_path = os.path.join(prefix, DOT_LMOD, 'lmodrc.lua') -# Lmod itself doesn't care about compute capability so remove this duplication from -# the install path (if it exists) +# Lmod itself doesn't care about the accelerator subdir so remove this duplication from +# the target path (if it exists) accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: lmodrc_path = lmodrc_path.replace("/accel/%s" % accel_subdir, '') From 74c382447bb5bccd3a467263a910fc5bad15bb00 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:31:58 +0200 Subject: [PATCH 068/152] {2023.06}[foss/2023a,zen4] WhatsHap v2.2 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 7ced0d9218..4bae944d45 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -10,3 +10,4 @@ easyconfigs: - LHAPDF-6.5.4-GCC-12.3.0.eb - LoopTools-2.15-GCC-12.3.0.eb - ncdu-1.18-GCC-12.3.0.eb + - WhatsHap-2.2-foss-2023a.eb From 973ee2d18079683dd683daae4e7ea069a3c29c02 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 15:14:04 +0200 Subject: [PATCH 069/152] Allow EESSI-extend to work even when LMOD_EXACT_MATCH is set --- EESSI-extend-2023.06-easybuild.eb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-extend-2023.06-easybuild.eb b/EESSI-extend-2023.06-easybuild.eb index b525ee462d..ba8629c02e 100644 --- a/EESSI-extend-2023.06-easybuild.eb +++ b/EESSI-extend-2023.06-easybuild.eb @@ -166,7 +166,7 @@ elseif (project_modulepath ~= nil) then end -- Make sure EasyBuild itself is loaded if not ( isloaded("EasyBuild") ) then - load("EasyBuild") + load(latest("EasyBuild")) end """ From 9f1971c52800881157098bde58e3bac82649c99c Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 15:15:27 +0200 Subject: [PATCH 070/152] Dont use hasattr on ec, since it's a dict, not an object --- eb_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 2b503f4f9d..4f9cd88bd3 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -304,10 +304,10 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): """ if ec.name == 'FreeImage' and ec.version in ('3.18.0',): if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': - if not hasattr(ec, 'toolchainopts'): + if toolchainopts not in ec: ec['toolchainopts'] = {} ec['toolchainopts']['pic'] = True - print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) + print_msg("Changed toolchainopts for %s: %s", ec.name, ec['toolchainopts']) def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): """ From eebd184890922cd205ae0c8c2e66892701e1491f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 15:25:31 +0200 Subject: [PATCH 071/152] Should be a string of course --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4f9cd88bd3..9cff30f939 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -304,7 +304,7 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): """ if ec.name == 'FreeImage' and ec.version in ('3.18.0',): if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': - if toolchainopts not in ec: + if 'toolchainopts' not in ec: ec['toolchainopts'] = {} ec['toolchainopts']['pic'] = True print_msg("Changed toolchainopts for %s: %s", ec.name, ec['toolchainopts']) From fc735871760159595a43ec5cbf12bf0679316db2 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 15:30:10 +0200 Subject: [PATCH 072/152] Also add the rebuild yml --- .../2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml new file mode 100644 index 0000000000..9cd1b451cd --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-EESSI-extend.yml @@ -0,0 +1,6 @@ +# 2024.09.25 +# EESSI-extend did not support LMOD_EXACT_MATCH +# (see https://github.com/EESSI/software-layer/pull/747) +easyconfigs: + - EESSI-extend-2023.06-easybuild.eb + From 150188043d361fa165e30cafe76eebb6122a709f Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 25 Sep 2024 17:05:41 +0200 Subject: [PATCH 073/152] Add ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..d9429ad8a8 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,5 @@ +easyconfigs: + - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c From 00157593ecc1795d58b82462b59a84f31ef00194 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 22:04:54 +0200 Subject: [PATCH 074/152] UCC CUDA rebuild now that we have an accel prefix --- .../rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml new file mode 100644 index 0000000000..a418086c44 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml @@ -0,0 +1,4 @@ +# 2024.09.19 +# We need to reinstall UCC-CUDA in the accelerator prefixes +easyconfigs: + - UCC-CUDA-1.2.0-GCCcore-12.3.0-CUDA-12.1.1.eb From 61c61237028ba23b3643fc2478a2e1071c86554f Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 22:58:45 +0200 Subject: [PATCH 075/152] Fix issue with missing symbol --- eb_hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 9cff30f939..0538650595 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -186,7 +186,7 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): ): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if cpu_target == CPU_TARGET_NEOVERSE_V1: - if not hasattr(ec, 'toolchainopts'): + if 'toolchainopts' not in ec: ec['toolchainopts'] = {} ec['toolchainopts']['vectorize'] = False print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) @@ -307,8 +307,10 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): if 'toolchainopts' not in ec: ec['toolchainopts'] = {} ec['toolchainopts']['pic'] = True + ec['toolchainopts']['extracflags'] = '-DPNG_ARM_NEON_OPT=0' print_msg("Changed toolchainopts for %s: %s", ec.name, ec['toolchainopts']) + def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): """ Remove x86_64 specific dependencies for the CI to pass on aarch64 From 4273dab9cafffa7f28c3d67e6d999caa5d08ce99 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 23:10:33 +0200 Subject: [PATCH 076/152] Use correct keyword --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 0538650595..a33b71a812 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -307,7 +307,7 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): if 'toolchainopts' not in ec: ec['toolchainopts'] = {} ec['toolchainopts']['pic'] = True - ec['toolchainopts']['extracflags'] = '-DPNG_ARM_NEON_OPT=0' + ec['toolchainopts']['extra_cflags'] = '-DPNG_ARM_NEON_OPT=0' print_msg("Changed toolchainopts for %s: %s", ec.name, ec['toolchainopts']) From d5870f5816fcfd6addb36219ad4e1e7d448c790f Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Thu, 26 Sep 2024 09:02:44 +0200 Subject: [PATCH 077/152] Update eessi-2023.06-eb-4.9.3-2023a.yml --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 4bae944d45..4f7f88f938 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -11,3 +11,4 @@ easyconfigs: - LoopTools-2.15-GCC-12.3.0.eb - ncdu-1.18-GCC-12.3.0.eb - WhatsHap-2.2-foss-2023a.eb + - PyOpenGL-3.1.7-GCCcore-12.3.0.eb From 22be85241b18bc2837bc3dabdc953f3cd46967fe Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 26 Sep 2024 14:27:46 +0200 Subject: [PATCH 078/152] Move CUDA builds to accel/ subdir --- .../eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/{2023.06/eessi-2023.06-eb-4.9.3-2023a.yml => accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml rename to easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml From d933c8c440197b61d94f7b60ed8e6af99197aba1 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 26 Sep 2024 14:43:44 +0200 Subject: [PATCH 079/152] Fix EasyBuild PR link --- .../accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml index d9429ad8a8..f41bc986ec 100644 --- a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml +++ b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -1,5 +1,5 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c From e20f1dde2c72d85a5beee41afe7a5a9a4191295b Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 26 Sep 2024 15:10:52 +0200 Subject: [PATCH 080/152] use subdir accel for CUDA builds --- .../eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023-CUDA.yml => accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml rename to easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml From 5915390cafe46e26f3d03a33691c53113b6ac752 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 26 Sep 2024 15:47:36 +0200 Subject: [PATCH 081/152] create nvidia directory --- .../accel/{ => nvidia}/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/accel/{ => nvidia}/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml (100%) diff --git a/easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml From 7368d39d3ebbe9370f59755eb4fb304ac76e3811 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 26 Sep 2024 20:40:04 +0200 Subject: [PATCH 082/152] use easystack file in accel/nvidia for ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb --- .../accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 4 ++++ .../accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 5 ----- 2 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml index 91c4b1f472..8935a3f3c3 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -1,2 +1,6 @@ easyconfigs: - LAMMPS-2Aug2023_update2-foss-2023a-kokkos-CUDA-12.1.1.eb + - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 + from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c diff --git a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml deleted file mode 100644 index f41bc986ec..0000000000 --- a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml +++ /dev/null @@ -1,5 +0,0 @@ -easyconfigs: - - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 - from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c From 0b5de15b937e525a465fbb23c3f1c91af532be1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 10:28:57 +0200 Subject: [PATCH 083/152] add mapping for cpu->accelerators and check for missing accelerator builds --- .github/workflows/test-software.eessi.io.yml | 36 +++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index d4d980901f..c4e3286f6c 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -2,11 +2,25 @@ name: Check for missing software installations in software.eessi.io on: push: - branches: [ "*-software.eessi.io" ] +# branches: [ "*-software.eessi.io" ] + branches: "*" pull_request: workflow_dispatch: permissions: contents: read # to fetch code (actions/checkout) +env: + EESSI_ACCELERATOR_TARGETS: | + aarch64/generic: + aarch64/neoverse_n1: + aarch64/neoverse_v1: + x86_64/generic: + x86_64/amd/zen2: + - nvidia/cc80 + x86_64/amd/zen3: + - nvidia/cc80 + x86_64/amd/zen4: + x86_64/intel/haswell: + x86_64/intel/skylake_avx512: jobs: check_missing: runs-on: ubuntu-latest @@ -48,6 +62,8 @@ jobs: export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux env | grep ^EESSI | sort + + # first check the CPU-only builds for this CPU target echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do echo "check missing installations for ${easystack_file}..." @@ -56,6 +72,24 @@ jobs: if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi done + # now check the accelerator builds for this CPU target + accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".${EESSI_SOFTWARE_SUBDIR_OVERRIDE}[]") + if [ -z ${accelerators} ]; then + echo "no accelerator targets defined for ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" + else + for accel in ${accelerators}; do + module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all + echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" + for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/nvidia/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + echo "check missing installations for ${easystack_file}..." + ./check_missing_installations.sh ${easystack_file} + ec=$? + if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi + done + module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all + done + fi + - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} From 4a5808c5f343b34190b0fd050023bd892ce23928 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 11:01:11 +0200 Subject: [PATCH 084/152] remove entries without an accelerator --- .github/workflows/test-software.eessi.io.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index c4e3286f6c..bad392aef8 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -10,17 +10,10 @@ permissions: contents: read # to fetch code (actions/checkout) env: EESSI_ACCELERATOR_TARGETS: | - aarch64/generic: - aarch64/neoverse_n1: - aarch64/neoverse_v1: - x86_64/generic: x86_64/amd/zen2: - nvidia/cc80 x86_64/amd/zen3: - nvidia/cc80 - x86_64/amd/zen4: - x86_64/intel/haswell: - x86_64/intel/skylake_avx512: jobs: check_missing: runs-on: ubuntu-latest From eaf656a9d38e1af007484589f4fee615c99d5778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 11:07:35 +0200 Subject: [PATCH 085/152] replace hardcoded nvidia in easystack dir by "dirname $accel" --- .github/workflows/test-software.eessi.io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index bad392aef8..48df5f9ca3 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -73,7 +73,7 @@ jobs: for accel in ${accelerators}; do module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" - for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/nvidia/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/$(dirname ${accel})/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From 3c5520a81832fa10071413ceb10a5b5c595b4e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 11:27:08 +0200 Subject: [PATCH 086/152] only run on *-software.eessi.io branches again --- .github/workflows/test-software.eessi.io.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 48df5f9ca3..e247c96676 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -2,8 +2,7 @@ name: Check for missing software installations in software.eessi.io on: push: -# branches: [ "*-software.eessi.io" ] - branches: "*" + branches: [ "*-software.eessi.io" ] pull_request: workflow_dispatch: permissions: From 45c59946f060bbbe7c9e4661dedaab99ceb11886 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 27 Sep 2024 12:39:00 +0000 Subject: [PATCH 087/152] {2023.06}[foss/2022b] vis-packages --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 5f63a7bc14..0f866b3aa0 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -32,3 +32,6 @@ easyconfigs: - KaHIP-3.14-gompi-2022b.eb - MPC-1.3.1-GCCcore-12.2.0.eb - MUMPS-5.6.1-foss-2022b-metis.eb + - GL2PS-1.4.2-GCCcore-12.2.0.eb + - GST-plugins-base-1.22.1-GCC-12.2.0.eb + - wxWidgets-3.2.2.1-GCC-12.2.0.eb From 225accc6dba397b07f1b8642864f8e1993ac050c Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 25 Sep 2024 09:51:34 +0200 Subject: [PATCH 088/152] {2023.06}[system] EasyBuild v4.9.4 --- .../2023.06/eessi-2023.06-eb-4.9.3-001-system.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml new file mode 100644 index 0000000000..d9c6075561 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-001-system.yml @@ -0,0 +1,5 @@ +easyconfigs: + - EasyBuild-4.9.4.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + from-commit: 39cdebd7bd2cb4a9c170ee22439401316b2e7a25 From f3820a1d43121d6aa8d9bdeb6bead0ff9c5bdbdc Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 24 Sep 2024 12:51:22 +0200 Subject: [PATCH 089/152] Limit CUDA hook to EESSI installs only, and remove duplication when creating symlinks --- eb_hooks.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index b3e457cfe3..e3c6c4faeb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -684,7 +684,8 @@ def post_sanitycheck_cuda(self, *args, **kwargs): Remove files from CUDA installation that we are not allowed to ship, and replace them with a symlink to a corresponding installation under host_injections. """ - if self.name == 'CUDA': + # Make sure we only do this for CUDA and only if we are doing a CVMFS installation + if self.name == 'CUDA' and self.installdir.startswith('/cvmfs/software.eessi.io/versions'): print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped @@ -733,6 +734,9 @@ def post_sanitycheck_cuda(self, *args, **kwargs): basename, full_path) # if it is not in the allowlist, delete the file and create a symlink to host_injections host_inj_path = full_path.replace('versions', 'host_injections') + # CUDA itself doesn't care about compute capability so remove this duplication from + # under host_injections + host_inj_path = re.sub(r"accel/nvidia/cc\d+/", '', host_inj_path) # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From a857bf11ccfdc9205f5f225574081a816c3d3800 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 08:41:25 +0200 Subject: [PATCH 090/152] Readd cuda reinstall --- .../20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml new file mode 100644 index 0000000000..755bea096e --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240918-eb-4.9.3-CUDA-12.1.1-in-accel-prefix.yml @@ -0,0 +1,7 @@ +# 2024.09.18 +# We need to reinstall CUDA in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/720 +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA From 1a72a4c2ca36c62d494516c1a8d7ec28341cdcc6 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 24 Sep 2024 15:46:08 +0200 Subject: [PATCH 091/152] Use EESSI_ACCELERATOR_TARGET rather than regex --- eb_hooks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index e3c6c4faeb..0a42a91d49 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -736,7 +736,9 @@ def post_sanitycheck_cuda(self, *args, **kwargs): host_inj_path = full_path.replace('versions', 'host_injections') # CUDA itself doesn't care about compute capability so remove this duplication from # under host_injections - host_inj_path = re.sub(r"accel/nvidia/cc\d+/", '', host_inj_path) + accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") + if accel_subdir: + host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, 'host_injections') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From 5d66e4ce96d9e673d286dfd781f137c5e9037a15 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 24 Sep 2024 16:00:32 +0200 Subject: [PATCH 092/152] Ensure we are making an EESSI install when using the CUDA hook --- eb_hooks.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 0a42a91d49..4d94357e8c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -685,7 +685,12 @@ def post_sanitycheck_cuda(self, *args, **kwargs): and replace them with a symlink to a corresponding installation under host_injections. """ # Make sure we only do this for CUDA and only if we are doing a CVMFS installation - if self.name == 'CUDA' and self.installdir.startswith('/cvmfs/software.eessi.io/versions'): + is_eessi_install = ( + self.installdir.startswith("/cvmfs/software.eessi.io/versions") + and not build_option("sanity_check_only") + and not build_option("module_only") + ) + if self.name == 'CUDA' and is_eessi_install: print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped From ae55491b75e77b1f445654a77d06394040a370aa Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 10:54:02 +0200 Subject: [PATCH 093/152] Move CUDA hook to post-install, allow hook to trigger for any EESSI distributed repo (but always make symlinks to software.eessi.io) --- eb_hooks.py | 76 +++++++++++++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 4d94357e8c..788a0419ba 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -131,7 +131,8 @@ def pre_prepare_hook(self, *args, **kwargs): def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs): """ Post-configure hook for GCCcore: - - copy RPATH wrapper script for linker commands to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu' + - copy RPATH wrapper script for linker commands to also have a wrapper in + place with system type prefix like 'x86_64-pc-linux-gnu' """ if self.name == 'GCCcore': config_guess = obtain_config_guess() @@ -279,10 +280,10 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix): Disable check for QtWebEngine in Qt5 as workaround for problem with determining glibc version. """ if ec.name == 'Qt5': - # workaround for glibc version being reported as "UNKNOWN" in Gentoo Prefix environment by EasyBuild v4.7.2, - # see also https://github.com/easybuilders/easybuild-framework/pull/4290 - ec['check_qtwebengine'] = False - print_msg("Checking for QtWebEgine in Qt5 installation has been disabled") + # workaround for glibc version being reported as "UNKNOWN" in Gentoo Prefix environment by EasyBuild v4.7.2, + # see also https://github.com/easybuilders/easybuild-framework/pull/4290 + ec['check_qtwebengine'] = False + print_msg("Checking for QtWebEgine in Qt5 installation has been disabled") else: raise EasyBuildError("Qt5-specific hook triggered for non-Qt5 easyconfig?!") @@ -341,7 +342,7 @@ def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwarg if self.name == 'Highway': tcname, tcversion = self.toolchain.name, self.toolchain.version cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - # note: keep condition in sync with the one used in + # note: keep condition in sync with the one used in # post_prepare_hook_highway_handle_test_compilation_issues if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': if cpu_target in [CPU_TARGET_A64FX, CPU_TARGET_NEOVERSE_V1]: @@ -360,12 +361,13 @@ def post_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwar if self.name == 'Highway': tcname, tcversion = self.toolchain.name, self.toolchain.version cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') - # note: keep condition in sync with the one used in + # note: keep condition in sync with the one used in # pre_prepare_hook_highway_handle_test_compilation_issues if self.version in ['1.0.4'] and tcname == 'GCCcore' and tcversion == '12.3.0': if cpu_target == CPU_TARGET_NEOVERSE_N1: update_build_option('optarch', self.orig_optarch) + def pre_configure_hook(self, *args, **kwargs): """Main pre-configure hook: trigger custom functions based on software name.""" if self.name in PRE_CONFIGURE_HOOKS: @@ -389,6 +391,7 @@ def pre_configure_hook_BLIS_a64fx(self, *args, **kwargs): else: raise EasyBuildError("BLIS-specific hook triggered for non-BLIS easyconfig?!") + def pre_configure_hook_extrae(self, *args, **kwargs): """ Pre-configure hook for Extrae @@ -414,7 +417,11 @@ def pre_configure_hook_extrae(self, *args, **kwargs): # replace use of 'which' with 'command -v', since 'which' is broken in EESSI build container; # this must be done *after* running configure script, because initial configuration re-writes configure script, # and problem due to use of which only pops up when running make ?! - self.cfg.update('prebuildopts', "cp config/mpi-macros.m4 config/mpi-macros.m4.orig && sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && ") + self.cfg.update( + 'prebuildopts', + "cp config/mpi-macros.m4 config/mpi-macros.m4.orig &&" + "sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && " + ) else: raise EasyBuildError("Extrae-specific hook triggered for non-Extrae easyconfig?!") @@ -445,7 +452,10 @@ def pre_configure_hook_gromacs(self, *args, **kwargs): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if LooseVersion(self.version) <= LooseVersion('2024.1') and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg.update('configopts', '-DGMX_SIMD=ARM_NEON_ASIMD') - print_msg("Avoiding use of SVE instructions for GROMACS %s by using ARM_NEON_ASIMD as GMX_SIMD value", self.version) + print_msg( + "Avoiding use of SVE instructions for GROMACS %s by using ARM_NEON_ASIMD as GMX_SIMD value", + self.version + ) else: raise EasyBuildError("GROMACS-specific hook triggered for non-GROMACS easyconfig?!") @@ -506,12 +516,12 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): pattern = "Linux x86_64 ppc64le, gfortran" repl = "Linux x86_64 aarch64 ppc64le, gfortran" if LooseVersion(self.version) <= LooseVersion('3.9.0'): - self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl)) - print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) + self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl)) + print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) if LooseVersion('4.0.0') <= LooseVersion(self.version) <= LooseVersion('4.2.1'): - self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl)) - print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) + self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl)) + print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts']) else: raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") @@ -533,7 +543,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") -def pre_test_hook(self,*args, **kwargs): +def pre_test_hook(self, *args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: PRE_TEST_HOOKS[self.name](self, *args, **kwargs) @@ -596,6 +606,7 @@ def pre_test_hook_ignore_failing_tests_SciPybundle(self, *args, **kwargs): elif cpu_target == CPU_TARGET_A64FX and self.version in scipy_bundle_versions_a64fx: self.cfg['testopts'] = "|| echo ignoring failing tests" + def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): """ Pre-test hook for netCDF: skip failing tests for selected netCDF versions on neoverse_v1 @@ -609,6 +620,7 @@ def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs): if self.name == 'netCDF' and self.version == '4.9.2' and cpu_target == CPU_TARGET_NEOVERSE_V1: self.cfg['testopts'] = "|| echo ignoring failing tests" + def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs): """ Pre-test hook for PyTorch: increase max failing tests for ARM for PyTorch 2.1.2 @@ -673,24 +685,24 @@ def pre_single_extension_testthat(ext, *args, **kwargs): ext.cfg['preinstallopts'] = "sed -i 's/SIGSTKSZ/32768/g' inst/include/testthat/vendor/catch.h && " -def post_sanitycheck_hook(self, *args, **kwargs): - """Main post-sanity-check hook: trigger custom functions based on software name.""" - if self.name in POST_SANITYCHECK_HOOKS: - POST_SANITYCHECK_HOOKS[self.name](self, *args, **kwargs) +def post_postproc_hook(self, *args, **kwargs): + """Main post-postprocessing hook: trigger custom functions based on software name.""" + if self.name in POST_POSTPROC_HOOKS: + POST_POSTPROC_HOOKS[self.name](self, *args, **kwargs) -def post_sanitycheck_cuda(self, *args, **kwargs): +def post_postproc_cuda(self, *args, **kwargs): """ Remove files from CUDA installation that we are not allowed to ship, and replace them with a symlink to a corresponding installation under host_injections. """ - # Make sure we only do this for CUDA and only if we are doing a CVMFS installation - is_eessi_install = ( - self.installdir.startswith("/cvmfs/software.eessi.io/versions") - and not build_option("sanity_check_only") - and not build_option("module_only") - ) - if self.name == 'CUDA' and is_eessi_install: + + # We need to check if we are doing an EESSI-distributed installation + eessi_pattern = r"^/cvmfs/[^/]*.eessi.io/versions/" + host_injections_location = "/cvmfs/software.eessi.io/host_injections/" + eessi_installation = bool(re.search(eessi_pattern, self.installdir)) + + if self.name == 'CUDA' and eessi_installation: print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") # read CUDA EULA, construct allowlist based on section 2.6 that specifies list of files that can be shipped @@ -738,12 +750,14 @@ def post_sanitycheck_cuda(self, *args, **kwargs): self.log.debug("%s is not found in allowlist, so replacing it with symlink: %s", basename, full_path) # if it is not in the allowlist, delete the file and create a symlink to host_injections - host_inj_path = full_path.replace('versions', 'host_injections') + + # the host_injections path is under a fixed repo/location for CUDA + host_inj_path = re.sub(eessi_pattern, host_injections_location, full_path) # CUDA itself doesn't care about compute capability so remove this duplication from # under host_injections accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: - host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, 'host_injections') + host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, '') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " @@ -775,7 +789,7 @@ def inject_gpu_property(ec): ec_dict['builddependencies'].append(dep) value = '\n'.join([value, 'setenv("EESSICUDAVERSION","%s")' % cuda_version]) if key in ec_dict: - if not value in ec_dict[key]: + if value not in ec_dict[key]: ec[key] = '\n'.join([ec_dict[key], value]) else: ec[key] = value @@ -835,6 +849,6 @@ def inject_gpu_property(ec): 'numpy': post_single_extension_numpy, } -POST_SANITYCHECK_HOOKS = { - 'CUDA': post_sanitycheck_cuda, +POST_POSTPROC_HOOKS = { + 'CUDA': post_postproc_cuda, } From 7a95dd2b980b4e6796b3e7e1295b44417aa36d79 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 24 Sep 2024 16:08:37 +0200 Subject: [PATCH 094/152] Update eb_hooks.py --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 788a0419ba..a8182df971 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -757,7 +757,7 @@ def post_postproc_cuda(self, *args, **kwargs): # under host_injections accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: - host_inj_path = host_inj_path.replace('/accel/%s' % accel_subdir, '') + host_inj_path = host_inj_path.replace("/accel/%s" % accel_subdir, '') # make sure source and target of symlink are not the same if full_path == host_inj_path: raise EasyBuildError("Source (%s) and target (%s) are the same location, are you sure you " From 64a0f378fb7bdf12537d99b9b45d0a9623f18575 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Wed, 25 Sep 2024 11:14:29 +0200 Subject: [PATCH 095/152] Address review comments --- eb_hooks.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index a8182df971..9b0e9c8dcb 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -32,6 +32,9 @@ SYSTEM = EASYCONFIG_CONSTANTS['SYSTEM'][0] +EESSI_INSTALLATION_REGEX = r"^/cvmfs/[^/]*.eessi.io/versions/" +HOST_INJECTIONS_LOCATION = "/cvmfs/software.eessi.io/host_injections/" + def get_eessi_envvar(eessi_envvar): """Get an EESSI environment variable from the environment""" @@ -419,7 +422,7 @@ def pre_configure_hook_extrae(self, *args, **kwargs): # and problem due to use of which only pops up when running make ?! self.cfg.update( 'prebuildopts', - "cp config/mpi-macros.m4 config/mpi-macros.m4.orig &&" + "cp config/mpi-macros.m4 config/mpi-macros.m4.orig && " "sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && " ) else: @@ -698,9 +701,7 @@ def post_postproc_cuda(self, *args, **kwargs): """ # We need to check if we are doing an EESSI-distributed installation - eessi_pattern = r"^/cvmfs/[^/]*.eessi.io/versions/" - host_injections_location = "/cvmfs/software.eessi.io/host_injections/" - eessi_installation = bool(re.search(eessi_pattern, self.installdir)) + eessi_installation = bool(re.search(EESSI_INSTALLATION_REGEX, self.installdir)) if self.name == 'CUDA' and eessi_installation: print_msg("Replacing files in CUDA installation that we can not ship with symlinks to host_injections...") @@ -752,9 +753,10 @@ def post_postproc_cuda(self, *args, **kwargs): # if it is not in the allowlist, delete the file and create a symlink to host_injections # the host_injections path is under a fixed repo/location for CUDA - host_inj_path = re.sub(eessi_pattern, host_injections_location, full_path) + host_inj_path = re.sub(EESSI_INSTALLATION_REGEX, HOST_INJECTIONS_LOCATION, full_path) # CUDA itself doesn't care about compute capability so remove this duplication from - # under host_injections + # under host_injections (symlink to a single CUDA installation for all compute + # capabilities) accel_subdir = os.getenv("EESSI_ACCELERATOR_TARGET") if accel_subdir: host_inj_path = host_inj_path.replace("/accel/%s" % accel_subdir, '') From a5ca128e550c4afaac678fabcc20c843e31a59f3 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Wed, 25 Sep 2024 17:05:41 +0200 Subject: [PATCH 096/152] Add ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..d9429ad8a8 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,5 @@ +easyconfigs: + - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c From 730869a1a336514246c9c4e73c075247356a108b Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 26 Sep 2024 14:27:46 +0200 Subject: [PATCH 097/152] Move CUDA builds to accel/ subdir --- .../eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/{2023.06/eessi-2023.06-eb-4.9.3-2023a.yml => accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml rename to easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml From d0259dcdcff0c451c61683323b8e75feb9ed4ff0 Mon Sep 17 00:00:00 2001 From: Neves-P Date: Thu, 26 Sep 2024 14:43:44 +0200 Subject: [PATCH 098/152] Fix EasyBuild PR link --- .../accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml index d9429ad8a8..f41bc986ec 100644 --- a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml +++ b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -1,5 +1,5 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21465 + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c From c2cdf3b7867ff9af0ab77fae7de39c58336fd8c2 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 18 Sep 2024 12:45:55 +0200 Subject: [PATCH 099/152] {2023.06}[2023a,GPU] LAMMPS 2Aug2023 CUDA 12.1.1 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml new file mode 100644 index 0000000000..91c4b1f472 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - LAMMPS-2Aug2023_update2-foss-2023a-kokkos-CUDA-12.1.1.eb From ea6fdec621655d8a689e3940d4eb98e5160e8973 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:29:31 +0200 Subject: [PATCH 100/152] UCX cuda 1.14.1 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml new file mode 100644 index 0000000000..bfe4a4dc52 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - UCX-CUDA-1.14.1-GCCcore-12.3.0-CUDA-12.1.1.eb From 1e19767896d3b58d5d4f36b73ea990c54b50446e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:34:39 +0200 Subject: [PATCH 101/152] Move to rebuilds --- .../20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023a-CUDA.yml => rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml From 8b3a3cde4a2ee51f044a7c7cd4999b80b871d5ac Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:49:58 +0200 Subject: [PATCH 102/152] Add comment --- .../rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml index bfe4a4dc52..d347af335a 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-uxc-cuda-in-accel-prefix.yml @@ -1,2 +1,5 @@ +# 2024.09.19 +# We need to reinstall UCX-CUDA in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/719 easyconfigs: - UCX-CUDA-1.14.1-GCCcore-12.3.0-CUDA-12.1.1.eb From 25475f789c3d40b25f389f58cfcfd46761094b15 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:05:17 +0200 Subject: [PATCH 103/152] Add CUDA-Samples for the accelerator prefix --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml new file mode 100644 index 0000000000..01a47bbc99 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb From f7f2e26fa0f05f7bd3fe9bca137474456fd41433 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:31:39 +0200 Subject: [PATCH 104/152] Move to rebuilds --- .../20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023a-CUDA.yml => rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml From c30842df015a8f6217ab694f982778dce1a0d03e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:49:20 +0200 Subject: [PATCH 105/152] Add comment --- .../20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml index 01a47bbc99..da2c06ae1e 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-Cuda-Samples-in-accel-prefix.yml @@ -1,2 +1,5 @@ +# 2024.09.19 +# We need to reinstall CUDA-Samples in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/715 easyconfigs: - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb From 1d91f4836b848e97416cd6ba0bd878501e8062e1 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 20 Sep 2024 08:23:46 +0200 Subject: [PATCH 106/152] {2023.06}[foss/2023a] NCCL v2.18.3 w/ CUDA 12.1.1 --- .../20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml new file mode 100644 index 0000000000..6ff3b1bfbf --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml @@ -0,0 +1,5 @@ +# 2024.09.19 +# We need to reinstall NCCL in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/487 +easyconfigs: + - NCCL-2.18.3-GCCcore-12.3.0-CUDA-12.1.1.eb From b09b90fb8b53b452e27945c34322328aa8f3776a Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 25 Sep 2024 10:05:21 +0200 Subject: [PATCH 107/152] {2023.06}[foss/2023a] NCCL 2.18.3 w/ CUDA 12.1.1 --- ...ix.yml => 20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename easystacks/software.eessi.io/2023.06/rebuilds/{20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml => 20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml} (92%) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml similarity index 92% rename from easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml index 6ff3b1bfbf..d6667af9a1 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240920-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.4-NCCL-2.18.3-in-accel-prefix.yml @@ -1,4 +1,4 @@ -# 2024.09.19 +# 2024.09.25 # We need to reinstall NCCL in the accelerator prefixes # See https://github.com/EESSI/software-layer/pull/487 easyconfigs: From 6c63de6b8913d76b9f0039df772d77da1c6b1bee Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 22:04:54 +0200 Subject: [PATCH 108/152] UCC CUDA rebuild now that we have an accel prefix --- .../rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml new file mode 100644 index 0000000000..a418086c44 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240925-eb-4.9.3-ucc-cuda-in-accel-prefix.yml @@ -0,0 +1,4 @@ +# 2024.09.19 +# We need to reinstall UCC-CUDA in the accelerator prefixes +easyconfigs: + - UCC-CUDA-1.2.0-GCCcore-12.3.0-CUDA-12.1.1.eb From 9182b21d95d67a1f5604378d2d50d3bfba990057 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Tue, 24 Sep 2024 16:12:22 +0200 Subject: [PATCH 109/152] Add various tools --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml new file mode 100644 index 0000000000..d3927ea3f1 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -0,0 +1,6 @@ +easyconfigs: + - ccache-4.9-GCCcore-12.3.0.eb + - GDB-13.2-GCCcore-12.3.0.eb + - mold-1.11.0-GCCcore-12.3.0.eb + - tmux-3.3a-GCCcore-12.3.0.eb + - Vim-9.1.0004-GCCcore-12.3.0.eb From e181237f32b9c11a0cd5081703c8e31bb8f7f4ca Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 25 Sep 2024 11:10:51 +0200 Subject: [PATCH 110/152] Remove mold, it's a complicated case since it is a linker, and it's not urgent now --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml index d3927ea3f1..0a4a4e5554 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -1,6 +1,5 @@ easyconfigs: - ccache-4.9-GCCcore-12.3.0.eb - GDB-13.2-GCCcore-12.3.0.eb - - mold-1.11.0-GCCcore-12.3.0.eb - tmux-3.3a-GCCcore-12.3.0.eb - Vim-9.1.0004-GCCcore-12.3.0.eb From c906e0c1e2e97970fb207ec7d17d598aa0c22a9e Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 26 Sep 2024 15:10:52 +0200 Subject: [PATCH 111/152] use subdir accel for CUDA builds --- .../eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023-CUDA.yml => accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023-CUDA.yml rename to easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml From 7d59905789554b13f464ba452edba5c330ac5bf0 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 26 Sep 2024 15:47:36 +0200 Subject: [PATCH 112/152] create nvidia directory --- .../accel/{ => nvidia}/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/accel/{ => nvidia}/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml (100%) diff --git a/easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml From 31b37d609eaa50914868cd0bd29abb5def416b22 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 18 Sep 2024 21:10:44 +0200 Subject: [PATCH 113/152] Add OSU Microbenchmarks 7.2 --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml new file mode 100644 index 0000000000..cccbfa6808 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb From 0f159b4bfdca7bfd43584a059fb321aca02e3613 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:44:42 +0200 Subject: [PATCH 114/152] Move into rebuilds --- .../20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{eessi-2023.06-eb-4.9.3-2023a-CUDA.yml => rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml} (100%) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml rename to easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml From 5d85411d63199b3de4a21bfa57879294404cdcec Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Thu, 19 Sep 2024 10:45:44 +0200 Subject: [PATCH 115/152] Add comment --- .../20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml index cccbfa6808..23801e0250 100644 --- a/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240919-eb-4.9.3-osu-microbenchmarks-in-accel-prefix.yml @@ -1,2 +1,5 @@ +# 2024.09.19 +# We need to reinstall OSU-Micro-Benchmarks in the accelerator prefixes +# See https://github.com/EESSI/software-layer/pull/716 easyconfigs: - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb From 925b48e7e5e84b2b39258659d050ceb6776a0b75 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Thu, 26 Sep 2024 20:40:04 +0200 Subject: [PATCH 116/152] use easystack file in accel/nvidia for ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb --- .../accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 4 ++++ .../accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml | 5 ----- 2 files changed, 4 insertions(+), 5 deletions(-) delete mode 100644 easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml index 91c4b1f472..8935a3f3c3 100644 --- a/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml +++ b/easystacks/software.eessi.io/2023.06/accel/nvidia/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml @@ -1,2 +1,6 @@ easyconfigs: - LAMMPS-2Aug2023_update2-foss-2023a-kokkos-CUDA-12.1.1.eb + - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 + from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c diff --git a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml b/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml deleted file mode 100644 index f41bc986ec..0000000000 --- a/easystacks/software.eessi.io/accel/eessi-2023.06-eb-4.9.3-2023a-CUDA.yml +++ /dev/null @@ -1,5 +0,0 @@ -easyconfigs: - - ESPResSo-4.2.2-foss-2023a-CUDA-12.1.1.eb: - options: - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21440 - from-commit: 5525968921d7b5eae54f7d16391201e17ffae13c From 46ee666d435d4e2e50d6ecbf5e463a13bdca6342 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 10:28:57 +0200 Subject: [PATCH 117/152] add mapping for cpu->accelerators and check for missing accelerator builds --- .github/workflows/test-software.eessi.io.yml | 36 +++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index d4d980901f..c4e3286f6c 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -2,11 +2,25 @@ name: Check for missing software installations in software.eessi.io on: push: - branches: [ "*-software.eessi.io" ] +# branches: [ "*-software.eessi.io" ] + branches: "*" pull_request: workflow_dispatch: permissions: contents: read # to fetch code (actions/checkout) +env: + EESSI_ACCELERATOR_TARGETS: | + aarch64/generic: + aarch64/neoverse_n1: + aarch64/neoverse_v1: + x86_64/generic: + x86_64/amd/zen2: + - nvidia/cc80 + x86_64/amd/zen3: + - nvidia/cc80 + x86_64/amd/zen4: + x86_64/intel/haswell: + x86_64/intel/skylake_avx512: jobs: check_missing: runs-on: ubuntu-latest @@ -48,6 +62,8 @@ jobs: export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}} export EESSI_OS_TYPE=linux env | grep ^EESSI | sort + + # first check the CPU-only builds for this CPU target echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do echo "check missing installations for ${easystack_file}..." @@ -56,6 +72,24 @@ jobs: if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi done + # now check the accelerator builds for this CPU target + accelerators=$(echo "${EESSI_ACCELERATOR_TARGETS}" | yq ".${EESSI_SOFTWARE_SUBDIR_OVERRIDE}[]") + if [ -z ${accelerators} ]; then + echo "no accelerator targets defined for ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" + else + for accel in ${accelerators}; do + module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all + echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" + for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/nvidia/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + echo "check missing installations for ${easystack_file}..." + ./check_missing_installations.sh ${easystack_file} + ec=$? + if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi + done + module unuse ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all + done + fi + - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} From bd0edd18d33ed4e1ea21f9c09fe7cb07546e30af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 11:01:11 +0200 Subject: [PATCH 118/152] remove entries without an accelerator --- .github/workflows/test-software.eessi.io.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index c4e3286f6c..bad392aef8 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -10,17 +10,10 @@ permissions: contents: read # to fetch code (actions/checkout) env: EESSI_ACCELERATOR_TARGETS: | - aarch64/generic: - aarch64/neoverse_n1: - aarch64/neoverse_v1: - x86_64/generic: x86_64/amd/zen2: - nvidia/cc80 x86_64/amd/zen3: - nvidia/cc80 - x86_64/amd/zen4: - x86_64/intel/haswell: - x86_64/intel/skylake_avx512: jobs: check_missing: runs-on: ubuntu-latest From af7dfc31b8f950df11d35d2ec964e8c68f1e393d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 11:07:35 +0200 Subject: [PATCH 119/152] replace hardcoded nvidia in easystack dir by "dirname $accel" --- .github/workflows/test-software.eessi.io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index bad392aef8..48df5f9ca3 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -73,7 +73,7 @@ jobs: for accel in ${accelerators}; do module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}" - for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/nvidia/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/accel/$(dirname ${accel})/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From 645ea271a4f4781ed9b2781964fb1a3b8d6b08b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= Date: Fri, 27 Sep 2024 11:27:08 +0200 Subject: [PATCH 120/152] only run on *-software.eessi.io branches again --- .github/workflows/test-software.eessi.io.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 48df5f9ca3..e247c96676 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -2,8 +2,7 @@ name: Check for missing software installations in software.eessi.io on: push: -# branches: [ "*-software.eessi.io" ] - branches: "*" + branches: [ "*-software.eessi.io" ] pull_request: workflow_dispatch: permissions: From c81857074772b906890b9bee76b25f478b241a6c Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 15:08:15 +0200 Subject: [PATCH 121/152] {2023.06}[foss/2023a,zen4] SAMtools v1.18 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 4bae944d45..21b1886555 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -11,3 +11,4 @@ easyconfigs: - LoopTools-2.15-GCC-12.3.0.eb - ncdu-1.18-GCC-12.3.0.eb - WhatsHap-2.2-foss-2023a.eb + - SAMtools-1.18-GCC-12.3.0.eb From 0e19946274fbedce25bd7b05d6ee627daccff410 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 15:30:20 +0200 Subject: [PATCH 122/152] update eb_hook so that EasyBuildError is not not trigered for unlisted LAMMPS versions --- eb_hooks.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index d999c735c3..f7d19235c4 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -301,15 +301,16 @@ def parse_hook_lammps_remove_deps_for_aarch64(ec, *args, **kwargs): """ Remove x86_64 specific dependencies for the CI and missing installations to pass on aarch64 """ - if ec.name == 'LAMMPS' and ec.version in ('2Aug2023_update2', '29Aug2024'): - if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': - # ScaFaCoS and tbb are not compatible with aarch64/* CPU targets, - # so remove them as dependencies for LAMMPS (they're optional); - # see also https://github.com/easybuilders/easybuild-easyconfigs/pull/19164 + - # https://github.com/easybuilders/easybuild-easyconfigs/pull/19000; - # we need this hook because we check for missing installations for all CPU targets - # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true) - ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('ScaFaCoS', 'tbb',)] + if ec.name == 'LAMMPS': + if ec.version in ('2Aug2023_update2', '29Aug2024'): + if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': + # ScaFaCoS and tbb are not compatible with aarch64/* CPU targets, + # so remove them as dependencies for LAMMPS (they're optional); + # see also https://github.com/easybuilders/easybuild-easyconfigs/pull/19164 + + # https://github.com/easybuilders/easybuild-easyconfigs/pull/19000; + # we need this hook because we check for missing installations for all CPU targets + # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true) + ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('ScaFaCoS', 'tbb',)] else: raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") From aa675ab20690704a69750a1ac18216f7921dffd6 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 16:09:00 +0200 Subject: [PATCH 123/152] add missing check for zen4 --- .github/workflows/test-software.eessi.io.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index e247c96676..e774180d99 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -27,6 +27,7 @@ jobs: - aarch64/neoverse_v1 - x86_64/amd/zen2 - x86_64/amd/zen3 + - x86_64/amd/zen4 - x86_64/intel/haswell - x86_64/intel/skylake_avx512 - x86_64/generic @@ -58,6 +59,9 @@ jobs: # first check the CPU-only builds for this CPU target echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do + if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ] && grep -q 2022b <<<"${easystack_file}"; then + continue + fi echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} ec=$? From aa5fade441cf052a65fe197f56d4da0b72279668 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 27 Sep 2024 18:24:56 +0200 Subject: [PATCH 124/152] set kokkos hoop for zen4 Zen4 nodes in Azure are not detected as `zen4` by archspec --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index f7d19235c4..8a357a8cde 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -525,7 +525,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'LAMMPS': - if self.version == '2Aug2023_update2': + if ec.version in ('2Aug2023_update2', '29Aug2024'): if get_cpu_architecture() == X86_64: if cpu_target == CPU_TARGET_ZEN4: # There is no support for ZEN4 in LAMMPS yet so falling back to ZEN3 From abbbf35db63271508a60dd003c2cfe94b2f691ce Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 27 Sep 2024 19:32:17 +0200 Subject: [PATCH 125/152] fix hook --- eb_hooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eb_hooks.py b/eb_hooks.py index 8a357a8cde..4d13523e40 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -525,7 +525,7 @@ def pre_configure_hook_LAMMPS_zen4(self, *args, **kwargs): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if self.name == 'LAMMPS': - if ec.version in ('2Aug2023_update2', '29Aug2024'): + if self.version in ('2Aug2023_update2', '29Aug2024'): if get_cpu_architecture() == X86_64: if cpu_target == CPU_TARGET_ZEN4: # There is no support for ZEN4 in LAMMPS yet so falling back to ZEN3 From 748300c7cdb9df39bf75b01d4265b388f8a92246 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 19:43:54 +0200 Subject: [PATCH 126/152] {2023.06}[system,zen4] EasyBuild v4.9.0 and v4.8.2 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml index 1e30631e57..25337649ce 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-001-system.yml @@ -1,2 +1,4 @@ easyconfigs: - Nextflow-23.10.0.eb + - EasyBuild-4.8.2.eb + - EasyBuild-4.9.0.eb From 64b7958071fe672499e0b4f571be0805cac80ea7 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 27 Sep 2024 21:26:03 +0200 Subject: [PATCH 127/152] exclude CUDA builds in CPU path --- .github/workflows/test-software.eessi.io.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index e774180d99..39972ef18f 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -59,7 +59,7 @@ jobs: # first check the CPU-only builds for this CPU target echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do - if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ] && grep -q 2022b <<<"${easystack_file}"; then + if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ] && grep -q 2022b <<<"${easystack_file}" || grep -q CUDA <<<"${easystack_file}"; then continue fi echo "check missing installations for ${easystack_file}..." From ede35425aa97cc898997c939b1f0036cc0b8797d Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Fri, 27 Sep 2024 21:43:46 +0200 Subject: [PATCH 128/152] fix conditional for CUDA --- .github/workflows/test-software.eessi.io.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 39972ef18f..29af8f57f1 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -59,8 +59,12 @@ jobs: # first check the CPU-only builds for this CPU target echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)" for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do - if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ] && grep -q 2022b <<<"${easystack_file}" || grep -q CUDA <<<"${easystack_file}"; then - continue + if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then + if grep -q 2022b <<<"${easystack_file}"; then + continue + elif grep -q CUDA <<<"${easystack_file}"; then + continue + fi fi echo "check missing installations for ${easystack_file}..." ./check_missing_installations.sh ${easystack_file} From 33f5f4cb6112771c96a3220ad18fa2b66e9e59e0 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 21:50:54 +0200 Subject: [PATCH 129/152] {2023.06}[2023a,zen4] OSU-Micro-Benchmarks v7.1-1 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 48ac47cbfb..4153b4a5e4 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -13,3 +13,4 @@ easyconfigs: - WhatsHap-2.2-foss-2023a.eb - PyOpenGL-3.1.7-GCCcore-12.3.0.eb - SAMtools-1.18-GCC-12.3.0.eb + - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb From 60f0d10f0931e0dfe0ac4595f586c76614d4a71b Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 21:56:40 +0200 Subject: [PATCH 130/152] {2023.06}[gompi/2023a] CDO 2.2.2 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 48ac47cbfb..d787241397 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -13,3 +13,4 @@ easyconfigs: - WhatsHap-2.2-foss-2023a.eb - PyOpenGL-3.1.7-GCCcore-12.3.0.eb - SAMtools-1.18-GCC-12.3.0.eb + - CDO-2.2.2-gompi-2023a.eb From 3e5fefb1b33af1574f09b9146f71c7bae57d7c61 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 27 Sep 2024 23:22:22 +0200 Subject: [PATCH 131/152] {2023.06}{GCCcore/12.3.0,zen4} BWA 0.7.17.20220923 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 7b93c754c3..730594a22b 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -15,3 +15,4 @@ easyconfigs: - SAMtools-1.18-GCC-12.3.0.eb - CDO-2.2.2-gompi-2023a.eb - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb + - BWA-0.7.17-20220923-GCCcore-12.3.0.eb From a0712df5ba08bc84f285871e22b9f71dc730d31d Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Mon, 30 Sep 2024 09:01:02 +0200 Subject: [PATCH 132/152] move CUDA installations in CPU-only paths to dedicated easyconfigs --- .../2023.06/eessi-2023.06-eb-4.8.2-2023a-CUDA.yml | 9 +++++++++ .../2023.06/eessi-2023.06-eb-4.8.2-2023a.yml | 8 -------- .../2023.06/eessi-2023.06-eb-4.9.0-2023a-CUDA.yml | 2 ++ .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 1 - 4 files changed, 11 insertions(+), 9 deletions(-) create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a-CUDA.yml create mode 100644 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a-CUDA.yml diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a-CUDA.yml new file mode 100644 index 0000000000..f8bde420de --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a-CUDA.yml @@ -0,0 +1,9 @@ +easyconfigs: + - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb: + # use easyconfig that only install subset of CUDA samples, + # to circumvent problem with nvcc linking to glibc of host OS, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19189; + # and where additional samples are excluded because they fail to build on aarch64, + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19451; + options: + from-pr: 19451 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml index 7244219dc3..43b081b122 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-2023a.yml @@ -35,14 +35,6 @@ easyconfigs: - Boost-1.82.0-GCC-12.3.0.eb - netCDF-4.9.2-gompi-2023a.eb - FFmpeg-6.0-GCCcore-12.3.0.eb - - CUDA-Samples-12.1-GCC-12.3.0-CUDA-12.1.1.eb: - # use easyconfig that only install subset of CUDA samples, - # to circumvent problem with nvcc linking to glibc of host OS, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19189; - # and where additional samples are excluded because they fail to build on aarch64, - # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19451; - options: - from-pr: 19451 - ALL-0.9.2-foss-2023a.eb: options: from-pr: 19455 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a-CUDA.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a-CUDA.yml new file mode 100644 index 0000000000..cccbfa6808 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a-CUDA.yml @@ -0,0 +1,2 @@ +easyconfigs: + - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 4b58cb6106..3f6590c3cd 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -34,7 +34,6 @@ easyconfigs: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19996 from-pr: 19996 - dask-2023.9.2-foss-2023a.eb - - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb: options: From 7234585d806171868e7f7304f034599d952978c4 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Mon, 30 Sep 2024 07:10:44 +0000 Subject: [PATCH 133/152] {2023.06}[GCCcore/12.2.0] libs-tools --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 0f866b3aa0..ca35e95bd4 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -35,3 +35,9 @@ easyconfigs: - GL2PS-1.4.2-GCCcore-12.2.0.eb - GST-plugins-base-1.22.1-GCC-12.2.0.eb - wxWidgets-3.2.2.1-GCC-12.2.0.eb + - Archive-Zip-1.68-GCCcore-12.2.0.eb + - jemalloc-5.3.0-GCCcore-12.2.0.eb + - Judy-1.0.5-GCCcore-12.2.0.eb + - libaio-0.3.113-GCCcore-12.2.0.eb + - Z3-4.12.2-GCCcore-12.2.0.eb + - tbb-2021.10.0-GCCcore-12.2.0.eb From a9a9c3925a36443254ea3b82c069b74ca68c495a Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 30 Sep 2024 10:22:27 +0200 Subject: [PATCH 134/152] {2023.06}[2023a,2023b,zen4] Valgrind v3.21.0 and v3.23.0 for zen4 --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml | 1 + .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml index 730594a22b..519d7701dc 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023a.yml @@ -16,3 +16,4 @@ easyconfigs: - CDO-2.2.2-gompi-2023a.eb - OSU-Micro-Benchmarks-7.1-1-gompi-2023a.eb - BWA-0.7.17-20220923-GCCcore-12.3.0.eb + - Valgrind-3.21.0-gompi-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml new file mode 100644 index 0000000000..c33ed99240 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - Valgrind-3.23.0-gompi-2023b.eb From ecd8853f0f5a3ab5138c4211f606b1922fadf28c Mon Sep 17 00:00:00 2001 From: lara Date: Mon, 30 Sep 2024 10:58:34 +0200 Subject: [PATCH 135/152] remove because Valgrind on 2023b is already build --- .../2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml deleted file mode 100644 index c33ed99240..0000000000 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.3-2023b.yml +++ /dev/null @@ -1,2 +0,0 @@ -easyconfigs: - - Valgrind-3.23.0-gompi-2023b.eb From e3682e61f136215486016389b73dfd0570621c07 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Mon, 30 Sep 2024 14:53:59 +0200 Subject: [PATCH 136/152] add comments to CI --- .github/workflows/test-software.eessi.io.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 29af8f57f1..6f592cf4c4 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -61,8 +61,10 @@ jobs: for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do if [ ${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}} = "x86_64/amd/zen4" ]; then if grep -q 2022b <<<"${easystack_file}"; then + # skip the check of installed software on zen4 for foss/2022b builds continue elif grep -q CUDA <<<"${easystack_file}"; then + # skip the check of install CUDA software in the CPU path for zen4 continue fi fi From 7b103b30bedb94097c09f4b5174445849df63419 Mon Sep 17 00:00:00 2001 From: Richard Top Date: Tue, 1 Oct 2024 12:01:00 +0000 Subject: [PATCH 137/152] {2023.06}[foss/2022b] data-lang --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index ca35e95bd4..016ddc6c0b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -41,3 +41,6 @@ easyconfigs: - libaio-0.3.113-GCCcore-12.2.0.eb - Z3-4.12.2-GCCcore-12.2.0.eb - tbb-2021.10.0-GCCcore-12.2.0.eb + - dask-2023.7.1-foss-2022b.eb + - netcdf4-python-1.6.3-foss-2022b.eb + - Ruby-3.2.2-GCCcore-12.2.0.eb From 114a8eb850cc40da18070f0f7ab990dd54634ec6 Mon Sep 17 00:00:00 2001 From: parosen Date: Tue, 1 Oct 2024 14:17:47 +0200 Subject: [PATCH 138/152] {2023.06}[foss/2023a] ROOT 6.30.06 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 27c18a487e..6823ea5cca 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -22,3 +22,4 @@ easyconfigs: from-pr: 20784 - Valgrind-3.21.0-gompi-2023a.eb - OrthoFinder-2.5.5-foss-2023a.eb + - ROOT-6.30.06-foss-2023a.eb From 1c82faa752e2144a01525c122926768a98f622b5 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 1 Oct 2024 14:46:33 +0200 Subject: [PATCH 139/152] Install ROOT with EasyBuild 4.9.4 --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml diff --git a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml new file mode 100644 index 0000000000..992e4c301c --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -0,0 +1,2 @@ +easyconfigs: + - ROOT-6.30.06-foss-2023a.eb From 2b2e57d092230f755c635cf9abe031b01bb968dc Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 1 Oct 2024 14:46:58 +0200 Subject: [PATCH 140/152] Update eessi-2023.06-eb-4.9.1-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 6823ea5cca..27c18a487e 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -22,4 +22,3 @@ easyconfigs: from-pr: 20784 - Valgrind-3.21.0-gompi-2023a.eb - OrthoFinder-2.5.5-foss-2023a.eb - - ROOT-6.30.06-foss-2023a.eb From 0080c9c9b971532482aaba5cb7cc37dad34a9d49 Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Tue, 1 Oct 2024 20:36:33 +0200 Subject: [PATCH 141/152] add gmsh & co easyconfig at bottom of easystack file --- .../2023.06/eessi-2023.06-eb-4.9.3-2023a.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml index 9e5582485c..0c863f0025 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.3-2023a.yml @@ -1,8 +1,8 @@ easyconfigs: - - gmsh-4.12.2-foss-2023a.eb - - basemap-1.3.9-foss-2023a.eb - - geopandas-0.14.2-foss-2023a.eb - ccache-4.9-GCCcore-12.3.0.eb - GDB-13.2-GCCcore-12.3.0.eb - tmux-3.3a-GCCcore-12.3.0.eb - - Vim-9.1.0004-GCCcore-12.3.0.eb \ No newline at end of file + - Vim-9.1.0004-GCCcore-12.3.0.eb + - gmsh-4.12.2-foss-2023a.eb + - basemap-1.3.9-foss-2023a.eb + - geopandas-0.14.2-foss-2023a.eb \ No newline at end of file From cde0cc8db464e69baefb9280c3e187b254e5974e Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 2 Oct 2024 10:50:52 +0200 Subject: [PATCH 142/152] More safely add to the toolchainopts --- eb_hooks.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 3754564c5c..79bdeeee0d 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -190,7 +190,9 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix): ): cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') if cpu_target == CPU_TARGET_NEOVERSE_V1: - if 'toolchainopts' not in ec: + # Make sure the toolchainopts key exists, and the value is a dict, + # before we add the option to disable vectorization + if 'toolchainopts' not in ec or ec['toolchainopts'] is None: ec['toolchainopts'] = {} ec['toolchainopts']['vectorize'] = False print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts']) @@ -308,7 +310,9 @@ def parse_hook_freeimage_aarch64(ec, *args, **kwargs): """ if ec.name == 'FreeImage' and ec.version in ('3.18.0',): if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': - if 'toolchainopts' not in ec: + # Make sure the toolchainopts key exists, and the value is a dict, + # before we add the option to enable PIC and disable PNG_ARM_NEON_OPT + if 'toolchainopts' not in ec or ec['toolchainopts'] is None: ec['toolchainopts'] = {} ec['toolchainopts']['pic'] = True ec['toolchainopts']['extra_cflags'] = '-DPNG_ARM_NEON_OPT=0' From c9613e79ef0585443a9e36a9773683f2904ca1ef Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 2 Oct 2024 13:22:05 +0200 Subject: [PATCH 143/152] only copy module_files.list.txt if it exists in create_tarball.sh script --- create_tarball.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/create_tarball.sh b/create_tarball.sh index 9c212681a5..01f498e1ac 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -89,13 +89,17 @@ for subdir in ${cpu_arch_subdir} ${cpu_arch_subdir}/accel/${accel_subdir}; do done # add a bit debug output -echo "wrote file list to ${files_list}" -[ -r ${files_list} ] && cat ${files_list} -echo "wrote module file list to ${module_files_list}" -[ -r ${module_files_list} ] && cat ${module_files_list} +if [ -r ${files_list} ]; then + echo "wrote file list to ${files_list}" + cat ${files_list} +fi +if [ -r ${module_files_list} ]; then + echo "wrote module file list to ${module_files_list}" + cat ${module_files_list} -# Copy the module files list to current workindg dir for later use in the test step -cp ${module_files_list} ${current_workdir}/module_files.list.txt + # Copy the module files list to current workindg dir for later use in the test step + cp ${module_files_list} ${current_workdir}/module_files.list.txt +fi topdir=${cvmfs_repo}/versions/ From a636f63df20a9367c9e4eb772856052697cdcf3a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 2 Oct 2024 13:46:23 +0200 Subject: [PATCH 144/152] For PRs that only deploy scripts, such as our second attempt at https://github.com/EESSI/software-layer/pull/736, the test step should be skipped. --- bot/check-test.sh | 14 ++++++++++++++ test_suite.sh | 4 ++++ 2 files changed, 18 insertions(+) diff --git a/bot/check-test.sh b/bot/check-test.sh index 3b16e5c415..9ff383a13b 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -23,6 +23,16 @@ else [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" fi +# First, account for the scenario where we skipped the ReFrame test suite +SKIPPED=-1 +if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then + GP_skipped='Skipping EESSI test site run' + grep_reframe_skipped=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_skipped}") + [[ $? -eq 0 ]] && SKIPPED=1 || SKIPPED=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_skipped}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_skipped}" +fi # ReFrame prints e.g. #[----------] start processing checks @@ -90,6 +100,10 @@ elif [[ ${SUCCESS} -eq 1 ]]; then summary=":grin: SUCCESS" reason="" status="SUCCESS" +elif [[ ${SKIPPED} -eq 1]]; then + summary="SKIPPED" + reason="The EESSI test suite was skipped. If this PR only deploys scripts, that's expected behavior." + status="SUCCESS" # Should come before general errors: if FAILED==1, it indicates the test suite ran # otherwise the pattern wouldn't have been there elif [[ ${FAILED} -eq 1 ]]; then diff --git a/test_suite.sh b/test_suite.sh index e7151e00e7..ad756f9c76 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -198,6 +198,10 @@ fi # Get the subset of test names based on the test mapping and tags (e.g. CI, 1_node) module_list="module_files.list.txt" mapping_config="tests/eessi_test_mapping/software_to_tests.yml" +if [[ ! -f "$module_list" || ! -f "$mapping_config" ]]; then + fatal_error "No new module files were found. Skipping EESSI test site run." +fi + # Run with --debug for easier debugging in case there are issues: python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}" --debug REFRAME_NAME_ARGS=$(python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}") From 30dd71727a50b8dffd14de57f47cf396c634da1a Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 2 Oct 2024 14:00:33 +0200 Subject: [PATCH 145/152] Only run default tests if the mapping file isn't found --- bot/check-test.sh | 15 ------- test_suite.sh | 19 +++++---- .../map_software_to_test.py | 39 +++++++++++-------- 3 files changed, 34 insertions(+), 39 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index 9ff383a13b..ed81198885 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -23,17 +23,6 @@ else [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" fi -# First, account for the scenario where we skipped the ReFrame test suite -SKIPPED=-1 -if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then - GP_skipped='Skipping EESSI test site run' - grep_reframe_skipped=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_skipped}") - [[ $? -eq 0 ]] && SKIPPED=1 || SKIPPED=0 - # have to be careful to not add searched for pattern into slurm out file - [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_skipped}"'" - [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_skipped}" -fi - # ReFrame prints e.g. #[----------] start processing checks #[ RUN ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default @@ -100,10 +89,6 @@ elif [[ ${SUCCESS} -eq 1 ]]; then summary=":grin: SUCCESS" reason="" status="SUCCESS" -elif [[ ${SKIPPED} -eq 1]]; then - summary="SKIPPED" - reason="The EESSI test suite was skipped. If this PR only deploys scripts, that's expected behavior." - status="SUCCESS" # Should come before general errors: if FAILED==1, it indicates the test suite ran # otherwise the pattern wouldn't have been there elif [[ ${FAILED} -eq 1 ]]; then diff --git a/test_suite.sh b/test_suite.sh index ad756f9c76..1f0b91c477 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -198,14 +198,19 @@ fi # Get the subset of test names based on the test mapping and tags (e.g. CI, 1_node) module_list="module_files.list.txt" mapping_config="tests/eessi_test_mapping/software_to_tests.yml" -if [[ ! -f "$module_list" || ! -f "$mapping_config" ]]; then - fatal_error "No new module files were found. Skipping EESSI test site run." +if [[ ! -f "$module_list" ]]; then + echo_green "File ${module_list} not found, so only running the default set of tests from ${mapping_config}" + # Run with --debug for easier debugging in case there are issues: + python3 tests/eessi_test_mapping/map_software_to_test.py --mapping-file "${mapping_config}" --debug --defaults-only + REFRAME_NAME_ARGS=$(python3 tests/eessi_test_mapping/map_software_to_test.py --mapping-file "${mapping_config}" --defaults-only) + test_selection_exit_code=$? +else + # Run with --debug for easier debugging in case there are issues: + python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}" --debug + REFRAME_NAME_ARGS=$(python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}") + test_selection_exit_code=$? fi - -# Run with --debug for easier debugging in case there are issues: -python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}" --debug -REFRAME_NAME_ARGS=$(python3 tests/eessi_test_mapping/map_software_to_test.py --module-list "${module_list}" --mapping-file "${mapping_config}") -test_selection_exit_code=$? +# Check exit status if [[ ${test_selection_exit_code} -eq 0 ]]; then echo_green "Succesfully extracted names of tests to run: ${REFRAME_NAME_ARGS}" else diff --git a/tests/eessi_test_mapping/map_software_to_test.py b/tests/eessi_test_mapping/map_software_to_test.py index 24cf246ef1..a0da6258c8 100644 --- a/tests/eessi_test_mapping/map_software_to_test.py +++ b/tests/eessi_test_mapping/map_software_to_test.py @@ -33,29 +33,32 @@ def get_tests_for_software(software_name, mappings): return [] -def main(yaml_file, module_file, debug): +def main(yaml_file, module_file, debug, defaults_only): """Main function to process software names and their tests.""" mappings = load_mappings(yaml_file) if debug: print(f"Loaded mappings from '{yaml_file}'") - software_names = read_software_names(module_file) - if debug: - print(f"Read software names from '{module_file}'") + if not defaults_only: + software_names = read_software_names(module_file) + if debug: + print(f"Read software names from '{module_file}'") tests_to_run = [] arg_string = "" - # For each module name, get the relevant set of tests - for software_name in software_names: - additional_tests = get_tests_for_software(software_name, mappings) - for test in additional_tests: - if test not in tests_to_run: - tests_to_run.append(test) - - if additional_tests and debug: - print(f"Software: {software_name} -> Tests: {additional_tests}") - elif debug: - print(f"Software: {software_name} -> No tests found") + + if not defaults_only: + # For each module name, get the relevant set of tests + for software_name in software_names: + additional_tests = get_tests_for_software(software_name, mappings) + for test in additional_tests: + if test not in tests_to_run: + tests_to_run.append(test) + + if additional_tests and debug: + print(f"Software: {software_name} -> Tests: {additional_tests}") + elif debug: + print(f"Software: {software_name} -> No tests found") # Always add the default set of tests, if default_tests is specified if 'default_tests' in mappings: @@ -83,8 +86,10 @@ def main(yaml_file, module_file, debug): parser = argparse.ArgumentParser(description="Map software names to their tests based on a YAML configuration.") parser.add_argument('--mapping-file', type=str, help='Path to the YAML file containing the test mappings.') parser.add_argument('--module-list', type=str, help='Path to the file containing the list of software names.') - parser.add_argument('--debug', action='store_true', help='Enable debug output.') + defaults_help = "Don't consider the module-list file, only return the default tests from the mapping file" + parser.add_argument('--defaults-only', action='store_true', default=False, help=defaults_help) + parser.add_argument('--debug', action='store_true', default=False, help='Enable debug output.') args = parser.parse_args() - main(args.mapping_file, args.module_list, args.debug) + main(args.mapping_file, args.module_list, args.debug, args.defaults_only) From fba72eb561d0e404bf8cf321fc7eb5be12c93c15 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 2 Oct 2024 14:55:07 +0200 Subject: [PATCH 146/152] More extensive reporting of the test results --- bot/check-test.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index ed81198885..aae1948d33 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -65,7 +65,7 @@ SUCCESS=-1 # Grep for the success pattern, so we can report the amount of tests run if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_success='\[\s*PASSED\s*\].*Ran .* test case' - grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}") + grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success}") [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0 # have to be careful to not add searched for pattern into slurm out file [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'" @@ -75,7 +75,10 @@ fi if [[ ! -z ${grep_reframe_failed} ]]; then grep_reframe_result=${grep_reframe_failed} else - grep_reframe_result=${grep_reframe_success} + # Grep the entire output of ReFrame, so that we can report it in the foldable section of the test report + GP_success_full='(?s)\[----------\] start processing checks.*?\[==========\] Finished on [a-zA-Z0-9 ]*' + grep_reframe_success_full=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success}") + grep_reframe_result=${grep_reframe_success_full} fi echo "[TEST]" > ${job_test_result_file} From e06e303c3a0840d1ef07f136d56c691e4b9c5cf5 Mon Sep 17 00:00:00 2001 From: Caspar van Leeuwen Date: Wed, 2 Oct 2024 14:56:27 +0200 Subject: [PATCH 147/152] This grep can be a simple one --- bot/check-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/check-test.sh b/bot/check-test.sh index aae1948d33..8ea6eddcb0 100755 --- a/bot/check-test.sh +++ b/bot/check-test.sh @@ -65,7 +65,7 @@ SUCCESS=-1 # Grep for the success pattern, so we can report the amount of tests run if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then GP_success='\[\s*PASSED\s*\].*Ran .* test case' - grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep -Pzo "${GP_success}") + grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}") [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0 # have to be careful to not add searched for pattern into slurm out file [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'" From 139f422fda241768ec9e033fa1cdc032ba7c328d Mon Sep 17 00:00:00 2001 From: ocaisa Date: Wed, 2 Oct 2024 16:56:45 +0200 Subject: [PATCH 148/152] Update eessi-2023.06-eb-4.9.4-2023a.yml --- .../2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 992e4c301c..3ca4d5f4c8 100644 --- a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -1,2 +1,5 @@ easyconfigs: - - ROOT-6.30.06-foss-2023a.eb + - ROOT-6.30.06-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 + include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 From 148e8efae37d2ad1e51756cd64574661fccb9ca5 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 3 Oct 2024 14:17:36 +0200 Subject: [PATCH 149/152] Also use patched easyconfig --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 3ca4d5f4c8..2a8cb36994 100644 --- a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -1,5 +1,7 @@ easyconfigs: - ROOT-6.30.06-foss-2023a.eb: options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 + from-commit: 0707601ccd7dd497a419ed39b1b5527ddd0450f4 # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 From 719b5628ea93bdd321c96efc17e16d7c8d66f3ac Mon Sep 17 00:00:00 2001 From: ocaisa Date: Thu, 3 Oct 2024 15:29:10 +0200 Subject: [PATCH 150/152] Update eessi-2023.06-eb-4.9.4-2023a.yml --- .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml index 2a8cb36994..19d9aedfae 100644 --- a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml @@ -2,6 +2,6 @@ easyconfigs: - ROOT-6.30.06-foss-2023a.eb: options: # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 - from-commit: 0707601ccd7dd497a419ed39b1b5527ddd0450f4 + from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601 From bcaeec7737d6eb8679483f014c2b01c5a203c3c0 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 4 Oct 2024 07:38:05 +0200 Subject: [PATCH 151/152] Move file to correct location --- .../2023.06 => }/eessi-2023.06-eb-4.9.4-2023a.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename easystacks/software.eessi.io/2023.06/{easystacks/software.eessi.io/2023.06 => }/eessi-2023.06-eb-4.9.4-2023a.yml (100%) diff --git a/easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml similarity index 100% rename from easystacks/software.eessi.io/2023.06/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml rename to easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.4-2023a.yml From a34ecc5bbec7845d5acfef9bfd2eb60e8360599f Mon Sep 17 00:00:00 2001 From: Richard Top Date: Fri, 4 Oct 2024 06:34:47 +0000 Subject: [PATCH 152/152] {2023.06}[foss/2022b] ROOT v6.26.10 --- .../2023.06/eessi-2023.06-eb-4.9.2-2022b.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml index 016ddc6c0b..969b0d469b 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -44,3 +44,9 @@ easyconfigs: - dask-2023.7.1-foss-2022b.eb - netcdf4-python-1.6.3-foss-2022b.eb - Ruby-3.2.2-GCCcore-12.2.0.eb + - ROOT-6.26.10-foss-2022b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21526 + from-commit: 6cbfbd7d7a55dc7243f46d0beea510278f4718df + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3467 + include-easyblocks-from-commit: c3aebe1f133d064a228c5d6c282e898b83d74601