diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index dc18fd584a..7eff557094 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -40,7 +40,7 @@ jobs: persist-credentials: false - name: "Run analysis" - uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6 + uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1 with: results_file: results.sarif results_format: sarif diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 4944d9beaa..76d19d29fe 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -3,9 +3,9 @@ name: Tests for scripts on: push: paths: - - build_container.sh - create_directory_tarballs.sh - create_lmodsitepackage.py + - eessi_container.sh - EESSI-install-software.sh - install_software_layer.sh - load_easybuild_module.sh @@ -15,9 +15,9 @@ on: pull_request: paths: - - build_container.sh - create_directory_tarballs.sh - create_lmodsitepackage.py + - eessi_container.sh - EESSI-install-software.sh - install_software_layer.sh - load_easybuild_module.sh diff --git a/build_container.sh b/build_container.sh deleted file mode 100755 index 23a9e665c9..0000000000 --- a/build_container.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -base_dir=$(dirname $(realpath $0)) - -BUILD_CONTAINER="docker://ghcr.io/eessi/build-node:debian11" - -if [ $# -lt 2 ]; then - echo "Usage: $0 " >&2 - exit 1 -fi -SHELL_OR_RUN=$1 -EESSI_TMPDIR=$2 -shift 2 - -if [ "$SHELL_OR_RUN" == "run" ] && [ $# -eq 0 ]; then - echo "ERROR: No command specified to run?!" >&2 - exit 1 -fi - -# make sure specified temporary directory exists -mkdir -p $EESSI_TMPDIR - -echo "Using $EESSI_TMPDIR as parent for temporary directories..." - -# create temporary directories -mkdir -p $EESSI_TMPDIR/{home,overlay-upper,overlay-work} -mkdir -p $EESSI_TMPDIR/{var-lib-cvmfs,var-run-cvmfs} -# configure Singularity -export SINGULARITY_CACHEDIR=$EESSI_TMPDIR/singularity_cache - -# take into account that $SINGULARITY_BIND may be defined already, to bind additional paths into the build container -BIND_PATHS="$EESSI_TMPDIR/var-run-cvmfs:/var/run/cvmfs,$EESSI_TMPDIR/var-lib-cvmfs:/var/lib/cvmfs,$EESSI_TMPDIR" -if [ -z $SINGULARITY_BIND ]; then - export SINGULARITY_BIND="$BIND_PATHS" -else - export SINGULARITY_BIND="$SINGULARITY_BIND,$BIND_PATHS" -fi - -# allow that SINGULARITY_HOME is defined before script is run -if [ -z $SINGULARITY_HOME ]; then - export SINGULARITY_HOME="$EESSI_TMPDIR/home:/home/$USER" -fi - -source ${base_dir}/init/eessi_defaults -# strip "/cvmfs/" from default setting -repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} - -# set environment variables for fuse mounts in Singularity container -export EESSI_PILOT_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" -export EESSI_PILOT_WRITABLE_OVERLAY="container:fuse-overlayfs -o lowerdir=/cvmfs_ro/${repo_name} -o upperdir=$EESSI_TMPDIR/overlay-upper -o workdir=$EESSI_TMPDIR/overlay-work ${EESSI_CVMFS_REPO}" - -# pass $EESSI_SOFTWARE_SUBDIR_OVERRIDE into build container (if set) -if [ ! -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then - export SINGULARITYENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} - # also specify via $APPTAINERENV_* (future proof, cfr. https://apptainer.org/docs/user/latest/singularity_compatibility.html#singularity-environment-variable-compatibility) - export APPTAINERENV_EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE} -fi - -if [ "$SHELL_OR_RUN" == "shell" ]; then - # start shell in Singularity container, with EESSI repository mounted with writable overlay - echo "Starting Singularity build container..." - singularity shell --fusemount "$EESSI_PILOT_READONLY" --fusemount "$EESSI_PILOT_WRITABLE_OVERLAY" $BUILD_CONTAINER -elif [ "$SHELL_OR_RUN" == "run" ]; then - echo "Running '$@' in Singularity build container..." - singularity exec --fusemount "$EESSI_PILOT_READONLY" --fusemount "$EESSI_PILOT_WRITABLE_OVERLAY" $BUILD_CONTAINER "$@" -else - echo "ERROR: Unknown action specified: $SHELL_OR_RUN (should be either 'shell' or 'run')" >&2 - exit 1 -fi diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py index 0761f2cdf1..47aa20e51e 100755 --- a/create_lmodsitepackage.py +++ b/create_lmodsitepackage.py @@ -20,6 +20,28 @@ return content end +local function from_eessi_prefix(t) + -- eessi_prefix is the prefix with official EESSI modules + -- e.g. /cvmfs/software.eessi.io/versions/2023.06 + local eessi_prefix = os.getenv("EESSI_PREFIX") + + -- If EESSI_PREFIX wasn't defined, we cannot check if this module was from the EESSI environment + -- In that case, we assume it isn't, otherwise EESSI_PREFIX would (probably) have been set + if eessi_prefix == nil then + return False + else + -- NOTE: exact paths for site so may need to be updated later. + -- See https://github.com/EESSI/software-layer/pull/371 + + -- eessi_prefix_host_injections is the prefix with site-extensions (i.e. additional modules) + -- to the official EESSI modules, e.g. /cvmfs/software.eessi.io/host_injections/2023.06 + local eessi_prefix_host_injections = string.gsub(eessi_prefix, 'versions', 'host_injections') + + -- Check if the full modulepath starts with the eessi_prefix_* + return string.find(t.fn, "^" .. eessi_prefix) ~= nil or string.find(t.fn, "^" .. eessi_prefix_host_injections) ~= nil + end +end + local function load_site_specific_hooks() -- This function will be run after the EESSI hooks are registered -- It will load a local SitePackage.lua that is architecture independent (if it exists) from e.g. @@ -153,10 +175,13 @@ -- Combine both functions into a single one, as we can only register one function as load hook in lmod -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed function eessi_load_hook(t) - eessi_cuda_enabled_load_hook(t) + -- Only apply CUDA hooks if the loaded module is in the EESSI prefix + -- This avoids getting an Lmod Error when trying to load a CUDA module from a local software stack + if from_eessi_prefix(t) then + eessi_cuda_enabled_load_hook(t) + end end - hook.register("load", eessi_load_hook) -- Note that this needs to happen at the end, so that any EESSI specific hooks can be overwritten by the site diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml index 44952e2765..8f5608d881 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml @@ -35,7 +35,6 @@ easyconfigs: from-pr: 19996 - dask-2023.9.2-foss-2023a.eb - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb - - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb: options: diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml index 86c5106c85..15c02951d7 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml @@ -57,3 +57,4 @@ easyconfigs: - Qt5-5.15.13-GCCcore-13.2.0.eb: options: from-pr: 20201 + - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml new file mode 100644 index 0000000000..e547e45f2b --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml @@ -0,0 +1,4 @@ +easyconfigs: + - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb: + options: + from-pr: 20379 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 3b51db3b11..e65747e4a5 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -1,2 +1,6 @@ easyconfigs: - ncdu-1.18-GCC-12.3.0.eb + - SAMtools-1.18-GCC-12.3.0.eb + - R-bundle-Bioconductor-3.18-foss-2023a-R-4.3.2.eb: + options: + from-pr: 20379 diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml new file mode 100644 index 0000000000..1392fa2f5a --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml @@ -0,0 +1,4 @@ +easyconfigs: + - GROMACS-2024.1-foss-2023b.eb: + options: + from-pr: 20439 diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml b/easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml new file mode 100644 index 0000000000..058ab75e80 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/2024.05.06-eb-4.9.1-CUDA-12.1.1-ship-full-runtime.yml @@ -0,0 +1,9 @@ +# 2024.05.06 +# Original matching of files we could ship was not done correctly. We were +# matching the basename for files (e.g., libcudart.so from libcudart.so.12) +# rather than the name stub (libcudart) +# See https://github.com/EESSI/software-layer/pull/559 +easyconfigs: + - CUDA-12.1.1.eb: + options: + accept-eula-for: CUDA diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml new file mode 100644 index 0000000000..56ea7a46a1 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240419-eb-4.9.1-move-setuptools_scm-from-hatchling-to-Python.yml @@ -0,0 +1,13 @@ +# 2024-04-19 +# Move setuptools_scm extension from hatchling to Python by rebuilding +# all affected modules with EasyBuild 4.9.1. +# This solves an issue with pyarrow, which is part of the Arrow installation. +# https://github.com/easybuilders/easybuild-easyconfigs/pull/19777 +# https://github.com/easybuilders/easybuild-easyconfigs/issues/19849 +easyconfigs: + - hatchling-1.18.0-GCCcore-12.3.0.eb + - hatchling-1.18.0-GCCcore-13.2.0.eb + - Python-bundle-PyPI-2023.06-GCCcore-12.3.0.eb + - Python-bundle-PyPI-2023.10-GCCcore-13.2.0.eb + - Python-3.11.3-GCCcore-12.3.0.eb + - Python-3.11.5-GCCcore-13.2.0.eb diff --git a/eb_hooks.py b/eb_hooks.py index 20b5f76cfc..8b0a11b0ed 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -335,6 +335,21 @@ def pre_configure_hook(self, *args, **kwargs): PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) +def pre_configure_hook_gromacs(self, *args, **kwargs): + """ + Pre-configure hook for GROMACS: + - avoid building with SVE instructions on Neoverse V1 as workaround for failing tests, + see https://gitlab.com/gromacs/gromacs/-/issues/5057 + https://gitlab.com/eessi/support/-/issues/47 + """ + if self.name == 'GROMACS': + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + if LooseVersion(self.version) <= LooseVersion('2024.1') and cpu_target == CPU_TARGET_NEOVERSE_V1: + self.cfg.update('configopts', '-DGMX_SIMD=ARM_NEON_ASIMD') + print_msg("Avoiding use of SVE instructions for GROMACS %s by using ARM_NEON_ASIMD as GMX_SIMD value", self.version) + else: + raise EasyBuildError("GROMACS-specific hook triggered for non-GROMACS easyconfig?!") + + def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs): """ Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC @@ -597,8 +612,8 @@ def post_sanitycheck_cuda(self, *args, **kwargs): full_path = os.path.join(dir_path, filename) # we only really care about real files, i.e. not symlinks if not os.path.islink(full_path): - # check if the current file is part of the allowlist - basename = os.path.splitext(filename)[0] + # check if the current file name stub is part of the allowlist + basename = filename.split('.')[0] if basename in allowlist: self.log.debug("%s is found in allowlist, so keeping it: %s", basename, full_path) else: @@ -665,6 +680,7 @@ def inject_gpu_property(ec): } PRE_CONFIGURE_HOOKS = { + 'GROMACS': pre_configure_hook_gromacs, 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, diff --git a/eessi-2023.06-known-issues.yml b/eessi-2023.06-known-issues.yml index 011cb2dc08..2d1256354f 100644 --- a/eessi-2023.06-known-issues.yml +++ b/eessi-2023.06-known-issues.yml @@ -19,6 +19,9 @@ - FFTW.MPI-3.3.10-gompi-2023b: - issue: https://github.com/EESSI/software-layer/issues/325 - info: "Flaky FFTW tests, random failures" + - GROMACS-2024.1-foss-2023b: + - issue: https://github.com/EESSI/software-layer/issues/557 + - info: "SVE disabled due to known bug which causes test failures" - Highway-1.0.3-GCCcore-12.2.0.eb: - issue: https://github.com/EESSI/software-layer/issues/469 - info: "failing SVE test due to wrong expected value" diff --git a/init/arch_specs/eessi_arch_x86.spec b/init/arch_specs/eessi_arch_x86.spec index 81a7429a8d..bfbc5b4be1 100755 --- a/init/arch_specs/eessi_arch_x86.spec +++ b/init/arch_specs/eessi_arch_x86.spec @@ -1,6 +1,6 @@ # x86_64 CPU architecture specifications # Software path in EESSI | Vendor ID | List of defining CPU features -"x86_64/intel/haswell" "GenuineIntel" "avx2 fma" # Intel Haswell, Broadwell +"x86_64/intel/haswell" "GenuineIntel" "avx2 fma" # Intel Haswell, Broadwell "x86_64/intel/skylake_avx512" "GenuineIntel" "avx2 fma avx512f avx512bw avx512cd avx512dq avx512vl" # Intel Skylake, Cascade Lake "x86_64/amd/zen2" "AuthenticAMD" "avx2 fma" # AMD Rome "x86_64/amd/zen3" "AuthenticAMD" "avx2 fma vaes" # AMD Milan, Milan-X