Merge branch '2023.06-software.eessi.io' of github-trz:EESSI/software…

…-layer into build_highway_zen4

.github/workflows/scorecards.yml

@@ -10,7 +10,7 @@ on:
     - cron: '25 15 * * 3'
   push:
     branches:
-      - '2023.06-software.eessi.io'
+      - '*-software.eessi.io'
 
 # Declare default permissions as read only.
 permissions: read-all

.github/workflows/test-software.eessi.io.yml

@@ -1,6 +1,10 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Check for missing software installations in software.eessi.io
-on: [push, pull_request, workflow_dispatch]
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
+  workflow_dispatch:
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:

.github/workflows/test_eessi_container_script.yml

@@ -1,6 +1,10 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests for eessi_container.sh script
-on: [push, pull_request, workflow_dispatch]
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
+  workflow_dispatch:
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
@@ -45,7 +49,8 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
                 outfile=out_listrepos.txt
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
-                grep "EESSI" ${outfile}
+                # make sure that the default EESSI software repository is available
+                grep "software.eessi.io" ${outfile}
 
               # test use of --list-repos with custom repos.cfg
               elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
@@ -57,11 +62,12 @@ jobs:
                 echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
                 echo "repo_version = 20HT.TP" >> cfg/repos.cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
-                grep "EESSI" ${outfile}
+                # make sure that the default EESSI software repository is available
+                grep "software.eessi.io" ${outfile}
 
                 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile2}
-                grep "[EESSI/2023.02]" ${outfile2}
+                grep "EESSI/20AB.CD" ${outfile2}
 
               # test use of --mode run
               elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then

.github/workflows/test_licenses.yml

@@ -1,6 +1,9 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Test software licenses
-on: [push, pull_request]
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:

.github/workflows/tests.yml

@@ -1,6 +1,9 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests
-on: [push, pull_request]
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:

.github/workflows/tests_archdetect.yml

@@ -1,6 +1,9 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests for eessi_archdetect.sh
-on: [push, pull_request]
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:

.github/workflows/tests_init.yml

@@ -1,6 +1,9 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests for init scripts
-on: [push, pull_request]
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:

bot/build.sh

@@ -138,7 +138,7 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'"
 # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
 # "source init/eessi_defaults" via sourcing init/minimal_eessi_env
-export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name")
+export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name")
 echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'"
 
 # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE}

bot/test.sh

@@ -160,7 +160,7 @@ echo "bot/test.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}
 # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
 # "source init/eessi_defaults" via sourcing init/minimal_eessi_env
-export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name")
+export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name")
 echo "bot/test.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'"
 
 # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE}

create_tarball.sh

@@ -26,7 +26,8 @@ if [ ! -d ${software_dir} ]; then
     exit 2
 fi
 
-overlay_upper_dir="${eessi_tmpdir}/overlay-upper"
+cvmfs_repo_name=${cvmfs_repo#/cvmfs/}
+overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper"
 
 software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}"
 if [ ! -d ${software_dir_overlay} ]; then

easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml

@@ -3,3 +3,4 @@ easyconfigs:
   - foss-2023a.eb
   - SciPy-bundle-2023.07-gfbf-2023a.eb
   - ESPResSo-4.2.2-foss-2023a.eb
+  - ParaView-5.11.2-foss-2023a.eb

easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023b.yml

@@ -0,0 +1,6 @@
+easyconfigs:
+  - OpenBLAS-0.3.24-GCC-13.2.0.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20820
+        from-commit: 1fc885b35dacdeb2feef4af207a2daa2502bae08
+  - foss-2023b.eb

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml

@@ -0,0 +1,2 @@
+easyconfigs:
+  - BLAST+-2.14.0-gompi-2022b.eb

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml

@@ -0,0 +1,29 @@
+easyconfigs:
+  - BCFtools-1.18-GCC-12.3.0.eb
+  - BWA-0.7.18-GCCcore-12.3.0.eb
+  - CapnProto-1.0.1-GCCcore-12.3.0.eb
+  - DendroPy-4.6.1-GCCcore-12.3.0.eb
+  - DIAMOND-2.1.8-GCC-12.3.0.eb
+  - FastME-2.1.6.3-GCC-12.3.0.eb
+  - fastp-0.23.4-GCC-12.3.0.eb
+  - HMMER-3.4-gompi-2023a.eb
+  - IQ-TREE-2.3.5-gompi-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20955
+        from-commit: 185f88b9a03d65a7fb74edc7acb4221e87e90784      
+  - KronaTools-2.8.1-GCCcore-12.3.0.eb
+  - LSD2-2.4.1-GCCcore-12.3.0.eb
+  - MAFFT-7.520-GCC-12.3.0-with-extensions.eb
+  - ncbi-vdb-3.0.10-gompi-2023a.eb
+  - MetalWalls-21.06.1-foss-2023a.eb
+  - QuantumESPRESSO-7.3.1-foss-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20138
+        from-commit: dbdaacc0739fdee91baa9123864ea4428cf21273
+        # see https://github.com/easybuilders/easybuild-easyblocks/pull/3338
+        include-easyblocks-from-commit: 32e45bd1f2d916732ca5852d55d17fa4d99e388b
+  - CP2K-2023.1-foss-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20951
+        from-commit: a92667fe32396bbd4106243658625f7ff2adcd68
+  - amdahl-0.3.1-gompi-2023a.eb

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml

@@ -1,3 +1,17 @@
 easyconfigs:
   - IPython-8.17.2-GCCcore-13.2.0.eb
   - dlb-3.4-gompi-2023b.eb
+  - pystencils-1.3.4-gfbf-2023b.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20889
+        from-commit: c66c4788a17f7e4f55aa23f9fdb782aad97c9ce7
+  - Extrae-4.2.0-gompi-2023b.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21017
+        from-commit: 120f4d56efebd2bc61382db4c84a664a339c66cf
+        # see https://github.com/easybuilders/easybuild-easyblocks/pull/3393
+        include-easyblocks-from-commit: c4951c78d62fa5cf8e9f6fe0ead212d2a4d7cb9c
+  - pyMBE-0.8.0-foss-2023b.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034
+        from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2

easystacks/software.eessi.io/2023.06/rebuilds/20240706-eb-4.9.2-OpenFOAM-no-ftree-vectorize.yml

@@ -0,0 +1,12 @@
+# 2024.07.06
+# OpenFOAM 10 and 11 built with GCC 11.3.0 or 12.3.0 and -ftree-vectorize yields incorrect results,
+# see https://github.com/easybuilders/easybuild-easyconfigs/issues/20927
+easyconfigs:
+  - OpenFOAM-10-foss-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958
+        from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f
+  - OpenFOAM-11-foss-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958
+        from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f

easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-zen4.yml

@@ -0,0 +1,11 @@
+# 2024.07.19
+# GObject-Introspection sets $LD_LIBRARY_PATH (to many different paths, including $EPREFIX/lib)
+# when calling gcc, and this causes a lot of issues for, especially, scripts using /bin/bash.
+#
+# This rebuild ensures (by using a new EasyBuild hook) that GObject-Introspection will not set
+# environment variables that are configured to be filtered by EasyBuild.
+# 
+# See https://github.com/EESSI/software-layer/issues/196
+easyconfigs:
+  - GObject-Introspection-1.76.1-GCCcore-12.3.0.eb
+  - GObject-Introspection-1.78.1-GCCcore-13.2.0.eb

easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml

@@ -0,0 +1,14 @@
+# 2024.07.19
+# GObject-Introspection sets $LD_LIBRARY_PATH (to many different paths, including $EPREFIX/lib)
+# when calling gcc, and this causes a lot of issues for, especially, scripts using /bin/bash.
+#
+# This rebuild ensures (by using a new EasyBuild hook) that GObject-Introspection will not set
+# environment variables that are configured to be filtered by EasyBuild.
+# 
+# See https://github.com/EESSI/software-layer/issues/196
+easyconfigs:
+  - GObject-Introspection-1.74.0-GCCcore-12.2.0.eb
+  - GObject-Introspection-1.76.1-GCCcore-12.3.0.eb
+  - GObject-Introspection-1.78.1-GCCcore-13.2.0.eb
+  - at-spi2-core-2.46.0-GCCcore-12.2.0.eb
+  - at-spi2-core-2.49.91-GCCcore-12.3.0.eb

easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml

@@ -2,8 +2,17 @@ easyconfigs:
   - ESPResSo-4.2.2-foss-2023a.eb
   - TensorFlow-2.13.0-foss-2023a.eb
   - R-4.3.2-gfbf-2023a.eb
-  - Highway-1.0.4-GCCcore-12.3.0.eb
-  #- Highway-1.0.4-GCCcore-12.3.0.eb:
-  #    options:
-  #      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20942
-  #      from-pr: 20942
+  - ParaView-5.11.2-foss-2023a.eb
+  - OpenFOAM-10-foss-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958
+        from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f
+  - OpenFOAM-11-foss-2023a.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958
+        from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f
+  #- Highway-1.0.4-GCCcore-12.3.0.eb
+  - Highway-1.0.4-GCCcore-12.3.0.eb:
+      options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20942
+        from-pr: 20942

eb_hooks.py

@@ -1,5 +1,6 @@
 # Hooks to customize how EasyBuild installs software in EESSI
 # see https://docs.easybuild.io/en/latest/Hooks.html
+import glob
 import os
 import re
 
@@ -81,8 +82,12 @@ def post_ready_hook(self, *args, **kwargs):
     """
     # 'parallel' easyconfig parameter is set via EasyBlock.set_parallel in ready step based on available cores.
     # here we reduce parallellism to only use half of that for selected software,
-    # to avoid failing builds/tests due to out-of-memory problems
-    if self.name in ['TensorFlow', 'libxc']:
+    # to avoid failing builds/tests due to out-of-memory problems;
+    memory_hungry_build = self.name in ['libxc', 'TensorFlow']
+    # on A64FX systems, (HBM) memory is typically scarce, so we need to use fewer cores for some builds
+    cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
+    memory_hungry_build_a64fx = cpu_target == CPU_TARGET_A64FX and self.name in ['Qt5']
+    if memory_hungry_build or memory_hungry_build_a64fx:
         parallel = self.cfg['parallel']
         if parallel > 1:
             self.cfg['parallel'] = parallel // 2
@@ -294,6 +299,22 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs):
         raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!")
 
 
+def parse_hook_CP2K_remove_deps_for_aarch64(ec, *args, **kwargs):
+    """
+    Remove x86_64 specific dependencies for the CI and missing installations to pass on aarch64
+    """
+    if ec.name == 'CP2K' and ec.version in ('2023.1',):
+        if os.getenv('EESSI_CPU_FAMILY') == 'aarch64':
+            # LIBXSMM is not supported on ARM with GCC 12.2.0 and 12.3.0
+            # See https://www.cp2k.org/dev:compiler_support
+            # See https://github.com/easybuilders/easybuild-easyconfigs/pull/20951
+            # we need this hook because we check for missing installations for all CPU targets
+            # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true)
+            ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('libxsmm',)]
+    else:
+        raise EasyBuildError("CP2K-specific hook triggered for non-CP2K easyconfig?!")
+
+
 def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs):
     """
     Solve issues with compiling or running the tests on both
@@ -353,6 +374,51 @@ def pre_configure_hook_BLIS_a64fx(self, *args, **kwargs):
     else:
         raise EasyBuildError("BLIS-specific hook triggered for non-BLIS easyconfig?!")
 
+def pre_configure_hook_extrae(self, *args, **kwargs):
+    """
+    Pre-configure hook for Extrae
+    - avoid use of 'which' in configure script
+    - specify correct path to binutils/zlib (in compat layer)
+    """
+    if self.name == 'Extrae':
+
+        # determine path to Prefix installation in compat layer via $EPREFIX
+        eprefix = get_eessi_envvar('EPREFIX')
+
+        binutils_lib_path_glob_pattern = os.path.join(eprefix, 'usr', 'lib*', 'binutils', '*-linux-gnu', '2.*')
+        binutils_lib_path = glob.glob(binutils_lib_path_glob_pattern)
+        if len(binutils_lib_path) == 1:
+            self.cfg.update('configopts', '--with-binutils=' + binutils_lib_path[0])
+        else:
+            raise EasyBuildError("Failed to isolate path for binutils libraries using %s, got %s",
+                                 binutils_lib_path_glob_pattern, binutils_lib_path)
+
+        # zlib is a filtered dependency, so we need to manually specify it's location to avoid the host version
+        self.cfg.update('configopts', '--with-libz=' + eprefix)
+
+        # replace use of 'which' with 'command -v', since 'which' is broken in EESSI build container;
+        # this must be done *after* running configure script, because initial configuration re-writes configure script,
+        # and problem due to use of which only pops up when running make ?!
+        self.cfg.update('prebuildopts', "cp config/mpi-macros.m4 config/mpi-macros.m4.orig && sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && ")
+    else:
+        raise EasyBuildError("Extrae-specific hook triggered for non-Extrae easyconfig?!")
+
+
+def pre_configure_hook_gobject_introspection(self, *args, **kwargs):
+    """
+    pre-configure hook for GObject-Introspection:
+    - prevent GObject-Introspection from setting $LD_LIBRARY_PATH if EasyBuild is configured to filter it, see:
+      https://github.com/EESSI/software-layer/issues/196
+    """
+    if self.name == 'GObject-Introspection':
+        # inject a line that removes all items from runtime_path_envvar that are in $EASYBUILD_FILTER_ENVVARS
+        sed_cmd = r'sed -i "s@\(^\s*runtime_path_envvar = \)\(.*\)@'
+        sed_cmd += r'\1\2\n\1 [x for x in runtime_path_envvar if not x in os.environ.get(\'EASYBUILD_FILTER_ENV_VARS\', \'\').split(\',\')]@g"'
+        sed_cmd += ' %(start_dir)s/giscanner/ccompiler.py && '
+        self.cfg.update('preconfigopts', sed_cmd)
+    else:
+        raise EasyBuildError("GObject-Introspection-specific hook triggered for non-GObject-Introspection easyconfig?!")
+
 
 def pre_configure_hook_gromacs(self, *args, **kwargs):
     """
@@ -435,21 +501,6 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs):
         raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!")
 
 
-def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs):
-    """
-    pre-configure hook for at-spi2-core:
-    - instruct GObject-Introspection's g-ir-scanner tool to not set $LD_LIBRARY_PATH
-      when EasyBuild is configured to filter it, see:
-      https://github.com/EESSI/software-layer/issues/196
-    """
-    if self.name == 'at-spi2-core':
-        if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'):
-            sed_cmd = 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n  \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && '
-            self.cfg.update('preconfigopts', sed_cmd)
-    else:
-        raise EasyBuildError("at-spi2-core-specific hook triggered for non-at-spi2-core easyconfig?!")
-
-
 def pre_test_hook(self,*args, **kwargs):
     """Main pre-test hook: trigger custom functions based on software name."""
     if self.name in PRE_TEST_HOOKS:
@@ -693,6 +744,7 @@ def inject_gpu_property(ec):
     'CGAL': parse_hook_cgal_toolchainopts_precise,
     'fontconfig': parse_hook_fontconfig_add_fonts,
     'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64,
+    'CP2K': parse_hook_CP2K_remove_deps_for_aarch64,
     'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors,
     'pybind11': parse_hook_pybind11_replace_catch2,
     'Qt5': parse_hook_qt5_check_qtwebengine_disable,
@@ -709,8 +761,9 @@ def inject_gpu_property(ec):
 }
 
 PRE_CONFIGURE_HOOKS = {
-    'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path,
     'BLIS': pre_configure_hook_BLIS_a64fx,
+    'GObject-Introspection': pre_configure_hook_gobject_introspection,
+    'Extrae': pre_configure_hook_extrae,
     'GROMACS': pre_configure_hook_gromacs,
     'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic,
     'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep,