diff --git a/.github/workflows/scorecards.yml b/.github/workflows/scorecards.yml index 5b482763f2..91c2576b7d 100644 --- a/.github/workflows/scorecards.yml +++ b/.github/workflows/scorecards.yml @@ -10,7 +10,7 @@ on: - cron: '25 15 * * 3' push: branches: - - '2023.06-software.eessi.io' + - '*-software.eessi.io' # Declare default permissions as read only. permissions: read-all diff --git a/.github/workflows/test-software.eessi.io.yml b/.github/workflows/test-software.eessi.io.yml index 06c02c8834..d4d980901f 100644 --- a/.github/workflows/test-software.eessi.io.yml +++ b/.github/workflows/test-software.eessi.io.yml @@ -1,6 +1,10 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Check for missing software installations in software.eessi.io -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: + workflow_dispatch: permissions: contents: read # to fetch code (actions/checkout) jobs: diff --git a/.github/workflows/test_eessi_container_script.yml b/.github/workflows/test_eessi_container_script.yml index d31feb23fe..fb587ec489 100644 --- a/.github/workflows/test_eessi_container_script.yml +++ b/.github/workflows/test_eessi_container_script.yml @@ -1,6 +1,10 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests for eessi_container.sh script -on: [push, pull_request, workflow_dispatch] +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: + workflow_dispatch: permissions: contents: read # to fetch code (actions/checkout) jobs: @@ -45,7 +49,8 @@ jobs: elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then outfile=out_listrepos.txt ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI" ${outfile} + # make sure that the default EESSI software repository is available + grep "software.eessi.io" ${outfile} # test use of --list-repos with custom repos.cfg elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then @@ -57,11 +62,12 @@ jobs: echo "[EESSI/20HT.TP]" >> cfg/repos.cfg echo "repo_version = 20HT.TP" >> cfg/repos.cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile} - grep "EESSI" ${outfile} + # make sure that the default EESSI software repository is available + grep "software.eessi.io" ${outfile} export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg ./eessi_container.sh --verbose --list-repos | tee ${outfile2} - grep "[EESSI/2023.02]" ${outfile2} + grep "EESSI/20AB.CD" ${outfile2} # test use of --mode run elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then diff --git a/.github/workflows/test_licenses.yml b/.github/workflows/test_licenses.yml index 6b6387d47d..bf6965c620 100644 --- a/.github/workflows/test_licenses.yml +++ b/.github/workflows/test_licenses.yml @@ -1,6 +1,9 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Test software licenses -on: [push, pull_request] +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: permissions: contents: read # to fetch code (actions/checkout) jobs: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8249e50d9b..940614b6bb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,6 +1,9 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests -on: [push, pull_request] +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: permissions: contents: read # to fetch code (actions/checkout) jobs: diff --git a/.github/workflows/tests_archdetect.yml b/.github/workflows/tests_archdetect.yml index 4f5d3d174d..f5b42b3d55 100644 --- a/.github/workflows/tests_archdetect.yml +++ b/.github/workflows/tests_archdetect.yml @@ -1,6 +1,9 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests for eessi_archdetect.sh -on: [push, pull_request] +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: permissions: contents: read # to fetch code (actions/checkout) jobs: diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml index 6c356a9184..8581681d63 100644 --- a/.github/workflows/tests_init.yml +++ b/.github/workflows/tests_init.yml @@ -1,6 +1,9 @@ # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions name: Tests for init scripts -on: [push, pull_request] +on: + push: + branches: [ "*-software.eessi.io" ] + pull_request: permissions: contents: read # to fetch code (actions/checkout) jobs: diff --git a/bot/build.sh b/bot/build.sh index c9a362fdca..145be740d3 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -138,7 +138,7 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'" # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name") echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} diff --git a/bot/test.sh b/bot/test.sh index 04bff346cd..d3f3630ea8 100755 --- a/bot/test.sh +++ b/bot/test.sh @@ -160,7 +160,7 @@ echo "bot/test.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE} # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE} # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down # "source init/eessi_defaults" via sourcing init/minimal_eessi_env -export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name") +export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/$(cfg_get_value "repository" "repo_name") echo "bot/test.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'" # determine architecture to be used from entry .architecture in ${JOB_CFG_FILE} diff --git a/create_tarball.sh b/create_tarball.sh index 0a7669f73f..2dee665060 100755 --- a/create_tarball.sh +++ b/create_tarball.sh @@ -26,7 +26,8 @@ if [ ! -d ${software_dir} ]; then exit 2 fi -overlay_upper_dir="${eessi_tmpdir}/overlay-upper" +cvmfs_repo_name=${cvmfs_repo#/cvmfs/} +overlay_upper_dir="${eessi_tmpdir}/${cvmfs_repo_name}/overlay-upper" software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}" if [ ! -d ${software_dir_overlay} ]; then diff --git a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml index 3fe1e43d3f..5521b92398 100644 --- a/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/a64fx/eessi-2023.06-eb-4.9.2-2023a.yml @@ -3,3 +3,4 @@ easyconfigs: - foss-2023a.eb - SciPy-bundle-2023.07-gfbf-2023a.eb - ESPResSo-4.2.2-foss-2023a.eb + - ParaView-5.11.2-foss-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml new file mode 100644 index 0000000000..7a04eaa9d3 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - BLAST+-2.14.0-gompi-2022b.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml index 2536c4bcc5..af8cb2c0ff 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023a.yml @@ -1,4 +1,31 @@ easyconfigs: - BCFtools-1.18-GCC-12.3.0.eb + - BWA-0.7.18-GCCcore-12.3.0.eb + - CapnProto-1.0.1-GCCcore-12.3.0.eb + - DendroPy-4.6.1-GCCcore-12.3.0.eb + - DIAMOND-2.1.8-GCC-12.3.0.eb + - FastME-2.1.6.3-GCC-12.3.0.eb + - fastp-0.23.4-GCC-12.3.0.eb + - HMMER-3.4-gompi-2023a.eb + - IQ-TREE-2.3.5-gompi-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20955 + from-commit: 185f88b9a03d65a7fb74edc7acb4221e87e90784 + - KronaTools-2.8.1-GCCcore-12.3.0.eb + - LSD2-2.4.1-GCCcore-12.3.0.eb + - MAFFT-7.520-GCC-12.3.0-with-extensions.eb + - ncbi-vdb-3.0.10-gompi-2023a.eb + - MetalWalls-21.06.1-foss-2023a.eb + - QuantumESPRESSO-7.3.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20138 + from-commit: dbdaacc0739fdee91baa9123864ea4428cf21273 + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3338 + include-easyblocks-from-commit: 32e45bd1f2d916732ca5852d55d17fa4d99e388b + - CP2K-2023.1-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 + from-commit: a92667fe32396bbd4106243658625f7ff2adcd68 + - amdahl-0.3.1-gompi-2023a.eb - xarray-2023.9.0-gfbf-2023a.eb - SciTools-Iris-3.9.0-foss-2023a.eb diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml index 38bfa81142..883483f5d2 100644 --- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml +++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2023b.yml @@ -1,3 +1,17 @@ easyconfigs: - IPython-8.17.2-GCCcore-13.2.0.eb - dlb-3.4-gompi-2023b.eb + - pystencils-1.3.4-gfbf-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20889 + from-commit: c66c4788a17f7e4f55aa23f9fdb782aad97c9ce7 + - Extrae-4.2.0-gompi-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21017 + from-commit: 120f4d56efebd2bc61382db4c84a664a339c66cf + # see https://github.com/easybuilders/easybuild-easyblocks/pull/3393 + include-easyblocks-from-commit: c4951c78d62fa5cf8e9f6fe0ead212d2a4d7cb9c + - pyMBE-0.8.0-foss-2023b.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/21034 + from-commit: 76e7fc6657bab64bfbec826540a3a8f0040258f2 diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240706-eb-4.9.2-OpenFOAM-no-ftree-vectorize.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240706-eb-4.9.2-OpenFOAM-no-ftree-vectorize.yml new file mode 100644 index 0000000000..c12d244790 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240706-eb-4.9.2-OpenFOAM-no-ftree-vectorize.yml @@ -0,0 +1,12 @@ +# 2024.07.06 +# OpenFOAM 10 and 11 built with GCC 11.3.0 or 12.3.0 and -ftree-vectorize yields incorrect results, +# see https://github.com/easybuilders/easybuild-easyconfigs/issues/20927 +easyconfigs: + - OpenFOAM-10-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f + - OpenFOAM-11-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-zen4.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-zen4.yml new file mode 100644 index 0000000000..2c9b411736 --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars-zen4.yml @@ -0,0 +1,11 @@ +# 2024.07.19 +# GObject-Introspection sets $LD_LIBRARY_PATH (to many different paths, including $EPREFIX/lib) +# when calling gcc, and this causes a lot of issues for, especially, scripts using /bin/bash. +# +# This rebuild ensures (by using a new EasyBuild hook) that GObject-Introspection will not set +# environment variables that are configured to be filtered by EasyBuild. +# +# See https://github.com/EESSI/software-layer/issues/196 +easyconfigs: + - GObject-Introspection-1.76.1-GCCcore-12.3.0.eb + - GObject-Introspection-1.78.1-GCCcore-13.2.0.eb diff --git a/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml b/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml new file mode 100644 index 0000000000..a61cd9705b --- /dev/null +++ b/easystacks/software.eessi.io/2023.06/rebuilds/20240719-eb-4.9.2-GObject-Introspection-filter-envvars.yml @@ -0,0 +1,14 @@ +# 2024.07.19 +# GObject-Introspection sets $LD_LIBRARY_PATH (to many different paths, including $EPREFIX/lib) +# when calling gcc, and this causes a lot of issues for, especially, scripts using /bin/bash. +# +# This rebuild ensures (by using a new EasyBuild hook) that GObject-Introspection will not set +# environment variables that are configured to be filtered by EasyBuild. +# +# See https://github.com/EESSI/software-layer/issues/196 +easyconfigs: + - GObject-Introspection-1.74.0-GCCcore-12.2.0.eb + - GObject-Introspection-1.76.1-GCCcore-12.3.0.eb + - GObject-Introspection-1.78.1-GCCcore-13.2.0.eb + - at-spi2-core-2.46.0-GCCcore-12.2.0.eb + - at-spi2-core-2.49.91-GCCcore-12.3.0.eb diff --git a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml index 09f525364f..65a12695e4 100644 --- a/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml +++ b/easystacks/software.eessi.io/2023.06/zen4/eessi-2023.06-eb-4.9.2-2023a.yml @@ -2,3 +2,12 @@ easyconfigs: - ESPResSo-4.2.2-foss-2023a.eb - TensorFlow-2.13.0-foss-2023a.eb - R-4.3.2-gfbf-2023a.eb + - ParaView-5.11.2-foss-2023a.eb + - OpenFOAM-10-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f + - OpenFOAM-11-foss-2023a.eb: + options: + # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20958 + from-commit: dbadb2074464d816740ee0e95595c2cb31b6338f diff --git a/eb_hooks.py b/eb_hooks.py index e6f0cf0dd2..7f7e4e57bd 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -1,5 +1,6 @@ # Hooks to customize how EasyBuild installs software in EESSI # see https://docs.easybuild.io/en/latest/Hooks.html +import glob import os import re @@ -81,8 +82,12 @@ def post_ready_hook(self, *args, **kwargs): """ # 'parallel' easyconfig parameter is set via EasyBlock.set_parallel in ready step based on available cores. # here we reduce parallellism to only use half of that for selected software, - # to avoid failing builds/tests due to out-of-memory problems - if self.name in ['TensorFlow', 'libxc']: + # to avoid failing builds/tests due to out-of-memory problems; + memory_hungry_build = self.name in ['libxc', 'TensorFlow'] + # on A64FX systems, (HBM) memory is typically scarce, so we need to use fewer cores for some builds + cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR') + memory_hungry_build_a64fx = cpu_target == CPU_TARGET_A64FX and self.name in ['Qt5'] + if memory_hungry_build or memory_hungry_build_a64fx: parallel = self.cfg['parallel'] if parallel > 1: self.cfg['parallel'] = parallel // 2 @@ -294,6 +299,22 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs): raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!") +def parse_hook_CP2K_remove_deps_for_aarch64(ec, *args, **kwargs): + """ + Remove x86_64 specific dependencies for the CI and missing installations to pass on aarch64 + """ + if ec.name == 'CP2K' and ec.version in ('2023.1',): + if os.getenv('EESSI_CPU_FAMILY') == 'aarch64': + # LIBXSMM is not supported on ARM with GCC 12.2.0 and 12.3.0 + # See https://www.cp2k.org/dev:compiler_support + # See https://github.com/easybuilders/easybuild-easyconfigs/pull/20951 + # we need this hook because we check for missing installations for all CPU targets + # on an x86_64 VM in GitHub Actions (so condition based on ARCH in LAMMPS easyconfig is always true) + ec['dependencies'] = [dep for dep in ec['dependencies'] if dep[0] not in ('libxsmm',)] + else: + raise EasyBuildError("CP2K-specific hook triggered for non-CP2K easyconfig?!") + + def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs): """ Solve issues with compiling or running the tests on both @@ -353,6 +374,51 @@ def pre_configure_hook_BLIS_a64fx(self, *args, **kwargs): else: raise EasyBuildError("BLIS-specific hook triggered for non-BLIS easyconfig?!") +def pre_configure_hook_extrae(self, *args, **kwargs): + """ + Pre-configure hook for Extrae + - avoid use of 'which' in configure script + - specify correct path to binutils/zlib (in compat layer) + """ + if self.name == 'Extrae': + + # determine path to Prefix installation in compat layer via $EPREFIX + eprefix = get_eessi_envvar('EPREFIX') + + binutils_lib_path_glob_pattern = os.path.join(eprefix, 'usr', 'lib*', 'binutils', '*-linux-gnu', '2.*') + binutils_lib_path = glob.glob(binutils_lib_path_glob_pattern) + if len(binutils_lib_path) == 1: + self.cfg.update('configopts', '--with-binutils=' + binutils_lib_path[0]) + else: + raise EasyBuildError("Failed to isolate path for binutils libraries using %s, got %s", + binutils_lib_path_glob_pattern, binutils_lib_path) + + # zlib is a filtered dependency, so we need to manually specify it's location to avoid the host version + self.cfg.update('configopts', '--with-libz=' + eprefix) + + # replace use of 'which' with 'command -v', since 'which' is broken in EESSI build container; + # this must be done *after* running configure script, because initial configuration re-writes configure script, + # and problem due to use of which only pops up when running make ?! + self.cfg.update('prebuildopts', "cp config/mpi-macros.m4 config/mpi-macros.m4.orig && sed -i 's/`which /`command -v /g' config/mpi-macros.m4 && ") + else: + raise EasyBuildError("Extrae-specific hook triggered for non-Extrae easyconfig?!") + + +def pre_configure_hook_gobject_introspection(self, *args, **kwargs): + """ + pre-configure hook for GObject-Introspection: + - prevent GObject-Introspection from setting $LD_LIBRARY_PATH if EasyBuild is configured to filter it, see: + https://github.com/EESSI/software-layer/issues/196 + """ + if self.name == 'GObject-Introspection': + # inject a line that removes all items from runtime_path_envvar that are in $EASYBUILD_FILTER_ENVVARS + sed_cmd = r'sed -i "s@\(^\s*runtime_path_envvar = \)\(.*\)@' + sed_cmd += r'\1\2\n\1 [x for x in runtime_path_envvar if not x in os.environ.get(\'EASYBUILD_FILTER_ENV_VARS\', \'\').split(\',\')]@g"' + sed_cmd += ' %(start_dir)s/giscanner/ccompiler.py && ' + self.cfg.update('preconfigopts', sed_cmd) + else: + raise EasyBuildError("GObject-Introspection-specific hook triggered for non-GObject-Introspection easyconfig?!") + def pre_configure_hook_gromacs(self, *args, **kwargs): """ @@ -435,21 +501,6 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): raise EasyBuildError("WRF-specific hook triggered for non-WRF easyconfig?!") -def pre_configure_hook_atspi2core_filter_ld_library_path(self, *args, **kwargs): - """ - pre-configure hook for at-spi2-core: - - instruct GObject-Introspection's g-ir-scanner tool to not set $LD_LIBRARY_PATH - when EasyBuild is configured to filter it, see: - https://github.com/EESSI/software-layer/issues/196 - """ - if self.name == 'at-spi2-core': - if build_option('filter_env_vars') and 'LD_LIBRARY_PATH' in build_option('filter_env_vars'): - sed_cmd = 'sed -i "s/gir_extra_args = \[/gir_extra_args = \[\\n \'--lib-dirs-envvar=FILTER_LD_LIBRARY_PATH\',/g" %(start_dir)s/atspi/meson.build && ' - self.cfg.update('preconfigopts', sed_cmd) - else: - raise EasyBuildError("at-spi2-core-specific hook triggered for non-at-spi2-core easyconfig?!") - - def pre_test_hook(self,*args, **kwargs): """Main pre-test hook: trigger custom functions based on software name.""" if self.name in PRE_TEST_HOOKS: @@ -693,6 +744,7 @@ def inject_gpu_property(ec): 'CGAL': parse_hook_cgal_toolchainopts_precise, 'fontconfig': parse_hook_fontconfig_add_fonts, 'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64, + 'CP2K': parse_hook_CP2K_remove_deps_for_aarch64, 'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors, 'pybind11': parse_hook_pybind11_replace_catch2, 'Qt5': parse_hook_qt5_check_qtwebengine_disable, @@ -709,8 +761,9 @@ def inject_gpu_property(ec): } PRE_CONFIGURE_HOOKS = { - 'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path, 'BLIS': pre_configure_hook_BLIS_a64fx, + 'GObject-Introspection': pre_configure_hook_gobject_introspection, + 'Extrae': pre_configure_hook_extrae, 'GROMACS': pre_configure_hook_gromacs, 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, diff --git a/eessi_container.sh b/eessi_container.sh index fb14e2118f..e404b7ee18 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -70,7 +70,8 @@ export EESSI_REPOS_CFG_FILE="${EESSI_REPOS_CFG_DIR}/repos.cfg" display_help() { echo "usage: $0 [OPTIONS] [[--] SCRIPT or COMMAND]" echo " OPTIONS:" - echo " -a | --access {ro,rw} - ro (read-only), rw (read & write) [default: ro]" + echo " -a | --access {ro,rw} - sets access globally for all CVMFS repositories:" + echo " ro (read-only), rw (read & write) [default: ro]" echo " -b | --extra-bind-paths - specify extra paths to be bound into the container." echo " To specify multiple bind paths, separate by comma." echo " Example: '/src:/dest:ro,/src2:/dest2:rw'" @@ -89,8 +90,11 @@ display_help() { echo " MODE==install for a CUDA installation, MODE==run to" echo " attach a GPU, MODE==all for both [default: false]" echo " -r | --repository CFG - configuration file or identifier defining the" - echo " repository to use [default: EESSI via" - echo " default container, see --container]" + echo " repository to use; can be given multiple times;" + echo " CFG may include a suffix ',access={ro,rw}' to" + echo " overwrite the global access mode for this repository" + echo " [default: software.eessi.io via CVMFS config available" + echo " via default container, see --container]" echo " -u | --resume DIR/TGZ - resume a previous run from a directory or tarball," echo " where DIR points to a previously used tmp directory" echo " (check for output 'Using DIR as tmp ...' of a previous" @@ -123,7 +127,7 @@ STORAGE= LIST_REPOS=0 MODE="shell" SETUP_NVIDIA=0 -REPOSITORY="EESSI" +REPOSITORIES=() RESUME= SAVE= HTTP_PROXY=${http_proxy:-} @@ -179,7 +183,7 @@ while [[ $# -gt 0 ]]; do shift 2 ;; -r|--repository) - REPOSITORY="$2" + REPOSITORIES+=("$2") shift 2 ;; -s|--save) @@ -221,22 +225,53 @@ done set -- "${POSITIONAL_ARGS[@]}" +# define a list of CVMFS repositories that are accessible via the +# CVMFS config repository which is always mounted +# TODO instead of hard-coding the 'extra' and 'default' repositories here one +# could have another script in the GitHub and/or CVMFS repository which +# provides this "configuration" +declare -A eessi_cvmfs_repos=(["dev.eessi.io"]="extra", ["riscv.eessi.io"]="extra", ["software.eessi.io"]="default") +eessi_default_cvmfs_repo="software.eessi.io,access=${ACCESS}" + +# define a list of CVMFS repositories that are accessible via the +# configuration file provided via $EESSI_REPOS_CFG_FILE +declare -A cfg_cvmfs_repos=() +if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then + cfg_load ${EESSI_REPOS_CFG_FILE} + sections=$(cfg_sections) + while IFS= read -r repo_id + do + cfg_cvmfs_repos[${repo_id}]=${EESSI_REPOS_CFG_FILE} + done <<< "${sections}" +fi + if [[ ${LIST_REPOS} -eq 1 ]]; then - echo "Listing available repositories with format 'name [source]':" - echo " EESSI [default]" - if [[ -r ${EESSI_REPOS_CFG_FILE} ]]; then - cfg_load ${EESSI_REPOS_CFG_FILE} - sections=$(cfg_sections) - while IFS= read -r repo_id - do - echo " ${repo_id} [${EESSI_REPOS_CFG_FILE}]" - done <<< "${sections}" - fi + echo "Listing available repositories with format 'name [source[, 'default']]'." + echo "Note, without argument '--repository' the one labeled 'default' will be mounted." + for cvmfs_repo in "${!eessi_cvmfs_repos[@]}" + do + if [[ ${eessi_cvmfs_repos[${cvmfs_repo}]} == "default" ]] ; then + default_label=", default" + else + default_label="" + fi + echo " ${cvmfs_repo} [CVMFS config repo${default_label}]" + done + for cfg_repo in "${!cfg_cvmfs_repos[@]}" + do + echo " ${cfg_repo} [${cfg_cvmfs_repos[$cfg_repo]}]" + done exit 0 fi +# if REPOSITORIES is empty add default repository given above +if [[ ${#REPOSITORIES[@]} -eq 0 ]]; then + REPOSITORIES+=(${eessi_default_cvmfs_repo}) +fi + # 1. check if argument values are valid # (arg -a|--access) check if ACCESS is supported +# use the value as global setting, suffix to --repository can specify an access mode per repository if [[ "${ACCESS}" != "ro" && "${ACCESS}" != "rw" ]]; then fatal_error "unknown access method '${ACCESS}'" "${ACCESS_UNKNOWN_EXITCODE}" fi @@ -260,11 +295,47 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then fi fi -# TODO (arg -r|--repository) check if repository is known +# TODO (arg -r|--repository) check if all explicitly listed repositories are known # REPOSITORY_ERROR_EXITCODE -if [[ ! -z "${REPOSITORY}" && "${REPOSITORY}" != "EESSI" && ! -r ${EESSI_REPOS_CFG_FILE} ]]; then - fatal_error "arg '--repository ${REPOSITORY}' requires a cfg file at '${EESSI_REPOS_CFG_FILE}'" "${REPOSITORY_ERROR_EXITCODE}" -fi +# iterate over entries in REPOSITORIES and check if they are known +for cvmfs_repo in "${REPOSITORIES[@]}" +do + # split into name and access mode if ',access=' in $cvmfs_repo + if [[ ${cvmfs_repo} == *",access="* ]] ; then + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification + else + cvmfs_repo_name="${cvmfs_repo}" + fi + if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo_name}]}" && ! -n ${cfg_cvmfs_repos[${cvmfs_repo_name}]} ]]; then + fatal_error "The repository '${cvmfs_repo_name}' is not an EESSI CVMFS repository or it is not known how to mount it (could be due to a typo or missing configuration). Run '$0 -l' to obtain a list of available repositories." "${REPOSITORY_ERROR_EXITCODE}" + fi +done + +# make sure each repository is only listed once +declare -A listed_repos=() +for cvmfs_repo in "${REPOSITORIES[@]}" +do + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode + [[ ${VERBOSE} -eq 1 ]] && echo "checking for duplicates: '${cvmfs_repo}' and '${cvmfs_repo_name}'" + # if cvmfs_repo_name is not in eessi_cvmfs_repos, assume it's in cfg_cvmfs_repos + # and obtain actual repo_name from config + cfg_repo_id='' + if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then + [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..." + # cvmfs_repo_name is actually a repository ID, use that to obtain + # the actual name from the EESSI_REPOS_CFG_FILE + cfg_repo_id=${cvmfs_repo_name} + cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") + fi + if [[ -n "${listed_repos[${cvmfs_repo_name}]}" ]] ; then + via_cfg="" + if [[ -n "${cfg_repo_id}" ]] ; then + via_cfg=" (via repository ID '${cfg_repo_id}')" + fi + fatal_error "CVMFS repository '${cvmfs_repo_name}'${via_cfg} listed multiple times" + fi + listed_repos+=([${cvmfs_repo_name}]=true) +done # TODO (arg -u|--resume) check if it exists, if user has read permission, # if it contains data from a previous run @@ -340,19 +411,30 @@ fi # directory structure should be: # ${EESSI_HOST_STORAGE} # |-singularity_cache -# |-${CVMFS_VAR_LIB} -# |-${CVMFS_VAR_RUN} -# |-overlay-upper -# |-overlay-work # |-home # |-repos_cfg -# |-opt-eessi (unless otherwise specificed for host_injections) +# |-${CVMFS_VAR_LIB} +# |-${CVMFS_VAR_RUN} +# |-CVMFS_REPO_1 +# | |-repo_settings.sh (name, id, access, host_injections) +# | |-overlay-upper +# | |-overlay-work +# | |-opt-eessi (unless otherwise specificed for host_injections) +# |-CVMFS_REPO_n +# |-repo_settings.sh (name, id, access, host_injections) +# |-overlay-upper +# |-overlay-work +# |-opt-eessi (unless otherwise specificed for host_injections) # tmp dir for EESSI EESSI_TMPDIR=${EESSI_HOST_STORAGE} mkdir -p ${EESSI_TMPDIR} [[ ${VERBOSE} -eq 1 ]] && echo "EESSI_TMPDIR=${EESSI_TMPDIR}" +# TODO make this specific to repository? +# TODO move this code to when we already know which repositories we want to access +# actually we should know this already here, but we should rather move this to +# where repository args are being processed # Set host_injections directory and ensure it is a writable directory (if user provided) if [ -z ${USER_HOST_INJECTIONS+x} ]; then # Not set, so use our default @@ -444,6 +526,7 @@ fi # define paths to add to SINGULARITY_BIND (added later when all BIND mounts are defined) BIND_PATHS="${EESSI_CVMFS_VAR_LIB}:/var/lib/cvmfs,${EESSI_CVMFS_VAR_RUN}:/var/run/cvmfs,${HOST_INJECTIONS}:/opt/eessi" + # provide a '/tmp' inside the container BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}:${TMP_IN_CONTAINER}" if [[ ! -z ${EXTRA_BIND_PATHS} ]]; then @@ -489,91 +572,111 @@ fi # set up repository config (always create directory repos_cfg and populate it with info when # arg -r|--repository is used) mkdir -p ${EESSI_TMPDIR}/repos_cfg -if [[ "${REPOSITORY}" == "EESSI" ]]; then - # need to source defaults as late as possible (see other sourcing below) - source ${TOPDIR}/init/eessi_defaults - - # strip "/cvmfs/" from default setting - repo_name=${EESSI_CVMFS_REPO/\/cvmfs\//} -else - # TODO implement more flexible specification of repo cfgs - # REPOSITORY => repo-id OR repo-cfg-file (with a single section) OR - # repo-cfg-file:repo-id (repo-id defined in repo-cfg-file) - # - # for now, assuming repo-id is defined in config file pointed to - # EESSI_REPOS_CFG_FILE, which is to be copied into the working directory - # (could also become part of the software layer to define multiple - # standard EESSI repositories) - cfg_load ${EESSI_REPOS_CFG_FILE} - - # copy repos.cfg to job directory --> makes it easier to inspect the job - cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. - - # cfg file should include: repo_name, repo_version, config_bundle, - # map { local_filepath -> container_filepath } - # - # repo_name_domain is the domain part of the repo_name, e.g., - # eessi.io for software.eessi.io - # - # where config bundle includes the files (-> target location in container) - # - default.local -> /etc/cvmfs/default.local - # contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ... - # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf - # contains CVMFS settings, e.g., CVMFS_SERVER_URL (Stratum 1s), - # CVMFS_KEYS_DIR, CVMFS_USE_GEOAPI, ... - # - ${repo_name_domain}/ -> /etc/cvmfs/keys/${repo_name_domain} - # a directory that contains the public key to access the repository, key - # itself then doesn't need to be BIND mounted - # - ${repo_name_domain}/${repo_name}.pub - # (-> /etc/cvmfs/keys/${repo_name_domain}/${repo_name}.pub - # the public key to access the repository, key itself is BIND mounted - # via directory ${repo_name_domain} - repo_name=$(cfg_get_value ${REPOSITORY} "repo_name") - # derive domain part from repo_name (everything after first '.') - repo_name_domain=${repo_name#*.} - repo_version=$(cfg_get_value ${REPOSITORY} "repo_version") - config_bundle=$(cfg_get_value ${REPOSITORY} "config_bundle") - config_map=$(cfg_get_value ${REPOSITORY} "config_map") - - # convert config_map into associative array cfg_file_map - cfg_init_file_map "${config_map}" - [[ ${VERBOSE} -eq 1 ]] && cfg_print_map - - # use information to set up dir ${EESSI_TMPDIR}/repos_cfg, - # define BIND mounts and override repo name and version - # check if config_bundle exists, if so, unpack it into ${EESSI_TMPDIR}/repos_cfg - # if config_bundle is relative path (no '/' at start) prepend it with - # EESSI_REPOS_CFG_DIR - config_bundle_path= - if [[ ! "${config_bundle}" =~ ^/ ]]; then - config_bundle_path=${EESSI_REPOS_CFG_DIR}/${config_bundle} - else - config_bundle_path=${config_bundle} - fi +[[ ${VERBOSE} -eq 1 ]] && echo +[[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS before processing REPOSITORIES\n BIND_PATHS=${BIND_PATHS}" +[[ ${VERBOSE} -eq 1 ]] && echo +# iterate over repositories in array REPOSITORIES +for cvmfs_repo in "${REPOSITORIES[@]}" +do + [[ ${VERBOSE} -eq 1 ]] && echo "process CVMFS repo spec '${cvmfs_repo}'" + # split into name and access mode if ',access=' in $cvmfs_repo + if [[ ${cvmfs_repo} == *",access="* ]] ; then + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification + cvmfs_repo_access=${cvmfs_repo/*,access=/} # remove repo name part + else + cvmfs_repo_name="${cvmfs_repo}" + cvmfs_repo_access="${ACCESS}" # use globally defined access mode + fi + # if cvmfs_repo_name is in cfg_cvmfs_repos, it is a "repository ID" and was + # derived from information in EESSI_REPOS_CFG_FILE, namely the section + # names in that .ini-type file + # in the if-block below, we'll use cfg_repo_id to refer to that ID + # we need to process/provide the config from EESSI_REPOS_CFG_FILE, such + # that the necessary information for accessing a CVMFS repository is made + # available inside the container + if [[ -n "${cfg_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then + cfg_repo_id=${cvmfs_repo_name} + + # obtain CVMFS repository name from section for the given ID + cfg_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") + # derive domain part from (cfg_)repo_name (everything after first '.') + repo_name_domain=${repo_name#*.} + + # cfg_cvmfs_repos is populated through reading the file pointed to by + # EESSI_REPOS_CFG_FILE. We need to copy that file and data it needs + # into the job's working directory. + + # copy repos.cfg to job directory --> makes it easier to inspect the job + cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/. + + # cfg file should include sections (one per CVMFS repository to be mounted) + # with each section containing the settings: + # - repo_name, + # - repo_version, + # - config_bundle, and + # - a map { filepath_in_bundle -> container_filepath } + # + # The config_bundle includes the files which are mapped ('->') to a target + # location in container: + # - default.local -> /etc/cvmfs/default.local + # contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ... + # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf + # contains CVMFS settings, e.g., CVMFS_SERVER_URL (Stratum 1s), + # CVMFS_KEYS_DIR, CVMFS_USE_GEOAPI, ... + # - ${repo_name_domain}/ -> /etc/cvmfs/keys/${repo_name_domain} + # a directory that contains the public key to access the repository, key + # itself then doesn't need to be BIND mounted + # - ${repo_name_domain}/${cfg_repo_name}.pub + # (-> /etc/cvmfs/keys/${repo_name_domain}/${cfg_repo_name}.pub + # the public key to access the repository, key itself is BIND mounted + # via directory ${repo_name_domain} + cfg_repo_version=$(cfg_get_value ${cfg_repo_id} "repo_version") + cfg_config_bundle=$(cfg_get_value ${cfg_repo_id} "config_bundle") + cfg_config_map=$(cfg_get_value ${cfg_repo_id} "config_map") + + # convert cfg_config_map into associative array cfg_file_map + cfg_init_file_map "${cfg_config_map}" + [[ ${VERBOSE} -eq 1 ]] && cfg_print_map + + # use information to set up dir ${EESSI_TMPDIR}/repos_cfg and define + # BIND mounts + # check if config_bundle exists, if so, unpack it into + # ${EESSI_TMPDIR}/repos_cfg; if it doesn't, exit with an error + # if config_bundle is relative path (no '/' at start) prepend it with + # EESSI_REPOS_CFG_DIR + config_bundle_path= + if [[ ! "${cfg_config_bundle}" =~ ^/ ]]; then + config_bundle_path=${EESSI_REPOS_CFG_DIR}/${cfg_config_bundle} + else + config_bundle_path=${cfg_config_bundle} + fi - if [[ ! -r ${config_bundle_path} ]]; then - fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} - fi + if [[ ! -r ${config_bundle_path} ]]; then + fatal_error "config bundle '${config_bundle_path}' is not readable" ${REPOSITORY_ERROR_EXITCODE} + fi - # only unpack config_bundle if we're not resuming from a previous run - if [[ -z ${RESUME} ]]; then - tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg - fi + # only unpack cfg_config_bundle if we're not resuming from a previous run + if [[ -z ${RESUME} ]]; then + tar xf ${config_bundle_path} -C ${EESSI_TMPDIR}/repos_cfg + fi - for src in "${!cfg_file_map[@]}" - do - target=${cfg_file_map[${src}]} - BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" - done - export EESSI_VERSION_OVERRIDE=${repo_version} - export EESSI_CVMFS_REPO_OVERRIDE="/cvmfs/${repo_name}" - # need to source defaults as late as possible (after *_OVERRIDEs) - source ${TOPDIR}/init/eessi_defaults -fi + for src in "${!cfg_file_map[@]}" + do + target=${cfg_file_map[${src}]} + # if target is alreay BIND mounted, exit with an error + if [[ ${BIND_PATHS} =~ "${target}" ]]; then + fatal_error "target '${target}' is already listed in paths to bind mount into the container ('${BIND_PATHS}')" ${REPOSITORY_ERROR_EXITCODE} + fi + BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/${src}:${target}" + done + fi + [[ ${VERBOSE} -eq 1 ]] && echo -e "BIND_PATHS after processing '${cvmfs_repo}'\n BIND_PATHS=${BIND_PATHS}" + [[ ${VERBOSE} -eq 1 ]] && echo +done # if http_proxy is not empty, we assume that the machine accesses internet # via a proxy. then we need to add CVMFS_HTTP_PROXY to -# ${EESSI_TMPDIR}/repos_cfg/default.local on host (and possibly add a BIND +# ${EESSI_TMPDIR}/repos_cfg/default.local on the host (and possibly add a BIND # MOUNT if it was not yet in BIND_PATHS) if [[ ! -z ${http_proxy} ]]; then # TODO tolerate other formats for proxy URLs, for now assume format is @@ -591,45 +694,91 @@ if [[ ! -z ${http_proxy} ]]; then [[ ${VERBOSE} -eq 1 ]] && cat ${EESSI_TMPDIR}/repos_cfg/default.local # if default.local is not BIND mounted into container, add it to BIND_PATHS - if [[ ! ${BIND_PATHS} =~ "${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" ]]; then - export BIND_PATHS="${BIND_PATHS},${EESSI_TMPDIR}/repos_cfg/default.local:/etc/cvmfs/default.local" + src=${EESSI_TMPDIR}/repos_cfg/default.local + target=/etc/cvmfs/default.local + if [[ ${BIND_PATHS} =~ "${target}" ]]; then + fatal_error "BIND target in '${src}:${target}' is already in paths to be bind mounted into the container ('${BIND_PATHS}')" ${REPOSITORY_ERROR_EXITCODE} fi + BIND_PATHS="${BIND_PATHS},${src}:${target}" fi # 4. set up vars and dirs specific to a scenario declare -a EESSI_FUSE_MOUNTS=() -# always mount cvmfs-config repo (to get access to software.eessi.io) +# always mount cvmfs-config repo (to get access to EESSI repositories such as software.eessi.io) EESSI_FUSE_MOUNTS+=("--fusemount" "container:cvmfs2 cvmfs-config.cern.ch /cvmfs/cvmfs-config.cern.ch") -if [[ "${ACCESS}" == "ro" ]]; then - export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs/${repo_name}" +# iterate over REPOSITORIES and either use repository-specific access mode or global setting (possibly a global default) +for cvmfs_repo in "${REPOSITORIES[@]}" +do + unset cfg_repo_id + [[ ${VERBOSE} -eq 1 ]] && echo "add fusemount options for CVMFS repo '${cvmfs_repo}'" + # split into name and access mode if ',access=' in $cvmfs_repo + if [[ ${cvmfs_repo} == *",access="* ]] ; then + cvmfs_repo_name=${cvmfs_repo/,access=*/} # remove access mode specification + cvmfs_repo_access=${cvmfs_repo/*,access=/} # remove repo name part + else + cvmfs_repo_name="${cvmfs_repo}" + cvmfs_repo_access="${ACCESS}" # use globally defined access mode + fi + # obtain cvmfs_repo_name from EESSI_REPOS_CFG_FILE if cvmfs_repo is in cfg_cvmfs_repos + if [[ ${cfg_cvmfs_repos[${cvmfs_repo_name}]} ]]; then + [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..." + # cvmfs_repo_name is actually a repository ID, use that to obtain + # the actual name from the EESSI_REPOS_CFG_FILE + cfg_repo_id=${cvmfs_repo_name} + cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name") + fi - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - export EESSI_FUSE_MOUNTS -fi + # always create a directory for the repository (e.g., to store settings, ...) + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name} -if [[ "${ACCESS}" == "rw" ]]; then - mkdir -p ${EESSI_TMPDIR}/overlay-upper - mkdir -p ${EESSI_TMPDIR}/overlay-work + # add fusemount options depending on requested access mode ('ro' - read-only; 'rw' - read & write) + if [[ ${cvmfs_repo_access} == "ro" ]] ; then + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}" - # set environment variables for fuse mounts in Singularity container - export EESSI_READONLY="container:cvmfs2 ${repo_name} /cvmfs_ro/${repo_name}" + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + export EESSI_FUSE_MOUNTS + elif [[ ${cvmfs_repo_access} == "rw" ]] ; then + # use repo-specific overlay directories + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper + mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work + [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})" - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") + # set environment variables for fuse mounts in Singularity container + export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}" - EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" - EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${repo_name}" - EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/overlay-upper" - EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/overlay-work" - EESSI_WRITABLE_OVERLAY+=" ${EESSI_CVMFS_REPO}" - export EESSI_WRITABLE_OVERLAY + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}") - EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}") - export EESSI_FUSE_MOUNTS -fi + EESSI_WRITABLE_OVERLAY="container:fuse-overlayfs" + EESSI_WRITABLE_OVERLAY+=" -o lowerdir=/cvmfs_ro/${cvmfs_repo_name}" + EESSI_WRITABLE_OVERLAY+=" -o upperdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper" + EESSI_WRITABLE_OVERLAY+=" -o workdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-work" + EESSI_WRITABLE_OVERLAY+=" /cvmfs/${cvmfs_repo_name}" + export EESSI_WRITABLE_OVERLAY + EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_WRITABLE_OVERLAY}") + export EESSI_FUSE_MOUNTS + else + echo -e "ERROR: access mode '${cvmfs_repo_access}' for CVMFS repository\n '${cvmfs_repo_name}' is not known" + exit ${REPOSITORY_ERROR_EXITCODE} + fi + # create repo_settings.sh file in ${EESSI_TMPDIR}/${cvmfs_repo_name} to store + # (intention is that the file could be just sourced to obtain the settings) + # repo_name = ${cvmfs_repo_name} + # repo_id = ${cfg_repo_id} # empty if not an EESSI repo + # repo_access = ${cvmfs_repo_access} + # repo_host_injections = [ {"src_path":"target_path"}... ] # TODO + settings= + #[[ -n ${cfg_repo_id} ]] && settings="[${cvmfs_repo_name}]\n" || settings="[${cfg_repo_id}]\n" + settings="${settings}repo_name = ${cvmfs_repo_name}\n" + settings="${settings}repo_id = ${cfg_repo_id}\n" + settings="${settings}repo_access = ${cvmfs_repo_access}\n" + # TODO iterate over host_injections (first need means to define them (globally and/or per repository) + # settings="${settings}repo_host_injections = ${host_injections}\n" + echo -e "${settings}" > ${EESSI_TMPDIR}/${cvmfs_repo_name}/repo_settings.sh +done # 5. run container # final settings diff --git a/init/bash b/init/bash index 93471e075e..4ad09f6a1b 100644 --- a/init/bash +++ b/init/bash @@ -1,7 +1,7 @@ function show_msg { # only echo msg if EESSI_SILENT is unset msg=$1 - if [[ ! -v EESSI_SILENT ]]; then + if [[ -z ${EESSI_SILENT+x} ]]; then echo "$msg" fi } diff --git a/init/eessi_archdetect.sh b/init/eessi_archdetect.sh index e89bc75e13..ddfed1bfae 100755 --- a/init/eessi_archdetect.sh +++ b/init/eessi_archdetect.sh @@ -89,7 +89,7 @@ cpupath(){ *) log "ERROR" "cpupath: Unsupported CPU architecture $machine_type" esac # spec files are located in a subfolder with this script - local base_dir=$(dirname $(realpath $0)) + local base_dir=$(dirname $(readlink -f $0)) update_arch_specs "$base_dir/arch_specs/${spec_file}" # Identify the host CPU vendor diff --git a/init/eessi_defaults b/init/eessi_defaults index c1ce82d1ce..654a829425 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -12,7 +12,7 @@ if [[ $(uname -m) == "riscv64" ]]; then export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/riscv.eessi.io}" export EESSI_VERSION="${EESSI_VERSION_OVERRIDE:=20240402}" - if [[ ! -v EESSI_SILENT ]]; then + if [[ -z ${EESSI_SILENT+x} ]]; then echo "RISC-V architecture detected, but there is no RISC-V support yet in the production repository." echo "Automatically switching to version ${EESSI_VERSION} of the RISC-V development repository ${EESSI_CVMFS_REPO}." echo "For more details about this repository, see https://www.eessi.io/docs/repositories/riscv.eessi.io/." diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables index 78851a9c95..8c10b1fca8 100644 --- a/init/eessi_environment_variables +++ b/init/eessi_environment_variables @@ -1,6 +1,6 @@ # this script is *sourced*, not executed, so can't rely on $0 to determine path to self # $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028 -EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE)) +EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) function error() { echo -e "\e[31mERROR: $1\e[0m" >&2 @@ -10,7 +10,7 @@ function error() { function show_msg { # only echo msg if EESSI_SILENT is unset msg=$1 - if [[ ! -v EESSI_SILENT ]]; then + if [[ -z ${EESSI_SILENT+x} ]]; then echo "$msg" fi } @@ -106,6 +106,15 @@ if [ -d $EESSI_PREFIX ]; then false fi + # Fix wrong path for RHEL >=8 libcurl + # This is required here because we ship curl in our compat layer. If we only provided + # curl as a module file we could instead do this via a `modluafooter` in an EasyBuild + # hook (or via an Lmod hook) + rhel_libcurl_file="/etc/pki/tls/certs/ca-bundle.crt" + if [ -f $rhel_libcurl_file ]; then + show_msg "Found libcurl CAs file at RHEL location, setting CURL_CA_BUNDLE" + export CURL_CA_BUNDLE=$rhel_libcurl_file + fi else error "EESSI software layer at $EESSI_SOFTWARE_PATH not found!" diff --git a/init/minimal_eessi_env b/init/minimal_eessi_env index 03b337ccf7..5273f27862 100644 --- a/init/minimal_eessi_env +++ b/init/minimal_eessi_env @@ -2,7 +2,7 @@ # # this script is *sourced*, not executed, so can't rely on $0 to determine path to self # $BASH_SOURCE points to correct path, see also http://mywiki.wooledge.org/BashFAQ/028 -EESSI_INIT_DIR_PATH=$(dirname $(realpath $BASH_SOURCE)) +EESSI_INIT_DIR_PATH=$(dirname $(readlink -f $BASH_SOURCE)) # set up defaults: EESSI_CVMFS_REPO, EESSI_VERSION # script takes *_OVERRIDEs into account