From c95623b354efae35996400606fb037819062cd3e Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 30 Aug 2023 11:51:49 +0200 Subject: [PATCH 1/9] {2023.06}[foss/2021b] TensorFlow v2.7.1 --- eessi-2023.06-eb-4.8.0-2021b.yml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 eessi-2023.06-eb-4.8.0-2021b.yml diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml new file mode 100644 index 0000000000..2704f16698 --- /dev/null +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -0,0 +1,2 @@ +easyconfigs: + - TensorFlow-2.7.1-foss-2021b.eb From a866504ef8ef1b1c4b9ea97b179d571dfbe950f6 Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 30 Aug 2023 12:51:03 +0200 Subject: [PATCH 2/9] adding eessi-2023.06-eb-4.8.0-2021b.yml to stack --- .github/workflows/test_eessi.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 65994cab2d..c51d313490 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -26,6 +26,7 @@ jobs: - eessi-2023.06-eb-4.7.2-2022a.yml - eessi-2023.06-eb-4.7.2-2022b.yml - eessi-2023.06-eb-4.7.2-system.yml + - eessi-2023.06-eb-4.8.0-2021b.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 From d248297701505c20b5567b21a031f6f991990cdf Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 30 Aug 2023 13:47:32 +0200 Subject: [PATCH 3/9] adding eb 4.8.0 --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index 8110135cc0..b103938bb5 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -172,7 +172,7 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -for eb_version in '4.7.2'; do +for eb_version in '4.7.2', '4.8.0'; do # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} From 12176e9f6520401e24779f13e37cd2114507432f Mon Sep 17 00:00:00 2001 From: lara Date: Wed, 30 Aug 2023 14:37:17 +0200 Subject: [PATCH 4/9] adjust EESSI-pilot-install-software.sh --- EESSI-pilot-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index b103938bb5..f47ff62e95 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -172,7 +172,7 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -for eb_version in '4.7.2', '4.8.0'; do +for eb_version in '4.8.0'; do # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} From bc92d027dc098c730b0ec06b6c053a7615a91ec7 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 31 Aug 2023 09:19:55 +0200 Subject: [PATCH 5/9] add easyblock-from-pr --- EESSI-pilot-install-software.sh | 2 +- eessi-2023.06-eb-4.8.0-2021b.yml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index f47ff62e95..29fbb7aa62 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -172,7 +172,7 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -for eb_version in '4.8.0'; do +for eb_version in '4.7.2' '4.8.0'; do # load EasyBuild module (will be installed if it's not available yet) source ${TOPDIR}/load_easybuild_module.sh ${eb_version} diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml index 2704f16698..2598e351cf 100644 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -1,2 +1,4 @@ easyconfigs: - TensorFlow-2.7.1-foss-2021b.eb + options: + include-easyblocks-from-pr: 2218 From 7ca8d012d8da0ff1eb2f2fe17de6082a572c1ce6 Mon Sep 17 00:00:00 2001 From: lara Date: Thu, 31 Aug 2023 16:09:52 +0200 Subject: [PATCH 6/9] resolve parse error --- eessi-2023.06-eb-4.8.0-2021b.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml index 2598e351cf..d1c07c3f4b 100644 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -1,4 +1,4 @@ easyconfigs: - - TensorFlow-2.7.1-foss-2021b.eb + - TensorFlow-2.7.1-foss-2021b.eb: options: include-easyblocks-from-pr: 2218 From d5f73dba613993a8ffcd00c7ea7c1fa12aa763a3 Mon Sep 17 00:00:00 2001 From: lara Date: Fri, 1 Sep 2023 11:13:19 +0200 Subject: [PATCH 7/9] removing cURL from DEPS_TO_FILTER --- configure_easybuild | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure_easybuild b/configure_easybuild index 23f3920154..7dca1ce682 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -26,7 +26,7 @@ fi # note: filtering Bison may break some installations, like Qt5 (see https://github.com/EESSI/software-layer/issues/49) # filtering pkg-config breaks R-bundle-Bioconductor installation (see also https://github.com/easybuilders/easybuild-easyconfigs/pull/11104) # problems occur when filtering pkg-config with gnuplot too (picks up Lua 5.1 from $EPREFIX rather than from Lua 5.3 dependency) -DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,cURL,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib +DEPS_TO_FILTER=Autoconf,Automake,Autotools,binutils,bzip2,DBus,flex,gettext,gperf,help2man,intltool,libreadline,libtool,Lua,M4,makeinfo,ncurses,util-linux,XZ,zlib # For aarch64 we need to also filter out Yasm. # See https://github.com/easybuilders/easybuild-easyconfigs/issues/11190 if [[ "$EESSI_CPU_FAMILY" == "aarch64" ]]; then From b8749d31207da91e6cf5a49ed2153f0918e5e217 Mon Sep 17 00:00:00 2001 From: Lara Ramona Peeters <49882639+laraPPr@users.noreply.github.com> Date: Wed, 6 Sep 2023 10:11:48 +0200 Subject: [PATCH 8/9] Update eessi-2023.06-eb-4.8.0-2021b.yml Co-authored-by: Kenneth Hoste --- eessi-2023.06-eb-4.8.0-2021b.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eessi-2023.06-eb-4.8.0-2021b.yml b/eessi-2023.06-eb-4.8.0-2021b.yml index d1c07c3f4b..082a74423e 100644 --- a/eessi-2023.06-eb-4.8.0-2021b.yml +++ b/eessi-2023.06-eb-4.8.0-2021b.yml @@ -1,4 +1,6 @@ easyconfigs: - TensorFlow-2.7.1-foss-2021b.eb: + # correctly determine path to active binutils in TensorFlow easyblock, + # see https://github.com/easybuilders/easybuild-easyblocks/pull/2218 options: include-easyblocks-from-pr: 2218 From d24fa09b58b4b8b78007cb41200bc144c3f7a03b Mon Sep 17 00:00:00 2001 From: Kenneth Hoste Date: Wed, 27 Sep 2023 11:29:52 +0200 Subject: [PATCH 9/9] implement hook to limit parallellism to half the available cores for software that requires a lot of memory during the build/test, like TensorFlow --- eb_hooks.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index 488e53d929..0e80ba5b3e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -63,6 +63,21 @@ def parse_hook(ec, *args, **kwargs): PARSE_HOOKS[ec.name](ec, eprefix) +def post_ready_hook(self, *args, **kwargs): + """ + Post-ready hook: limit parallellism for selected builds, because they require a lot of memory per used core. + """ + # 'parallel' easyconfig parameter is set via EasyBlock.set_parallel in ready step based on available cores. + # here we reduce parallellism to only use half of that for selected software, + # to avoid failing builds/tests due to out-of-memory problems + if self.name in ['TensorFlow']: + parallel = self.cfg['parallel'] + if parallel > 1: + self.cfg['parallel'] = parallel // 2 + msg = "limiting parallelism to %s (was %s) for %s to avoid out-of-memory failures during building/testing" + print_msg(msg % (self.cfg['parallel'], parallel, self.name), log=self.log) + + def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions."""