From 36708102427f9ed97a8a706235ee7cc3ffc9030b Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 29 Feb 2024 17:18:17 +0100 Subject: [PATCH 1/9] Make EESSI compatible with archspec 0.2.3+ --- .github/workflows/tests.yml | 31 ---------------------- EESSI-install-software.sh | 15 ++++++++--- bot/inspect.sh | 15 ++++++++--- eessi_software_subdir.py | 53 ------------------------------------- 4 files changed, 22 insertions(+), 92 deletions(-) delete mode 100644 .github/workflows/tests.yml delete mode 100755 eessi_software_subdir.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index faa7eb82ff..0000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,31 +0,0 @@ -# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Tests -on: [push, pull_request] -permissions: - contents: read # to fetch code (actions/checkout) -jobs: - build: - runs-on: ubuntu-20.04 - strategy: - matrix: - python: [3.6, 3.7, 3.8, 3.9, '3.10'] - fail-fast: false - steps: - - name: checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: set up Python - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - with: - python-version: ${{matrix.python}} - architecture: x64 - - - name: install Python packages - run: | - pip install archspec==0.2.2 - - - name: test eessi_software_subdir.py script - run: | - ./eessi_software_subdir.py | tee out - grep '^x86_64/' out - egrep '/intel/|/amd/' out diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 69de9d1997..562d1189bc 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -122,20 +122,27 @@ fi # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory export PYTHONPYCACHEPREFIX=$TMPDIR/pycache -DETECTION_PARAMETERS='' GENERIC=0 EB='eb' if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then echo_yellow ">> GENERIC build requested, taking appropriate measures!" - DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic" GENERIC=1 EB='eb --optarch=GENERIC' fi echo ">> Determining software subdirectory to use for current build host..." if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) - echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" + possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath) + + if [[ "$GENERIC" -eq 1 ]]; then + # Last path is the generic case + override_subdir="${possible_subdir_paths##*:}" + else + # First path is the best option (according to archspec) + override_subdir="${possible_subdir_paths%%:*}" + fi + export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir" + echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'archdetect' script ($override_subdir from $possible_subdir_paths)" else echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" fi diff --git a/bot/inspect.sh b/bot/inspect.sh index 9d1fa87e1f..a1b439f70b 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -338,12 +338,10 @@ fi # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory export PYTHONPYCACHEPREFIX=$TMPDIR/pycache -DETECTION_PARAMETERS='' GENERIC=0 EB='eb' if [[ "$EASYBUILD_OPTARCH" == "GENERIC" || "$EESSI_SOFTWARE_SUBDIR_OVERRIDE" == *"/generic" ]]; then echo_yellow ">> GENERIC build requested, taking appropriate measures!" - DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic" GENERIC=1 export EASYBUILD_OPTARCH=GENERIC EB='eb --optarch=GENERIC' @@ -351,8 +349,17 @@ fi echo ">> Determining software subdirectory to use for current build host..." if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) - echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" + possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath) + + if [[ "$GENERIC" -eq 1 ]]; then + # Last path is the generic case + override_subdir="${possible_subdir_paths##*:}" + else + # First path is the best option (according to archspec) + override_subdir="${possible_subdir_paths%%:*}" + fi + export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir" + echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'archdetect' script ($override_subdir from $possible_subdir_paths)" else echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" fi diff --git a/eessi_software_subdir.py b/eessi_software_subdir.py deleted file mode 100755 index af9df722d9..0000000000 --- a/eessi_software_subdir.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# -# Determine EESSI software subdirectory to use for current build host, using archspec -# -import os -import argparse -from archspec.cpu.detect import compatible_microarchitectures, raw_info_dictionary - -software_subdir = os.getenv('EESSI_SOFTWARE_SUBDIR_OVERRIDE') -if software_subdir is None: - - parser = argparse.ArgumentParser(description='Determine EESSI software subdirectory to use for current build host.') - parser.add_argument('--generic', dest='generic', action='store_true', - default=False, help='Use generic for CPU name.') - args = parser.parse_args() - - # we can't directly use archspec.cpu.host(), because we may get back a virtual microarchitecture like x86_64_v3... - def sorting_fn(item): - """Helper function to sort compatible microarchitectures.""" - return len(item.ancestors), len(item.features) - - raw_cpu_info = raw_info_dictionary() - compat_targets = compatible_microarchitectures(raw_cpu_info) - - # filter out generic targets - non_generic_compat_targets = [t for t in compat_targets if t.vendor != "generic"] - - # Filter the candidates to be descendant of the best generic candidate - best_generic = max([t for t in compat_targets if t.vendor == "generic"], key=sorting_fn) - best_compat_targets = [t for t in non_generic_compat_targets if t > best_generic] - - if best_compat_targets: - host_cpu = max(best_compat_targets, key=sorting_fn) - else: - host_cpu = max(non_generic_compat_targets, key=sorting_fn) - - vendors = { - 'GenuineIntel': 'intel', - 'AuthenticAMD': 'amd', - } - - vendor = vendors.get(host_cpu.vendor) - - if args.generic: - parts = (host_cpu.family.name, 'generic') - elif vendor: - parts = (host_cpu.family.name, vendor, host_cpu.name) - else: - parts = (host_cpu.family.name, host_cpu.name) - - software_subdir = os.path.join(*parts) - -print(software_subdir) From a1b3a87949e1205cda714046301b23aee06958ac Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Thu, 29 Feb 2024 17:57:28 +0100 Subject: [PATCH 2/9] Make archspec usage compatible with 0.2.3 --- .github/workflows/tests_init.yml | 2 +- init/eessi_software_subdir_for_host.py | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml index 053acb9730..38ccbbad31 100644 --- a/.github/workflows/tests_init.yml +++ b/.github/workflows/tests_init.yml @@ -22,7 +22,7 @@ jobs: - name: install Python packages run: | - pip install archspec==0.2.2 pytest + pip install archspec pytest - name: unit tests for eessi_software_subdir_for_host.py script run: diff --git a/init/eessi_software_subdir_for_host.py b/init/eessi_software_subdir_for_host.py index 58e9cfd2e6..49de23b262 100755 --- a/init/eessi_software_subdir_for_host.py +++ b/init/eessi_software_subdir_for_host.py @@ -6,8 +6,22 @@ import os import platform import sys -import archspec.cpu -from archspec.cpu.detect import compatible_microarchitectures, raw_info_dictionary + +import archspec + +from archspec.cpu.detect import compatible_microarchitectures +if hasattr(archspec, '__version__'): + module_version = archspec.__version__ + legacy_version = '0.2.2' + + if module_version >= legacy_version: + from archspec.cpu.detect import detected_info as raw_info + else: + # Handle the case where the module version is not compatible + from archspec.cpu.detect import raw_info_dictionary as raw_info + + + VENDOR_MAP = { 'GenuineIntel': 'intel', @@ -39,7 +53,7 @@ def sorting_fn(item): """Helper function to sort compatible microarchitectures.""" return len(item.ancestors), len(item.features) - raw_cpu_info = raw_info_dictionary() + raw_cpu_info = raw_info() compat_targets = compatible_microarchitectures(raw_cpu_info) # filter out generic targets From 310a84b8efed633c4fed7a38c0360649926ea0c5 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Fri, 1 Mar 2024 16:13:23 +0100 Subject: [PATCH 3/9] Do basic version checking and add a test --- .github/workflows/tests_init.yml | 25 +++++++++++++++++++++++-- init/eessi_software_subdir_for_host.py | 3 ++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml index 38ccbbad31..deb36a1235 100644 --- a/.github/workflows/tests_init.yml +++ b/.github/workflows/tests_init.yml @@ -20,9 +20,30 @@ jobs: python-version: ${{matrix.python}} architecture: x64 - - name: install Python packages + - name: install Python packages with legacy archspec run: | - pip install archspec pytest + pip install archspec==0.2.2 pytest + + - name: unit tests for eessi_software_subdir_for_host.py script + run: + pytest -v init/test.py + + - name: test eessi_software_subdir_for_host.py script + env: + EESSI_X86_64: /tmp/EESSI/software/linux/x86_64 + run: | + mkdir -p ${EESSI_X86_64}/intel/{pentium4,sandybridge,haswell,skylake_avx512} ${EESSI_X86_64}/generic + python3 ./init/eessi_software_subdir_for_host.py /tmp/EESSI > out.txt + grep '^x86_64/' out.txt + + # check override via $EESSI_SOFTWARE_SUBDIR_OVERRIDE + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=x86_64/intel/pentium4 + python3 ./init/eessi_software_subdir_for_host.py /tmp/EESSI > out.txt + grep '^x86_64/intel/pentium4$' out.txt + + - name: upgrade Python package archspec + run: | + pip install --upgrade archspec - name: unit tests for eessi_software_subdir_for_host.py script run: diff --git a/init/eessi_software_subdir_for_host.py b/init/eessi_software_subdir_for_host.py index 49de23b262..94eab859bc 100755 --- a/init/eessi_software_subdir_for_host.py +++ b/init/eessi_software_subdir_for_host.py @@ -14,7 +14,8 @@ module_version = archspec.__version__ legacy_version = '0.2.2' - if module_version >= legacy_version: + # Let's assume well-behaved semantic versioning for archspec + if tuple(map(int, (module_version.split(".")))) > tuple(map(int, (legacy_version.split(".")))): from archspec.cpu.detect import detected_info as raw_info else: # Handle the case where the module version is not compatible From 1b3bd349dbb76287e955b7ef739c45180d4634ba Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 30 Jul 2024 09:38:53 +0200 Subject: [PATCH 4/9] Remove unnecessary tests --- .github/workflows/tests.yml | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 75340565c1..0000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,34 +0,0 @@ -# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Tests -on: - push: - branches: [ "*-software.eessi.io" ] - pull_request: -permissions: - contents: read # to fetch code (actions/checkout) -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python: [3.7, 3.8, 3.9, '3.10'] - fail-fast: false - steps: - - name: checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: set up Python - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - with: - python-version: ${{matrix.python}} - architecture: x64 - - - name: install Python packages - run: | - pip install archspec - - - name: test eessi_software_subdir.py script - run: | - ./eessi_software_subdir.py | tee out - grep '^x86_64/' out - egrep '/intel/|/amd/' out From 63354b0a9cde53d60163198aed49081695c62f6f Mon Sep 17 00:00:00 2001 From: ocaisa Date: Tue, 30 Jul 2024 12:52:11 +0200 Subject: [PATCH 5/9] Update EESSI-install-software.sh --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index f8a704a798..1f17114938 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -148,7 +148,7 @@ if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then # Last path is the generic case override_subdir="${possible_subdir_paths##*:}" else - # First path is the best option (according to archspec) + # First path is the best option (according to archdetect) override_subdir="${possible_subdir_paths%%:*}" fi export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir" From d58eadb927649ccbae3ae578b3e69135ab6a9c55 Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Tue, 30 Jul 2024 13:04:50 +0200 Subject: [PATCH 6/9] Replace all usages of archspec-related script --- EESSI-remove-software.sh | 15 ++++++++++++--- test_suite.sh | 13 +++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 446a156cb8..552e3c2e0a 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -17,7 +17,7 @@ POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do case $1 in -g|--generic) - DETECTION_PARAMETERS="--generic" + GENERIC=1 shift ;; -h|--help) @@ -46,8 +46,17 @@ source $TOPDIR/scripts/utils.sh echo ">> Determining software subdirectory to use for current build host..." if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) - echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" + possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath) + + if [[ "$GENERIC" -eq 1 ]]; then + # Last path is the generic case + override_subdir="${possible_subdir_paths##*:}" + else + # First path is the best option (according to archspec) + override_subdir="${possible_subdir_paths%%:*}" + fi + export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir" + echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'archdetect' script ($override_subdir from $possible_subdir_paths)" else echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" fi diff --git a/test_suite.sh b/test_suite.sh index 6e73fbd87c..69abe4698d 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -23,7 +23,7 @@ POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do case $1 in -g|--generic) - DETECTION_PARAMETERS="--generic" + GENERIC=1 shift ;; -h|--help) @@ -75,7 +75,16 @@ TMPDIR=$(mktemp -d) echo ">> Setting up environment..." module --force purge -export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) +possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath) + +if [[ "$GENERIC" -eq 1 ]]; then + # Last path is the generic case + override_subdir="${possible_subdir_paths##*:}" +else + # First path is the best option (according to archspec) + override_subdir="${possible_subdir_paths%%:*}" +fi +export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir" source $TOPDIR/init/bash From 785cefc84d4a3d0761ba1bf34499408c994da68e Mon Sep 17 00:00:00 2001 From: Alan O'Cais Date: Mon, 5 Aug 2024 12:41:25 +0200 Subject: [PATCH 7/9] Verify we get a generic directory when it is requested --- EESSI-install-software.sh | 3 +++ EESSI-remove-software.sh | 3 +++ test_suite.sh | 3 +++ 3 files changed, 9 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 1f17114938..0509505834 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -147,6 +147,9 @@ if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then if [[ "$GENERIC" -eq 1 ]]; then # Last path is the generic case override_subdir="${possible_subdir_paths##*:}" + if ! echo "$override_subdir" | grep -q "generic" ; then + fatal_error "Generic build requested but not found in selected software directory '$override_subdir'" + fi else # First path is the best option (according to archdetect) override_subdir="${possible_subdir_paths%%:*}" diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 552e3c2e0a..53d6e27569 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -51,6 +51,9 @@ if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then if [[ "$GENERIC" -eq 1 ]]; then # Last path is the generic case override_subdir="${possible_subdir_paths##*:}" + if ! echo "$override_subdir" | grep -q "generic" ; then + fatal_error "Generic build requested but not found in selected software directory '$override_subdir'" + fi else # First path is the best option (according to archspec) override_subdir="${possible_subdir_paths%%:*}" diff --git a/test_suite.sh b/test_suite.sh index 69abe4698d..ff6a606830 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -80,6 +80,9 @@ possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath) if [[ "$GENERIC" -eq 1 ]]; then # Last path is the generic case override_subdir="${possible_subdir_paths##*:}" + if ! echo "$override_subdir" | grep -q "generic" ; then + fatal_error "Generic build requested but not found in selected software directory '$override_subdir'" + fi else # First path is the best option (according to archspec) override_subdir="${possible_subdir_paths%%:*}" From 10b10c30f3adb1614bd65e4e18215b1b8e4d3226 Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 9 Aug 2024 14:46:20 +0200 Subject: [PATCH 8/9] Update EESSI-install-software.sh --- EESSI-install-software.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 0509505834..ac493cf83d 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -148,7 +148,7 @@ if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then # Last path is the generic case override_subdir="${possible_subdir_paths##*:}" if ! echo "$override_subdir" | grep -q "generic" ; then - fatal_error "Generic build requested but not found in selected software directory '$override_subdir'" + fatal_error "Generic build requested but not found in selected software directory $override_subdir" fi else # First path is the best option (according to archdetect) From 90c0b52b46198550eac3aff5bbf2e43ca181697b Mon Sep 17 00:00:00 2001 From: ocaisa Date: Fri, 9 Aug 2024 14:47:39 +0200 Subject: [PATCH 9/9] Apply suggestions from code review --- EESSI-remove-software.sh | 2 +- test_suite.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 53d6e27569..69fa9594f3 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -52,7 +52,7 @@ if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then # Last path is the generic case override_subdir="${possible_subdir_paths##*:}" if ! echo "$override_subdir" | grep -q "generic" ; then - fatal_error "Generic build requested but not found in selected software directory '$override_subdir'" + fatal_error "Generic build requested but not found in selected software directory $override_subdir" fi else # First path is the best option (according to archspec) diff --git a/test_suite.sh b/test_suite.sh index ff6a606830..3cece29499 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -81,7 +81,7 @@ if [[ "$GENERIC" -eq 1 ]]; then # Last path is the generic case override_subdir="${possible_subdir_paths##*:}" if ! echo "$override_subdir" | grep -q "generic" ; then - fatal_error "Generic build requested but not found in selected software directory '$override_subdir'" + fatal_error "Generic build requested but not found in selected software directory $override_subdir" fi else # First path is the best option (according to archspec)