diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index 940614b6bb..0000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,34 +0,0 @@ -# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions -name: Tests -on: - push: - branches: [ "*-software.eessi.io" ] - pull_request: -permissions: - contents: read # to fetch code (actions/checkout) -jobs: - build: - runs-on: ubuntu-latest - strategy: - matrix: - python: [3.7, 3.8, 3.9, '3.10'] - fail-fast: false - steps: - - name: checkout - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: set up Python - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0 - with: - python-version: ${{matrix.python}} - architecture: x64 - - - name: install Python packages - run: | - pip install archspec==0.2.2 - - - name: test eessi_software_subdir.py script - run: | - ./eessi_software_subdir.py | tee out - grep '^x86_64/' out - egrep '/intel/|/amd/' out diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml index 8581681d63..273fb5a698 100644 --- a/.github/workflows/tests_init.yml +++ b/.github/workflows/tests_init.yml @@ -23,7 +23,7 @@ jobs: python-version: ${{matrix.python}} architecture: x64 - - name: install Python packages + - name: install Python packages with legacy archspec run: | pip install archspec==0.2.2 pytest @@ -43,3 +43,24 @@ jobs: export EESSI_SOFTWARE_SUBDIR_OVERRIDE=x86_64/intel/pentium4 python3 ./init/eessi_software_subdir_for_host.py /tmp/EESSI > out.txt grep '^x86_64/intel/pentium4$' out.txt + + - name: upgrade Python package archspec + run: | + pip install --upgrade archspec + + - name: unit tests for eessi_software_subdir_for_host.py script + run: + pytest -v init/test.py + + - name: test eessi_software_subdir_for_host.py script + env: + EESSI_X86_64: /tmp/EESSI/software/linux/x86_64 + run: | + mkdir -p ${EESSI_X86_64}/intel/{pentium4,sandybridge,haswell,skylake_avx512} ${EESSI_X86_64}/generic + python3 ./init/eessi_software_subdir_for_host.py /tmp/EESSI > out.txt + grep '^x86_64/' out.txt + + # check override via $EESSI_SOFTWARE_SUBDIR_OVERRIDE + export EESSI_SOFTWARE_SUBDIR_OVERRIDE=x86_64/intel/pentium4 + python3 ./init/eessi_software_subdir_for_host.py /tmp/EESSI > out.txt + grep '^x86_64/intel/pentium4$' out.txt diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 7d358e205a..ac493cf83d 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -132,20 +132,30 @@ fi # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory export PYTHONPYCACHEPREFIX=$TMPDIR/pycache -DETECTION_PARAMETERS='' GENERIC=0 EB='eb' if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then echo_yellow ">> GENERIC build requested, taking appropriate measures!" - DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic" GENERIC=1 EB='eb --optarch=GENERIC' fi echo ">> Determining software subdirectory to use for current build host..." if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) - echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" + possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath) + + if [[ "$GENERIC" -eq 1 ]]; then + # Last path is the generic case + override_subdir="${possible_subdir_paths##*:}" + if ! echo "$override_subdir" | grep -q "generic" ; then + fatal_error "Generic build requested but not found in selected software directory $override_subdir" + fi + else + # First path is the best option (according to archdetect) + override_subdir="${possible_subdir_paths%%:*}" + fi + export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir" + echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'archdetect' script ($override_subdir from $possible_subdir_paths)" else echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect) diff --git a/EESSI-remove-software.sh b/EESSI-remove-software.sh index 446a156cb8..69fa9594f3 100755 --- a/EESSI-remove-software.sh +++ b/EESSI-remove-software.sh @@ -17,7 +17,7 @@ POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do case $1 in -g|--generic) - DETECTION_PARAMETERS="--generic" + GENERIC=1 shift ;; -h|--help) @@ -46,8 +46,20 @@ source $TOPDIR/scripts/utils.sh echo ">> Determining software subdirectory to use for current build host..." if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) - echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" + possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath) + + if [[ "$GENERIC" -eq 1 ]]; then + # Last path is the generic case + override_subdir="${possible_subdir_paths##*:}" + if ! echo "$override_subdir" | grep -q "generic" ; then + fatal_error "Generic build requested but not found in selected software directory $override_subdir" + fi + else + # First path is the best option (according to archspec) + override_subdir="${possible_subdir_paths%%:*}" + fi + export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir" + echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'archdetect' script ($override_subdir from $possible_subdir_paths)" else echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" fi diff --git a/bot/inspect.sh b/bot/inspect.sh index 533968bffc..5e7cb172d0 100755 --- a/bot/inspect.sh +++ b/bot/inspect.sh @@ -340,12 +340,10 @@ fi # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory export PYTHONPYCACHEPREFIX=$TMPDIR/pycache -DETECTION_PARAMETERS='' GENERIC=0 EB='eb' if [[ "$EASYBUILD_OPTARCH" == "GENERIC" || "$EESSI_SOFTWARE_SUBDIR_OVERRIDE" == *"/generic" ]]; then echo_yellow ">> GENERIC build requested, taking appropriate measures!" - DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic" GENERIC=1 export EASYBUILD_OPTARCH=GENERIC EB='eb --optarch=GENERIC' @@ -353,8 +351,17 @@ fi echo ">> Determining software subdirectory to use for current build host..." if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then - export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) - echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script" + possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath) + + if [[ "$GENERIC" -eq 1 ]]; then + # Last path is the generic case + override_subdir="${possible_subdir_paths##*:}" + else + # First path is the best option (according to archspec) + override_subdir="${possible_subdir_paths%%:*}" + fi + export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir" + echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'archdetect' script ($override_subdir from $possible_subdir_paths)" else echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" fi diff --git a/eessi_software_subdir.py b/eessi_software_subdir.py deleted file mode 100755 index af9df722d9..0000000000 --- a/eessi_software_subdir.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 -# -# Determine EESSI software subdirectory to use for current build host, using archspec -# -import os -import argparse -from archspec.cpu.detect import compatible_microarchitectures, raw_info_dictionary - -software_subdir = os.getenv('EESSI_SOFTWARE_SUBDIR_OVERRIDE') -if software_subdir is None: - - parser = argparse.ArgumentParser(description='Determine EESSI software subdirectory to use for current build host.') - parser.add_argument('--generic', dest='generic', action='store_true', - default=False, help='Use generic for CPU name.') - args = parser.parse_args() - - # we can't directly use archspec.cpu.host(), because we may get back a virtual microarchitecture like x86_64_v3... - def sorting_fn(item): - """Helper function to sort compatible microarchitectures.""" - return len(item.ancestors), len(item.features) - - raw_cpu_info = raw_info_dictionary() - compat_targets = compatible_microarchitectures(raw_cpu_info) - - # filter out generic targets - non_generic_compat_targets = [t for t in compat_targets if t.vendor != "generic"] - - # Filter the candidates to be descendant of the best generic candidate - best_generic = max([t for t in compat_targets if t.vendor == "generic"], key=sorting_fn) - best_compat_targets = [t for t in non_generic_compat_targets if t > best_generic] - - if best_compat_targets: - host_cpu = max(best_compat_targets, key=sorting_fn) - else: - host_cpu = max(non_generic_compat_targets, key=sorting_fn) - - vendors = { - 'GenuineIntel': 'intel', - 'AuthenticAMD': 'amd', - } - - vendor = vendors.get(host_cpu.vendor) - - if args.generic: - parts = (host_cpu.family.name, 'generic') - elif vendor: - parts = (host_cpu.family.name, vendor, host_cpu.name) - else: - parts = (host_cpu.family.name, host_cpu.name) - - software_subdir = os.path.join(*parts) - -print(software_subdir) diff --git a/init/eessi_software_subdir_for_host.py b/init/eessi_software_subdir_for_host.py index 58e9cfd2e6..94eab859bc 100755 --- a/init/eessi_software_subdir_for_host.py +++ b/init/eessi_software_subdir_for_host.py @@ -6,8 +6,23 @@ import os import platform import sys -import archspec.cpu -from archspec.cpu.detect import compatible_microarchitectures, raw_info_dictionary + +import archspec + +from archspec.cpu.detect import compatible_microarchitectures +if hasattr(archspec, '__version__'): + module_version = archspec.__version__ + legacy_version = '0.2.2' + + # Let's assume well-behaved semantic versioning for archspec + if tuple(map(int, (module_version.split(".")))) > tuple(map(int, (legacy_version.split(".")))): + from archspec.cpu.detect import detected_info as raw_info + else: + # Handle the case where the module version is not compatible + from archspec.cpu.detect import raw_info_dictionary as raw_info + + + VENDOR_MAP = { 'GenuineIntel': 'intel', @@ -39,7 +54,7 @@ def sorting_fn(item): """Helper function to sort compatible microarchitectures.""" return len(item.ancestors), len(item.features) - raw_cpu_info = raw_info_dictionary() + raw_cpu_info = raw_info() compat_targets = compatible_microarchitectures(raw_cpu_info) # filter out generic targets diff --git a/test_suite.sh b/test_suite.sh index 6e73fbd87c..3cece29499 100755 --- a/test_suite.sh +++ b/test_suite.sh @@ -23,7 +23,7 @@ POSITIONAL_ARGS=() while [[ $# -gt 0 ]]; do case $1 in -g|--generic) - DETECTION_PARAMETERS="--generic" + GENERIC=1 shift ;; -h|--help) @@ -75,7 +75,19 @@ TMPDIR=$(mktemp -d) echo ">> Setting up environment..." module --force purge -export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS) +possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath) + +if [[ "$GENERIC" -eq 1 ]]; then + # Last path is the generic case + override_subdir="${possible_subdir_paths##*:}" + if ! echo "$override_subdir" | grep -q "generic" ; then + fatal_error "Generic build requested but not found in selected software directory $override_subdir" + fi +else + # First path is the best option (according to archspec) + override_subdir="${possible_subdir_paths%%:*}" +fi +export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir" source $TOPDIR/init/bash