Prefer archdetect in our scripts, also make EESSI compatible with archspec 0.2.3+

.github/workflows/tests_init.yml

@@ -23,7 +23,7 @@ jobs:
         python-version: ${{matrix.python}}
         architecture: x64
 
-    - name: install Python packages
+    - name: install Python packages with legacy archspec
       run: |
         pip install archspec==0.2.2 pytest
 
@@ -43,3 +43,24 @@ jobs:
           export EESSI_SOFTWARE_SUBDIR_OVERRIDE=x86_64/intel/pentium4
           python3 ./init/eessi_software_subdir_for_host.py /tmp/EESSI > out.txt
           grep '^x86_64/intel/pentium4$' out.txt
+
+    - name: upgrade Python package archspec
+      run: |
+        pip install --upgrade archspec
+
+    - name: unit tests for eessi_software_subdir_for_host.py script
+      run:
+          pytest -v init/test.py
+
+    - name: test eessi_software_subdir_for_host.py script
+      env:
+          EESSI_X86_64: /tmp/EESSI/software/linux/x86_64
+      run: |
+          mkdir -p ${EESSI_X86_64}/intel/{pentium4,sandybridge,haswell,skylake_avx512} ${EESSI_X86_64}/generic
+          python3 ./init/eessi_software_subdir_for_host.py /tmp/EESSI > out.txt
+          grep '^x86_64/' out.txt
+
+          # check override via $EESSI_SOFTWARE_SUBDIR_OVERRIDE
+          export EESSI_SOFTWARE_SUBDIR_OVERRIDE=x86_64/intel/pentium4
+          python3 ./init/eessi_software_subdir_for_host.py /tmp/EESSI > out.txt
+          grep '^x86_64/intel/pentium4$' out.txt

EESSI-install-software.sh

@@ -132,20 +132,30 @@ fi
 # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
 export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
 
-DETECTION_PARAMETERS=''
 GENERIC=0
 EB='eb'
 if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
     echo_yellow ">> GENERIC build requested, taking appropriate measures!"
-    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
     GENERIC=1
     EB='eb --optarch=GENERIC'
 fi
 
 echo ">> Determining software subdirectory to use for current build host..."
 if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
-  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
-  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+  possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath)
+
+  if [[ "$GENERIC" -eq 1 ]]; then
+      # Last path is the generic case
+      override_subdir="${possible_subdir_paths##*:}"
+      if ! echo "$override_subdir" | grep -q "generic" ; then
+          fatal_error "Generic build requested but not found in selected software directory $override_subdir"
+      fi
+  else
+      # First path is the best option (according to archdetect)
+      override_subdir="${possible_subdir_paths%%:*}"
+  fi
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir"
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'archdetect' script ($override_subdir from $possible_subdir_paths)"
 else
   echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
   # make sure directory exists (since it's expected by init/eessi_environment_variables when using archdetect)

EESSI-remove-software.sh

@@ -17,7 +17,7 @@ POSITIONAL_ARGS=()
 while [[ $# -gt 0 ]]; do
   case $1 in
     -g|--generic)
-      DETECTION_PARAMETERS="--generic"
+      GENERIC=1
       shift
       ;;
     -h|--help)
@@ -46,8 +46,20 @@ source $TOPDIR/scripts/utils.sh
 
 echo ">> Determining software subdirectory to use for current build host..."
 if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
-  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
-  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+  possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath)
+
+  if [[ "$GENERIC" -eq 1 ]]; then
+      # Last path is the generic case
+      override_subdir="${possible_subdir_paths##*:}"
+      if ! echo "$override_subdir" | grep -q "generic" ; then
+          fatal_error "Generic build requested but not found in selected software directory '$override_subdir'"
+      fi
+  else
+      # First path is the best option (according to archspec)
+      override_subdir="${possible_subdir_paths%%:*}"
+  fi
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir"
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'archdetect' script ($override_subdir from $possible_subdir_paths)"
 else
   echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
 fi

bot/inspect.sh

@@ -340,21 +340,28 @@ fi
 # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
 export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
 
-DETECTION_PARAMETERS=''
 GENERIC=0
 EB='eb'
 if [[ "$EASYBUILD_OPTARCH" == "GENERIC" || "$EESSI_SOFTWARE_SUBDIR_OVERRIDE" == *"/generic" ]]; then
     echo_yellow ">> GENERIC build requested, taking appropriate measures!"
-    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
     GENERIC=1
     export EASYBUILD_OPTARCH=GENERIC
     EB='eb --optarch=GENERIC'
 fi
 
 echo ">> Determining software subdirectory to use for current build host..."
 if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
-  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
-  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+  possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath)
+
+  if [[ "$GENERIC" -eq 1 ]]; then
+      # Last path is the generic case
+      override_subdir="${possible_subdir_paths##*:}"
+  else
+      # First path is the best option (according to archspec)
+      override_subdir="${possible_subdir_paths%%:*}"
+  fi
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir"
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'archdetect' script ($override_subdir from $possible_subdir_paths)"
 else
   echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
 fi

init/eessi_software_subdir_for_host.py

@@ -6,8 +6,23 @@
 import os
 import platform
 import sys
-import archspec.cpu
-from archspec.cpu.detect import compatible_microarchitectures, raw_info_dictionary
+
+import archspec
+
+from archspec.cpu.detect import compatible_microarchitectures
+if hasattr(archspec, '__version__'):
+    module_version = archspec.__version__
+    legacy_version = '0.2.2'
+
+    # Let's assume well-behaved semantic versioning for archspec
+    if tuple(map(int, (module_version.split(".")))) > tuple(map(int, (legacy_version.split(".")))):
+        from archspec.cpu.detect import detected_info as raw_info
+    else:
+        # Handle the case where the module version is not compatible
+        from archspec.cpu.detect import raw_info_dictionary as raw_info
+
+
+
 
 VENDOR_MAP = {
     'GenuineIntel': 'intel',
@@ -39,7 +54,7 @@ def sorting_fn(item):
         """Helper function to sort compatible microarchitectures."""
         return len(item.ancestors), len(item.features)
 
-    raw_cpu_info = raw_info_dictionary()
+    raw_cpu_info = raw_info()
     compat_targets = compatible_microarchitectures(raw_cpu_info)
 
     # filter out generic targets

test_suite.sh

@@ -23,7 +23,7 @@ POSITIONAL_ARGS=()
 while [[ $# -gt 0 ]]; do
   case $1 in
     -g|--generic)
-      DETECTION_PARAMETERS="--generic"
+      GENERIC=1
       shift
       ;;
     -h|--help)
@@ -75,7 +75,19 @@ TMPDIR=$(mktemp -d)
 
 echo ">> Setting up environment..."
 module --force purge
-export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+possible_subdir_paths=$(bash $TOPDIR/init/eessi_archdetect.sh -a cpupath)
+
+if [[ "$GENERIC" -eq 1 ]]; then
+    # Last path is the generic case
+    override_subdir="${possible_subdir_paths##*:}"
+    if ! echo "$override_subdir" | grep -q "generic" ; then
+        fatal_error "Generic build requested but not found in selected software directory '$override_subdir'"
+    fi
+else
+    # First path is the best option (according to archspec)
+    override_subdir="${possible_subdir_paths%%:*}"
+fi
+export EESSI_SOFTWARE_SUBDIR_OVERRIDE="$override_subdir"
 
 source $TOPDIR/init/bash