Merge remote-tracking branch 'upstream/2023.06-software.eessi.io' int…

…o fix_archspec_usage_for_0.2.3

.github/workflows/scorecards.yml

@@ -4,24 +4,20 @@
 
 name: Scorecards supply-chain security
 on:
-  # For Branch-Protection check. Only the default branch is supported. See
-  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
-  branch_protection_rule:
   # To guarantee Maintained check is occasionally updated. See
   # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
   schedule:
     - cron: '25 15 * * 3'
   push:
-    branches: [ "main" ]
-  pull_request:
-    branches:    
-      - main
+    branches:
+      - '*-software.eessi.io'
 
 # Declare default permissions as read only.
 permissions: read-all
 
 jobs:
   analysis:
+    if: github.repository_owner == 'EESSI'   # Prevent running on forks
     name: Scorecards analysis
     runs-on: ubuntu-latest
     permissions:
@@ -35,12 +31,12 @@ jobs:
 
     steps:
       - name: "Checkout code"
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@99c53751e09b9529366343771cc321ec74e9bd3d # v2.0.6
+        uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # v2.3.3
         with:
           results_file: results.sarif
           results_format: sarif
@@ -62,14 +58,14 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8 # v3.1.0
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         with:
           name: SARIF file
           path: results.sarif
           retention-days: 5
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@807578363a7869ca324a79039e6db9c843e0e100 # v2.1.27
+        uses: github/codeql-action/upload-sarif@9fdb3e49720b44c48891d036bb502feb25684276 # v3.25.6
         with:
           sarif_file: results.sarif

.github/workflows/test-software.eessi.io.yml

@@ -1,11 +1,15 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Check for missing software installations in software.eessi.io
-on: [push, pull_request, workflow_dispatch]
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
+  workflow_dispatch:
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   check_missing:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:

.github/workflows/test_eessi_container_script.yml

@@ -1,11 +1,15 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests for eessi_container.sh script
-on: [push, pull_request, workflow_dispatch]
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
+  workflow_dispatch:
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   eessi_container_script:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
@@ -45,7 +49,8 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
                 outfile=out_listrepos.txt
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
-                grep "EESSI" ${outfile}
+                # make sure that the default EESSI software repository is available
+                grep "software.eessi.io" ${outfile}
 
               # test use of --list-repos with custom repos.cfg
               elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
@@ -57,11 +62,12 @@ jobs:
                 echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
                 echo "repo_version = 20HT.TP" >> cfg/repos.cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
-                grep "EESSI" ${outfile}
+                # make sure that the default EESSI software repository is available
+                grep "software.eessi.io" ${outfile}
 
                 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile2}
-                grep "[EESSI/2023.02]" ${outfile2}
+                grep "EESSI/20AB.CD" ${outfile2}
 
               # test use of --mode run
               elif [[ ${{matrix.SCRIPT_TEST}} == 'run' ]]; then

.github/workflows/test_licenses.yml

@@ -1,11 +1,14 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Test software licenses
-on: [push, pull_request]
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

.github/workflows/tests.yml

@@ -0,0 +1,34 @@
+# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
+name: Tests
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
+permissions:
+  contents: read # to fetch code (actions/checkout)
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python: [3.7, 3.8, 3.9, '3.10']
+      fail-fast: false
+    steps:
+    - name: checkout
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+    - name: set up Python
+      uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5.0.0
+      with:
+        python-version: ${{matrix.python}}
+        architecture: x64
+
+    - name: install Python packages
+      run: |
+        pip install archspec
+
+    - name: test eessi_software_subdir.py script
+      run: |
+          ./eessi_software_subdir.py | tee out
+          grep '^x86_64/' out
+          egrep '/intel/|/amd/' out

.github/workflows/tests_archdetect.yml

@@ -1,18 +1,23 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests for eessi_archdetect.sh
-on: [push, pull_request]
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
         proc_cpuinfo:
           - x86_64/intel/haswell/archspec-linux-E5-2680-v3
           - x86_64/intel/skylake_avx512/archspec-linux-6132
           - x86_64/amd/zen2/Azure-CentOS7-7V12
           - x86_64/amd/zen3/Azure-CentOS7-7V73X
+          - x86_64/amd/zen4/Azure-Alma8-9V33X
+          - x86_64/amd/zen4/Shinx-RHEL8-9654
           - aarch64/neoverse_n1/Azure-Ubuntu20-Altra
           - aarch64/neoverse_n1/AWS-awslinux-graviton2
           - aarch64/neoverse_v1/AWS-awslinux-graviton3

.github/workflows/tests_init.yml

@@ -1,14 +1,17 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
 name: Tests for init scripts
-on: [push, pull_request]
+on:
+  push:
+    branches: [ "*-software.eessi.io" ]
+  pull_request:
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: [3.6, 3.7, 3.8, 3.9, '3.10']
+        python: [3.7, 3.8, 3.9, '3.10']
       fail-fast: false
     steps:
     - name: checkout

.github/workflows/tests_readme.yml

@@ -14,7 +14,7 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

.github/workflows/tests_scripts.yml

@@ -3,8 +3,9 @@ name: Tests for scripts
 on:
   push:
     paths:
-      - build_container.sh
       - create_directory_tarballs.sh
+      - create_lmodsitepackage.py
+      - eessi_container.sh
       - EESSI-install-software.sh
       - install_software_layer.sh
       - load_easybuild_module.sh
@@ -14,8 +15,9 @@ on:
 
   pull_request:
     paths:
-      - build_container.sh
       - create_directory_tarballs.sh
+      - create_lmodsitepackage.py
+      - eessi_container.sh
       - EESSI-install-software.sh
       - install_software_layer.sh
       - load_easybuild_module.sh
@@ -26,7 +28,7 @@ permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
     - name: checkout
       uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@@ -83,6 +85,8 @@ jobs:
           export SINGULARITY_BIND="${PWD}:/software-layer"
           # force using x86_64/generic, to avoid triggering an installation from scratch
           sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh
+          # skip installation of CUDA SDKs, since this is too heavy for CI
+          sed -i "s@./EESSI-install-software.sh@./EESSI-install-software.sh --skip-cuda-install@g" install_software_layer.sh
           ./eessi_container.sh --mode run --verbose /software-layer/install_software_layer.sh
 
     - name: test create_directory_tarballs.sh script
@@ -94,3 +98,24 @@ jobs:
           ./eessi_container.sh --mode run --verbose /software-layer/create_directory_tarballs.sh 2023.06
           # check if tarballs have been produced
           ls -l *.tar.gz
+
+    - name: test create_lmodsitepackage.py script
+      run: |
+          # bind current directory into container as /software-layer
+          export SINGULARITY_BIND="${PWD}:/software-layer"
+
+          # Creates .lmod/SitePackage.lua in current dir, which then gets bind-mounted into /software-layer
+          python3 create_lmodsitepackage.py .
+          # run some commands to make sure that generated Lmod SitePackage file works
+          test_script="${PWD}/test_lmod_sitepackage.sh"
+          echo '#!/bin/bash' > ${test_script}
+          echo 'export LMOD_PACKAGE_PATH="/software-layer/.lmod"' > ${test_script}
+          echo 'ml --config' >> ${test_script}
+
+          chmod u+x ${test_script}
+
+          out="${PWD}/test_create_lmodsitepackage.out"
+          ./eessi_container.sh --mode run --verbose /software-layer/run_in_compat_layer_env.sh /software-layer/test_lmod_sitepackage.sh 2>&1 | tee ${out}
+          for pattern in "^Site Pkg location.*/software-layer/.lmod/SitePackage.lua" "LMOD_SITEPACKAGE_LOCATION.*/software-layer/.lmod/SitePackage.lua"; do
+            grep "${pattern}" ${out} || (echo "Pattern '${pattern}' not found in output!" && exit 1)
+          done