test installing of EasyBuild v4.8.2 to software.eessi.io/versions/2023.06

.github/workflows/test_eessi.yml → .github/workflows/test-software.eessi.io.yml

@@ -1,11 +1,11 @@
 # documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
-name: Tests relying on having EESSI pilot repo mounted
+name: Check for missing software installations in software.eessi.io
 on: [push, pull_request, workflow_dispatch]
 permissions:
   contents: read # to fetch code (actions/checkout)
 jobs:
   pilot:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -29,19 +29,19 @@ jobs:
           with:
               cvmfs_config_package: https://github.com/EESSI/filesystem-layer/releases/download/latest/cvmfs-config-eessi_latest_all.deb
               cvmfs_http_proxy: DIRECT
-              cvmfs_repositories: pilot.eessi-hpc.org
+              cvmfs_repositories: software.eessi.io
 
         - name: Test check_missing_installations.sh script
           run: |
-              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
               eb --version
-              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort
-              echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)"
-              for easystack_file in $(ls eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do
+              echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml)"
+              for easystack_file in $(ls easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-eb-*.yml); do
                   echo "check missing installations for ${easystack_file}..."
                   ./check_missing_installations.sh ${easystack_file}
                   ec=$?
@@ -50,10 +50,10 @@ jobs:
 
         - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
           run: |
-              source /cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}/init/bash
+              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
               module load EasyBuild
               eb --version
-              export EESSI_PREFIX=/cvmfs/pilot.eessi-hpc.org/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
               export EESSI_OS_TYPE=linux
               export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}}
               env | grep ^EESSI | sort

.github/workflows/test_eessi_container_script.yml

@@ -45,7 +45,7 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_default' ]]; then
                 outfile=out_listrepos.txt
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
-                grep "EESSI-pilot" ${outfile}
+                grep "EESSI" ${outfile}
 
               # test use of --list-repos with custom repos.cfg
               elif [[ ${{matrix.SCRIPT_TEST}} == 'listrepos_custom' ]]; then
@@ -57,7 +57,7 @@ jobs:
                 echo "[EESSI/20HT.TP]" >> cfg/repos.cfg
                 echo "repo_version = 20HT.TP" >> cfg/repos.cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile}
-                grep "EESSI-pilot" ${outfile}
+                grep "EESSI" ${outfile}
 
                 export EESSI_REPOS_CFG_DIR_OVERRIDE=${PWD}/cfg
                 ./eessi_container.sh --verbose --list-repos | tee ${outfile2}
@@ -90,15 +90,15 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'readwrite' ]]; then
                 outfile=out_readwrite.txt
                 fn="test_${RANDOM}.txt"
-                echo "touch /cvmfs/pilot.eessi-hpc.org/${fn}" > test_script.sh
+                echo "touch /cvmfs/software.eessi.io/${fn}" > test_script.sh
                 chmod u+x test_script.sh
                 export SINGULARITY_BIND="$PWD:/test"
                 ./eessi_container.sh --verbose --access rw --mode run /test/test_script.sh > ${outfile}
 
                 tmpdir=$(grep "\-\-resume" ${outfile} | sed "s/.*--resume \([^']*\).*/\1/g")
                 # note: must use '--access rw' again here, since touched file is in overlay upper dir
-                ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
-                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
+                ./eessi_container.sh --verbose --resume ${tmpdir} --access rw --mode shell <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile}
+                grep "/cvmfs/software.eessi.io/${fn}$" $outfile
 
               # test use of --resume
               elif [[ ${{matrix.SCRIPT_TEST}} == 'resume' ]]; then
@@ -120,12 +120,12 @@ jobs:
               elif [[ ${{matrix.SCRIPT_TEST}} == 'save' ]]; then
                 outfile=out_save.txt
                 fn="test_${RANDOM}.txt"
-                test_cmd="touch /cvmfs/pilot.eessi-hpc.org/${fn}"
+                test_cmd="touch /cvmfs/software.eessi.io/${fn}"
                 ./eessi_container.sh --verbose --mode shell --access rw --save test-save.tar <<< "${test_cmd}" 2>&1 | tee ${outfile}
                 rm -f ${outfile}
 
-                ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/pilot.eessi-hpc.org/${fn}" > ${outfile}
-                grep "/cvmfs/pilot.eessi-hpc.org/${fn}$" $outfile
+                ./eessi_container.sh --verbose --mode shell --access rw --resume test-save.tar <<< "ls -l /cvmfs/software.eessi.io/${fn}" > ${outfile}
+                grep "/cvmfs/software.eessi.io/${fn}$" $outfile
 
                 tar tfv test-save.tar | grep "overlay-upper/${fn}"
 

.github/workflows/tests_scripts.yml

@@ -5,7 +5,7 @@ on:
     paths:
       - build_container.sh
       - create_directory_tarballs.sh
-      - EESSI-pilot-install-software.sh
+      - EESSI-install-software.sh
       - install_software_layer.sh
       - load_easybuild_module.sh
       - run_in_compat_layer_env.sh
@@ -16,7 +16,7 @@ on:
     paths:
       - build_container.sh
       - create_directory_tarballs.sh
-      - EESSI-pilot-install-software.sh
+      - EESSI-install-software.sh
       - install_software_layer.sh
       - load_easybuild_module.sh
       - run_in_compat_layer_env.sh
@@ -40,7 +40,7 @@ jobs:
         # bind current directory into container as /software-layer
         export SINGULARITY_BIND="${PWD}:/software-layer"
 
-        # can't test with EasyBuild versions older than v4.5.2 when using EESSI pilot 2023.06,
+        # can't test with EasyBuild versions older than v4.5.2 when using EESSI 2023.06,
         # since Python in compat layer is Python 3.11.x;
         # testing with a single EasyBuild version takes a while in GitHub Actions, so stick to a single sensible version
         for EB_VERSION in '4.6.0'; do
@@ -84,7 +84,7 @@ jobs:
           cp -a * /tmp/
           cd /tmp
           # force using x86_64/generic, to avoid triggering an installation from scratch
-          sed -i "s@./EESSI-pilot-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-pilot-install-software.sh\"@g" install_software_layer.sh
+          sed -i "s@./EESSI-install-software.sh@\"export EESSI_SOFTWARE_SUBDIR_OVERRIDE='x86_64/generic'; ./EESSI-install-software.sh\"@g" install_software_layer.sh
           ./build_container.sh run /tmp/$USER/EESSI /tmp/install_software_layer.sh
 
     - name: test create_directory_tarballs.sh script

EESSI-pilot-install-software.sh → EESSI-install-software.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Script to install EESSI pilot software stack (version set through init/eessi_defaults)
+# Script to install EESSI software stack (version set through init/eessi_defaults)
 
 # see example parsing of command line arguments at
 #   https://wiki.bash-hackers.org/scripting/posparams#using_a_while_loop
@@ -172,8 +172,6 @@ if [ ! -z "${shared_fs_path}" ]; then
     export EASYBUILD_SOURCEPATH=${shared_eb_sourcepath}:${EASYBUILD_SOURCEPATH}
 fi
 
-${EB} --show-config
-
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
 module --force purge
@@ -200,6 +198,8 @@ for easystack_file in $(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[
     # load EasyBuild module (will be installed if it's not available yet)
     source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
 
+    ${EB} --show-config
+
     echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..."
 
     if [ -f ${easystack_file} ]; then

bot/build.sh

@@ -128,12 +128,12 @@ EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
 echo "bot/build.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'"
 
-# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE}
-# here, just set & export EESSI_PILOT_VERSION_OVERRIDE
+# determine EESSI version to be used from .repository.repo_version in ${JOB_CFG_FILE}
+# here, just set & export EESSI_VERSION_OVERRIDE
 # next script (eessi_container.sh) makes use of it via sourcing init scripts
 # (e.g., init/eessi_defaults or init/minimal_eessi_env)
-export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
-echo "bot/build.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'"
+export EESSI_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
+echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'"
 
 # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
@@ -210,9 +210,9 @@ BUILD_TMPDIR=$(grep ' as tmp directory ' ${build_outerr} | cut -d ' ' -f 2)
 TARBALL_STEP_ARGS+=("--resume" "${BUILD_TMPDIR}")
 
 timestamp=$(date +%s)
-# to set EESSI_PILOT_VERSION we need to source init/eessi_defaults now
+# to set EESSI_VERSION we need to source init/eessi_defaults now
 source init/eessi_defaults
-export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp})
+export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp})
 
 # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be
 # synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh
@@ -221,8 +221,8 @@ export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_PILOT_VERSION} $
 TMP_IN_CONTAINER=/tmp
 echo "Executing command to create tarball:"
 echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TARBALL_STEP_ARGS[@]}"
-echo "                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
+echo "                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}"
 ./eessi_container.sh "${COMMON_ARGS[@]}" "${TARBALL_STEP_ARGS[@]}" \
-                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_PILOT_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
+                     -- ./create_tarball.sh ${TMP_IN_CONTAINER} ${EESSI_VERSION} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} /eessi_bot_job/${TGZ} 2>&1 | tee -a ${tar_outerr}
 
 exit 0

bot/inspect.sh

@@ -195,12 +195,12 @@ EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
 export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
 echo "bot/inspect.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'"
 
-# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE}
-# here, just set & export EESSI_PILOT_VERSION_OVERRIDE
+# determine EESSI version to be used from .repository.repo_version in ${JOB_CFG_FILE}
+# here, just set & export EESSI_VERSION_OVERRIDE
 # next script (eessi_container.sh) makes use of it via sourcing init scripts
 # (e.g., init/eessi_defaults or init/minimal_eessi_env)
-export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
-echo "bot/inspect.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'"
+export EESSI_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
+echo "bot/inspect.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'"
 
 # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
@@ -260,11 +260,11 @@ base_dir=$(dirname $(realpath $0))
 # TODO better use script from tarball???
 source ${base_dir}/../init/eessi_defaults
 
-if [ -z $EESSI_PILOT_VERSION ]; then
-    echo "ERROR: \$EESSI_PILOT_VERSION must be set!" >&2
+if [ -z $EESSI_VERSION ]; then
+    echo "ERROR: \$EESSI_VERSION must be set!" >&2
     exit 1
 fi
-EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_PILOT_VERSION}/compat/linux/$(uname -m)"
+EESSI_COMPAT_LAYER_DIR="${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/compat/linux/$(uname -m)"
 
 # NOTE The below requires access to the CVMFS repository. We could make a first
 # test run with a container. For now we skip the test.
@@ -295,8 +295,8 @@ fi
 if [ ! -z ${EESSI_CVMFS_REPO_OVERRIDE} ]; then
     echo "export EESSI_CVMFS_REPO_OVERRIDE=${EESSI_CVMFS_REPO_OVERRIDE}" >> ${RESUME_SCRIPT}
 fi
-if [ ! -z ${EESSI_PILOT_VERSION_OVERRIDE} ]; then
-    echo "export EESSI_PILOT_VERSION_OVERRIDE=${EESSI_PILOT_VERSION_OVERRIDE}" >> ${RESUME_SCRIPT}
+if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then
+    echo "export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}" >> ${RESUME_SCRIPT}
 fi
 if [ ! -z ${http_proxy} ]; then
     echo "export http_proxy=${http_proxy}" >> ${RESUME_SCRIPT}
@@ -428,7 +428,7 @@ echo "Executing command to start interactive session to inspect build job:"
 # TODO possibly add information on how to init session after the prefix is
 # entered, initialization consists of
 # - environment variable settings (see 'run_in_compat_layer_env.sh')
-# - setup steps run in 'EESSI-pilot-install-software.sh'
+# - setup steps run in 'EESSI-install-software.sh'
 # These initializations are combined into a single script that is executed when
 # the shell in startprefix is started. We set the env variable BASH_ENV here.
 if [[ -z ${run_in_prefix} ]]; then

check_missing_installations.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Script to check for missing installations in EESSI pilot software stack (version 2023.06)
+# Script to check for missing installations in EESSI software stack
 #
 # author: Kenneth Hoste (@boegel)
 # author: Thomas Roeblitz (@trz42)

create_tarball.sh

@@ -5,11 +5,11 @@ set -e
 base_dir=$(dirname $(realpath $0))
 
 if [ $# -ne 4 ]; then
-    echo "ERROR: Usage: $0 <EESSI tmp dir (example: /tmp/$USER/EESSI)> <pilot version (example: 2021.03)> <CPU arch subdir (example: x86_64/amd/zen2)> <path to tarball>" >&2
+    echo "ERROR: Usage: $0 <EESSI tmp dir (example: /tmp/$USER/EESSI)> <version (example: 2023.06)> <CPU arch subdir (example: x86_64/amd/zen2)> <path to tarball>" >&2
     exit 1
 fi
 eessi_tmpdir=$1
-pilot_version=$2
+eessi_version=$2
 cpu_arch_subdir=$3
 target_tgz=$4
 
@@ -20,15 +20,15 @@ os="linux"
 source ${base_dir}/init/eessi_defaults
 cvmfs_repo=${EESSI_CVMFS_REPO}
 
-software_dir="${cvmfs_repo}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}"
+software_dir="${cvmfs_repo}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}"
 if [ ! -d ${software_dir} ]; then
     echo "Software directory ${software_dir} does not exist?!" >&2
     exit 2
 fi
 
 overlay_upper_dir="${eessi_tmpdir}/overlay-upper"
 
-software_dir_overlay="${overlay_upper_dir}/versions/${pilot_version}/software/${os}/${cpu_arch_subdir}"
+software_dir_overlay="${overlay_upper_dir}/versions/${eessi_version}/software/${os}/${cpu_arch_subdir}"
 if [ ! -d ${software_dir_overlay} ]; then
     echo "Software directory overlay ${software_dir_overlay} does not exist?!" >&2
     exit 3
@@ -40,22 +40,22 @@ echo ">> Collecting list of files/directories to include in tarball via ${PWD}..
 files_list=${tmpdir}/files.list.txt
 module_files_list=${tmpdir}/module_files.list.txt
 
-if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
+if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod ]; then
     # include Lmod cache and configuration file (lmodrc.lua),
     # skip whiteout files and backup copies of Lmod cache (spiderT.old.*)
-    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list}
+    find ${eessi_version}/software/${os}/${cpu_arch_subdir}/.lmod -type f | egrep -v '/\.wh\.|spiderT.old' > ${files_list}
 fi
-if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
+if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules ]; then
     # module files
-    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list}
+    find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type f | grep -v '/\.wh\.' >> ${files_list}
     # module symlinks
-    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
+    find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules -type l | grep -v '/\.wh\.' >> ${files_list}
     # module files and symlinks
-    find ${pilot_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \
+    find ${eessi_version}/software/${os}/${cpu_arch_subdir}/modules/all -type f -o -type l \
         | grep -v '/\.wh\.' | grep -v '/\.modulerc\.lua' | sed -e 's/.lua$//' | sed -e 's@.*/modules/all/@@g' | sort -u \
         >> ${module_files_list}
 fi
-if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then
+if [ -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${module_files_list} ]; then
     # installation directories but only those for which module files were created
     # Note, we assume that module names (as defined by 'PACKAGE_NAME/VERSION.lua'
     # using EasyBuild's standard module naming scheme) match the name of the
@@ -64,7 +64,7 @@ if [ -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software -a -r ${modu
     # installation directories), the procedure will likely not work.
     for package_version in $(cat ${module_files_list}); do
         echo "handling ${package_version}"
-        ls -d ${pilot_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \
+        ls -d ${eessi_version}/software/${os}/${cpu_arch_subdir}/software/${package_version} \
             | grep -v '/\.wh\.' >> ${files_list}
     done
 fi

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.8.2-system.yml

@@ -0,0 +1,4 @@
+easyconfigs:
+  - EasyBuild-4.8.2.eb:
+      options:
+        from-pr: 19105