From 6cc2e3ed53708bc07da7932adc0326faac9c670f Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sat, 14 Oct 2023 21:09:25 +0200
Subject: [PATCH 001/137] first version of bot/test.sh script

---
 bot/test.sh | 224 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 224 insertions(+)
 create mode 100755 bot/test.sh

diff --git a/bot/test.sh b/bot/test.sh
new file mode 100755
index 0000000000..6d217caf92
--- /dev/null
+++ b/bot/test.sh
@@ -0,0 +1,224 @@
+#!/usr/bin/env bash
+#
+# script to run tests or the test suite for the whole EESSI software layer or
+# just what has been built in a job. Intended use is that it is called
+# at the end of a (batch) job running on a compute node.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Thomas Roeblitz (@trz42)
+#
+# license: GPLv2
+#
+
+# ASSUMPTIONs:
+# + assumption for the build step (as run through bot/build.sh which is provided
+#   in this repository too)
+#  - working directory has been prepared by the bot with a checkout of a
+#    pull request (OR by some other means)
+#  - the working directory contains a directory 'cfg' where the main config
+#    file 'job.cfg' has been deposited
+#  - the directory may contain any additional files referenced in job.cfg
+# + assumptions for the test step
+#  - temporary storage is still available
+#    example
+#    Using /localscratch/9640860/NESSI/eessi.x765Dd8mFh as tmp directory (to resume session add '--resume /localscratch/9640860/NESSI/eessi.x765Dd8mFh').
+#  - run test-suite.sh inside build container using tmp storage from build step
+#    plus possibly additional settings (repo, etc.)
+#  - needed setup steps may be similar to bot/inspect.sh (PR#317)
+
+# stop as soon as something fails
+set -e
+
+# source utils.sh and cfg_files.sh
+source scripts/utils.sh
+source scripts/cfg_files.sh
+
+# defaults
+export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}"
+HOST_ARCH=$(uname -m)
+
+# check if ${JOB_CFG_FILE} exists
+if [[ ! -r "${JOB_CFG_FILE}" ]]; then
+    fatal_error "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable"
+fi
+echo "bot/test.sh: showing ${JOB_CFG_FILE} from software-layer side"
+cat ${JOB_CFG_FILE}
+
+echo "bot/test.sh: obtaining configuration settings from '${JOB_CFG_FILE}'"
+cfg_load ${JOB_CFG_FILE}
+
+# if http_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $http_proxy
+HTTP_PROXY=$(cfg_get_value "site_config" "http_proxy")
+HTTP_PROXY=${HTTP_PROXY:-${http_proxy}}
+echo "bot/test.sh: HTTP_PROXY='${HTTP_PROXY}'"
+
+# if https_proxy is defined in ${JOB_CFG_FILE} use it, if not use env var $https_proxy
+HTTPS_PROXY=$(cfg_get_value "site_config" "https_proxy")
+HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}}
+echo "bot/test.sh: HTTPS_PROXY='${HTTPS_PROXY}'"
+
+LOCAL_TMP=$(cfg_get_value "site_config" "local_tmp")
+echo "bot/test.sh: LOCAL_TMP='${LOCAL_TMP}'"
+# TODO should local_tmp be mandatory? --> then we check here and exit if it is not provided
+
+# check if path to copy build logs to is specified, so we can copy build logs for failing builds there
+BUILD_LOGS_DIR=$(cfg_get_value "site_config" "build_logs_dir")
+echo "bot/test.sh: BUILD_LOGS_DIR='${BUILD_LOGS_DIR}'"
+# if $BUILD_LOGS_DIR is set, add it to $SINGULARITY_BIND so the path is available in the build container
+if [[ ! -z ${BUILD_LOGS_DIR} ]]; then
+    mkdir -p ${BUILD_LOGS_DIR}
+    if [[ -z ${SINGULARITY_BIND} ]]; then
+        export SINGULARITY_BIND="${BUILD_LOGS_DIR}"
+    else
+        export SINGULARITY_BIND="${SINGULARITY_BIND},${BUILD_LOGS_DIR}"
+    fi
+fi
+
+# check if path to directory on shared filesystem is specified,
+# and use it as location for source tarballs used by EasyBuild if so
+SHARED_FS_PATH=$(cfg_get_value "site_config" "shared_fs_path")
+echo "bot/test.sh: SHARED_FS_PATH='${SHARED_FS_PATH}'"
+# if $SHARED_FS_PATH is set, add it to $SINGULARITY_BIND so the path is available in the build container
+if [[ ! -z ${SHARED_FS_PATH} ]]; then
+    mkdir -p ${SHARED_FS_PATH}
+    if [[ -z ${SINGULARITY_BIND} ]]; then
+        export SINGULARITY_BIND="${SHARED_FS_PATH}"
+    else
+        export SINGULARITY_BIND="${SINGULARITY_BIND},${SHARED_FS_PATH}"
+    fi
+fi
+
+SINGULARITY_CACHEDIR=$(cfg_get_value "site_config" "container_cachedir")
+echo "bot/test.sh: SINGULARITY_CACHEDIR='${SINGULARITY_CACHEDIR}'"
+if [[ ! -z ${SINGULARITY_CACHEDIR} ]]; then
+    # make sure that separate directories are used for different CPU families
+    SINGULARITY_CACHEDIR=${SINGULARITY_CACHEDIR}/${HOST_ARCH}
+    export SINGULARITY_CACHEDIR
+fi
+
+# try to determine tmp directory from build job
+RESUME_DIR=$(grep 'Using .* as tmp directory' slurm-${SLURM_JOBID}.out | head -1 | awk '{print $2}')
+
+if [[ -z ${RESUME_DIR} ]]; then
+  echo -n "setting \$STORAGE by replacing any var in '${LOCAL_TMP}' -> "
+  # replace any env variable in ${LOCAL_TMP} with its
+  #   current value (e.g., a value that is local to the job)
+  STORAGE=$(envsubst <<< ${LOCAL_TMP})
+  echo "'${STORAGE}'"
+
+  # make sure ${STORAGE} exists
+  mkdir -p ${STORAGE}
+
+  # make sure the base tmp storage is unique
+  JOB_STORAGE=$(mktemp --directory --tmpdir=${STORAGE} bot_job_tmp_XXX)
+  echo "bot/test.sh: created unique base tmp storage directory at ${JOB_STORAGE}"
+
+  RESUME_TGZ=${PWD}/previous_tmp/build_step/$(ls previous_tmp/build_step)
+  if [[ -z ${RESUME_TGZ} ]]; then
+    echo "bot/test.sh: no information about tmp directory and tarball of build step; --> giving up"
+    exit 2
+  fi
+fi
+
+# obtain list of modules to be loaded
+LOAD_MODULES=$(cfg_get_value "site_config" "load_modules")
+echo "bot/test.sh: LOAD_MODULES='${LOAD_MODULES}'"
+
+# singularity/apptainer settings: CONTAINER, HOME, TMPDIR, BIND
+CONTAINER=$(cfg_get_value "repository" "container")
+export SINGULARITY_HOME="${PWD}:/eessi_bot_job"
+export SINGULARITY_TMPDIR="${PWD}/singularity_tmpdir"
+mkdir -p ${SINGULARITY_TMPDIR}
+
+# load modules if LOAD_MODULES is not empty
+if [[ ! -z ${LOAD_MODULES} ]]; then
+    for mod in $(echo ${LOAD_MODULES} | tr ',' '\n')
+    do
+        echo "bot/test.sh: loading module '${mod}'"
+        module load ${mod}
+    done
+else
+    echo "bot/test.sh: no modules to be loaded"
+fi
+
+# determine repository to be used from entry .repository in ${JOB_CFG_FILE}
+REPOSITORY=$(cfg_get_value "repository" "repo_id")
+EESSI_REPOS_CFG_DIR_OVERRIDE=$(cfg_get_value "repository" "repos_cfg_dir")
+export EESSI_REPOS_CFG_DIR_OVERRIDE=${EESSI_REPOS_CFG_DIR_OVERRIDE:-${PWD}/cfg}
+echo "bot/test.sh: EESSI_REPOS_CFG_DIR_OVERRIDE='${EESSI_REPOS_CFG_DIR_OVERRIDE}'"
+
+# determine pilot version to be used from .repository.repo_version in ${JOB_CFG_FILE}
+# here, just set & export EESSI_PILOT_VERSION_OVERRIDE
+# next script (eessi_container.sh) makes use of it via sourcing init scripts
+# (e.g., init/eessi_defaults or init/minimal_eessi_env)
+export EESSI_PILOT_VERSION_OVERRIDE=$(cfg_get_value "repository" "repo_version")
+echo "bot/test.sh: EESSI_PILOT_VERSION_OVERRIDE='${EESSI_PILOT_VERSION_OVERRIDE}'"
+
+# determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
+# here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
+# "source init/eessi_defaults" via sourcing init/minimal_eessi_env
+export EESSI_CVMFS_REPO_OVERRIDE=$(cfg_get_value "repository" "repo_name")
+echo "bot/test.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'"
+
+# determine architecture to be used from entry .architecture in ${JOB_CFG_FILE}
+# fallbacks:
+#  - ${CPU_TARGET} handed over from bot
+#  - left empty to let downstream script(s) determine subdir to be used
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(cfg_get_value "architecture" "software_subdir")
+EESSI_SOFTWARE_SUBDIR_OVERRIDE=${EESSI_SOFTWARE_SUBDIR_OVERRIDE:-${CPU_TARGET}}
+export EESSI_SOFTWARE_SUBDIR_OVERRIDE
+echo "bot/test.sh: EESSI_SOFTWARE_SUBDIR_OVERRIDE='${EESSI_SOFTWARE_SUBDIR_OVERRIDE}'"
+
+# get EESSI_OS_TYPE from .architecture.os_type in ${JOB_CFG_FILE} (default: linux)
+EESSI_OS_TYPE=$(cfg_get_value "architecture" "os_type")
+export EESSI_OS_TYPE=${EESSI_OS_TYPE:-linux}
+echo "bot/test.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'"
+
+# prepare arguments to eessi_container.sh common to build and tarball steps
+declare -a COMMON_ARGS=()
+COMMON_ARGS+=("--verbose")
+COMMON_ARGS+=("--access" "rw")
+COMMON_ARGS+=("--mode" "run")
+[[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}")
+[[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}")
+[[ ! -z ${HTTPS_PROXY} ]] && COMMON_ARGS+=("--https-proxy" "${HTTPS_PROXY}")
+[[ ! -z ${REPOSITORY} ]] && COMMON_ARGS+=("--repository" "${REPOSITORY}")
+
+# make sure to use the same parent dir for storing tarballs of tmp
+PREVIOUS_TMP_DIR=${PWD}/previous_tmp
+
+# prepare directory to store tarball of tmp for build step
+TARBALL_TMP_TEST_STEP_DIR=${PREVIOUS_TMP_DIR}/test_step
+mkdir -p ${TARBALL_TMP_TEST_STEP_DIR}
+
+# prepare arguments to eessi_container.sh specific to test step
+declare -a TEST_STEP_ARGS=()
+TEST_STEP_ARGS+=("--save" "${TARBALL_TMP_TEST_STEP_DIR}")
+
+if [[ -z ${RESUME_DIR} ]]; then
+  TEST_STEP_ARGS+=("--storage" "${STORAGE}")
+  TEST_STEP_ARGS+=("--resume" "${RESUME_TGZ}")
+else
+  TEST_STEP_ARGS+=("--resume" "${RESUME_DIR}")
+fi
+
+# prepare arguments to test_suite.sh (specific to test step)
+declare -a TEST_SUITE_ARGS=()
+# if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
+#     TEST_SUITE_ARGS+=("--generic")
+# fi
+# [[ ! -z ${BUILD_LOGS_DIR} ]] && TEST_SUITE_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}")
+# [[ ! -z ${SHARED_FS_PATH} ]] && TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
+
+# create tmp file for output of build step
+test_outerr=$(mktemp test.outerr.XXXX)
+
+echo "Executing command to build software:"
+echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}"
+echo "                     -- ./test_suite.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}"
+./eessi_container.sh "${COMMON_ARGS[@]}" "${TEST_STEP_ARGS[@]}" \
+                     -- ./test_suite.sh "${TEST_SUITE_ARGS[@]}" "$@" 2>&1 | tee -a ${test_outerr}
+
+exit 0

From 9a32dc629b34e4bbc91fda349198611a6a96a082 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 15 Oct 2023 20:29:14 +0200
Subject: [PATCH 002/137] scripts to run tests after a build job has finished

---
 run_tests.sh  |   4 ++
 test_suite.sh | 184 ++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 188 insertions(+)
 create mode 100644 run_tests.sh
 create mode 100644 test_suite.sh

diff --git a/run_tests.sh b/run_tests.sh
new file mode 100644
index 0000000000..ccbc751f22
--- /dev/null
+++ b/run_tests.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+base_dir=$(dirname $(realpath $0))
+source ${base_dir}/init/eessi_defaults
+./run_in_compat_layer_env.sh ./test_suite.sh "$@"
diff --git a/test_suite.sh b/test_suite.sh
new file mode 100644
index 0000000000..fcaaa339d2
--- /dev/null
+++ b/test_suite.sh
@@ -0,0 +1,184 @@
+#!/bin/bash
+#
+# Run sanity check for all requested modules (and built dependencies)?
+# get ec from diff
+# set up EasyBuild
+# run `eb --sanity-only ec`
+
+display_help() {
+  echo "usage: $0 [OPTIONS]"
+  echo "  -g | --generic         -  instructs script to test for generic architecture target"
+  echo "  -h | --help            -  display this usage information"
+  echo "  -x | --http-proxy URL  -  provides URL for the environment variable http_proxy"
+  echo "  -y | --https-proxy URL -  provides URL for the environment variable https_proxy"
+}
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -g|--generic)
+      EASYBUILD_OPTARCH="GENERIC"
+      shift
+      ;;
+    -h|--help)
+      display_help  # Call your function
+      # no shifting needed here, we're done.
+      exit 0
+      ;;
+    -x|--http-proxy)
+      export http_proxy="$2"
+      shift 2
+      ;;
+    -y|--https-proxy)
+      export https_proxy="$2"
+      shift 2
+      ;;
+    --build-logs-dir)
+      export build_logs_dir="${2}"
+      shift 2
+      ;;
+    --shared-fs-path)
+      export shared_fs_path="${2}"
+      shift 2
+      ;;
+    -*|--*)
+      echo "Error: Unknown option: $1" >&2
+      exit 1
+      ;;
+    *)  # No more options
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+TOPDIR=$(dirname $(realpath $0))
+
+source $TOPDIR/scripts/utils.sh
+
+# honor $TMPDIR if it is already defined, use /tmp otherwise
+if [ -z $TMPDIR ]; then
+    export WORKDIR=/tmp/$USER
+else
+    export WORKDIR=$TMPDIR/$USER
+fi
+
+TMPDIR=$(mktemp -d)
+
+echo ">> Setting up environment..."
+
+source $TOPDIR/init/minimal_eessi_env
+
+if [ -d $EESSI_CVMFS_REPO ]; then
+    echo_green "$EESSI_CVMFS_REPO available, OK!"
+else
+    fatal_error "$EESSI_CVMFS_REPO is not available!"
+fi
+
+# make sure we're in Prefix environment by checking $SHELL
+if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then
+    echo_green ">> It looks like we're in a Gentoo Prefix environment, good!"
+else
+    fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
+fi
+
+# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
+export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
+
+DETECTION_PARAMETERS=''
+GENERIC=0
+EB='eb'
+if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
+    echo_yellow ">> GENERIC build/test requested, taking appropriate measures!"
+    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
+    GENERIC=1
+    EB='eb --optarch=GENERIC'
+fi
+
+echo ">> Determining software subdirectory to use for current build/test host..."
+if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
+  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
+else
+  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
+fi
+
+# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
+# $EESSI_SILENT - don't print any messages
+# $EESSI_BASIC_ENV - give a basic set of environment variables
+EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
+
+if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
+    fatal_error "Failed to determine software subdirectory?!"
+elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
+    fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!"
+else
+    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
+fi
+
+echo ">> Initializing Lmod..."
+source $EPREFIX/usr/share/Lmod/init/bash
+ml_version_out=$TMPDIR/ml.out
+ml --version &> $ml_version_out
+if [[ $? -eq 0 ]]; then
+    echo_green ">> Found Lmod ${LMOD_VERSION}"
+else
+    fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
+fi
+
+echo ">> Configuring EasyBuild..."
+source $TOPDIR/configure_easybuild
+
+echo ">> Setting up \$MODULEPATH..."
+# make sure no modules are loaded
+module --force purge
+# ignore current $MODULEPATH entirely
+module unuse $MODULEPATH
+module use $EASYBUILD_INSTALLPATH/modules/all
+if [[ -z ${MODULEPATH} ]]; then
+    fatal_error "Failed to set up \$MODULEPATH?!"
+else
+    echo_green ">> MODULEPATH set up: ${MODULEPATH}"
+fi
+
+# assume there's only one diff file that corresponds to the PR patch file
+pr_diff=$(ls [0-9]*.diff | head -1)
+
+# "split" the file by prefixing each line belonging to the same file with the
+# same number
+split_file=$(awk '/^\+\+\+/{n++}{print n, "  ", $0 }' ${pr_diff})
+
+# determine which easystack files may have changed
+changed_es_files=$(echo "${split_file}" | grep '^[0-9 ]*+++ ./eessi.*.yml$' | egrep -v 'known-issues|missing')
+
+# process all changed easystackfiles
+for es_file_num in $(echo "${changed_es_files}" | cut -f1 -d' ')
+do
+    # determine added lines that do not contain a yaml comment only
+    added_lines=$(echo "${split_file}" | grep "${es_file_num}    + " | sed -e "s/^"${es_file_num}"    + //" | grep -v "^[ ]*#")
+    # determine easyconfigs
+    easyconfigs=$(echo "${added_lines}" | cut -f3 -d' ')
+    # get easystack file name
+    easystack_file=$(echo "${changed_es_files}" | grep "^${es_file_num}" | sed -e "s/^"${es_file_num}"    ... .\///")
+    echo -e "Processing easystack file ${easystack_file}...\n\n"
+
+    # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
+    eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
+
+    # load EasyBuild module
+    module load EasyBuild/${eb_version}
+
+    echo_green "All set, let's run sanity checks for installed packages..."
+
+    for easyconfig in ${easyconfigs};
+    do
+        echo "Running sanity check for '${easyconfig}'..."
+        eb --sanity-check-only ${easyconfig}
+    done
+done
+
+echo ">> Cleaning up ${TMPDIR}..."
+rm -r ${TMPDIR}

From 17cfd02779833bd5f2df96655f3dc19bb5f880e8 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 15 Oct 2023 20:30:44 +0200
Subject: [PATCH 003/137] small adjustments to test script

---
 bot/test.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/bot/test.sh b/bot/test.sh
index 6d217caf92..89c1a6a8bf 100755
--- a/bot/test.sh
+++ b/bot/test.sh
@@ -206,9 +206,9 @@ fi
 
 # prepare arguments to test_suite.sh (specific to test step)
 declare -a TEST_SUITE_ARGS=()
-# if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
-#     TEST_SUITE_ARGS+=("--generic")
-# fi
+if [[ ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} =~ .*/generic$ ]]; then
+    TEST_SUITE_ARGS+=("--generic")
+fi
 # [[ ! -z ${BUILD_LOGS_DIR} ]] && TEST_SUITE_ARGS+=("--build-logs-dir" "${BUILD_LOGS_DIR}")
 # [[ ! -z ${SHARED_FS_PATH} ]] && TEST_SUITE_ARGS+=("--shared-fs-path" "${SHARED_FS_PATH}")
 
@@ -217,8 +217,8 @@ test_outerr=$(mktemp test.outerr.XXXX)
 
 echo "Executing command to build software:"
 echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}"
-echo "                     -- ./test_suite.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}"
+echo "                     -- ./run_tests.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}"
 ./eessi_container.sh "${COMMON_ARGS[@]}" "${TEST_STEP_ARGS[@]}" \
-                     -- ./test_suite.sh "${TEST_SUITE_ARGS[@]}" "$@" 2>&1 | tee -a ${test_outerr}
+                     -- ./run_tests.sh "${TEST_SUITE_ARGS[@]}" "$@" 2>&1 | tee -a ${test_outerr}
 
 exit 0

From 0f3646ef0eec6a89aa1a8a5479c8040264af5af3 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Mon, 5 Feb 2024 17:28:18 +0100
Subject: [PATCH 004/137] Second attempt at having the bot run the test suite,
 now based on Thomas' PR 366

---
 bot/check-test.sh | 66 ++++++++++++++++++++++++++++++++++
 run_tests.sh      | 10 +++++-
 test_suite.sh     | 92 +++++++++++++++++++++++++++++------------------
 3 files changed, 132 insertions(+), 36 deletions(-)
 create mode 100644 bot/check-test.sh

diff --git a/bot/check-test.sh b/bot/check-test.sh
new file mode 100644
index 0000000000..9fc783aa99
--- /dev/null
+++ b/bot/check-test.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+# Dummy script that only creates test result file for the bot, without actually checking anything
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Kenneth Hoste (HPC-UGent)
+#
+# license: GPLv2
+#
+job_dir=${PWD}
+job_out="slurm-${SLURM_JOB_ID}.out"
+job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
+
+# ReFrame prints e.g.
+#[----------] start processing checks
+#[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
+#[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default
+#[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default
+#[     FAIL ] (1/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:genoa+default
+#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_d597cff4'
+#[       OK ] (2/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
+#P: perf: 8.441 ns/day (r:0, l:None, u:None)
+#[     FAIL ] (3/3) GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=1_cpn_2_nodes %module_name=GROMACS/2021.3-foss-2021a /f4194106 @snellius:genoa+default
+#==> test failed during 'sanity': test staged in '/scratch-shared/casparl/reframe_output/staging/snellius/genoa/default/GROMACS_EESSI_f4194106'
+#[----------] all spawned checks have finished
+#[  FAILED  ] Ran 3/3 test case(s) from 2 check(s) (2 failure(s), 0 skipped, 0 aborted)
+
+# We will grep for the last and final line, since this reflects the overall result
+# Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
+FAILED=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_failed='\[\s*FAILED\s*\]'
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
+  [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+
+# Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
+# I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests
+ERROR=-1
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_error='ERROR: '
+  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
+  [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+fi
+
+echo "[TEST]" > ${job_test_result_file}
+if [[ ${ERROR} -eq 1 ]]; then
+    echo "comment_description = Failure to execute test step" >> ${job_test_result_file}
+    echo "status = FAILURE" >> ${job_test_result_file}
+elif [[ ${FAILED} -eq 1 ]]; then
+    echo "comment_description = EESSI test suite produced failures" >> ${job_test_result_file}
+else
+    echo "comment_description = Test step run succesfully" >> ${job_test_result_file}
+    echo "status = SUCCESS" >> ${job_test_result_file}
+fi
+
+exit 0
diff --git a/run_tests.sh b/run_tests.sh
index ccbc751f22..69672f18f3 100644
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -1,4 +1,12 @@
 #!/bin/bash
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
-./run_in_compat_layer_env.sh ./test_suite.sh "$@"
+
+# Note: for these tests, we _don't_ run in the compat layer env
+# These tests should mimic what users do, and they are typically not in a prefix environment
+
+# Run eb --sanity-check-only on changed easyconfigs
+# TODO: in the future we may implement this as a light first check.
+
+# Run the test suite
+./test_suite.sh "$@"
diff --git a/test_suite.sh b/test_suite.sh
index fcaaa339d2..36262a6cbf 100644
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -144,41 +144,63 @@ else
     echo_green ">> MODULEPATH set up: ${MODULEPATH}"
 fi
 
-# assume there's only one diff file that corresponds to the PR patch file
-pr_diff=$(ls [0-9]*.diff | head -1)
-
-# "split" the file by prefixing each line belonging to the same file with the
-# same number
-split_file=$(awk '/^\+\+\+/{n++}{print n, "  ", $0 }' ${pr_diff})
-
-# determine which easystack files may have changed
-changed_es_files=$(echo "${split_file}" | grep '^[0-9 ]*+++ ./eessi.*.yml$' | egrep -v 'known-issues|missing')
-
-# process all changed easystackfiles
-for es_file_num in $(echo "${changed_es_files}" | cut -f1 -d' ')
-do
-    # determine added lines that do not contain a yaml comment only
-    added_lines=$(echo "${split_file}" | grep "${es_file_num}    + " | sed -e "s/^"${es_file_num}"    + //" | grep -v "^[ ]*#")
-    # determine easyconfigs
-    easyconfigs=$(echo "${added_lines}" | cut -f3 -d' ')
-    # get easystack file name
-    easystack_file=$(echo "${changed_es_files}" | grep "^${es_file_num}" | sed -e "s/^"${es_file_num}"    ... .\///")
-    echo -e "Processing easystack file ${easystack_file}...\n\n"
-
-    # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
-    eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
-
-    # load EasyBuild module
-    module load EasyBuild/${eb_version}
-
-    echo_green "All set, let's run sanity checks for installed packages..."
-
-    for easyconfig in ${easyconfigs};
-    do
-        echo "Running sanity check for '${easyconfig}'..."
-        eb --sanity-check-only ${easyconfig}
-    done
-done
+# TODO: this should not be hardcoded. Ideally, we put some logic in place to discover the newest version
+# of the ReFrame module available in the current environment
+module load ReFrame/4.3.3
+if [[ $? -eq 0 ]]; then
+    echo_green ">> Loaded ReFrame/4.3.3"
+else
+    fatal_error "Failed to load the ReFrame module"
+fi
+
+# Check ReFrame came with the hpctestlib and we can import it
+python3 -c 'import hpctestlib.sciapps.gromacs'
+if [[ $? -eq 0 ]]; then
+    echo_green "Succesfully found and imported hpctestlib.sciapps.gromas"
+else
+    fatal_error "Failed to load hpctestlib"
+fi
+
+# Clone the EESSI test suite
+git clone https://github.com/EESSI/test-suite EESSI-test-suite
+export TESTSUITEPREFIX=$PWD/EESSI-test-suite
+export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH
+
+# Check that we can import from the testsuite
+python3 -c 'import eessi.testsuite'
+if [[ $? -eq 0 ]]; then
+    echo_green "Succesfully found and imported eessi.testsuite"
+else
+    fatal_error "FAILED to import from eessi.testsuite in Python"
+fi
+
+# Configure ReFrame
+export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/github_actions.py
+export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
+export RFM_CHECK_SEARCH_RECURSIVE=1
+export RFM_PREFIX=$PWD/reframe_runs
+
+# Check we can run reframe
+reframe --version
+if [[ $? -eq 0 ]]; then
+    echo_green "Succesfully ran reframe --version"
+else
+    fatal_error "Failed to run ReFrame --version"
+fi
+
+# List the tests we want to run
+export REFRAME_ARGS='--tag CI --tag 1_nodes'
+reframe "${REFRAME_ARGS}" --list
+if [[ $? -eq 0 ]]; then
+    echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
+else
+    fatal_error "Failed to list ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
+fi
+
+# Run all tests
+reframe "${REFRAME_ARGS}" --run
 
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}
+
+exit 0

From 34fca15420fb37eeb136da55542b44b0a474a63d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Mon, 5 Feb 2024 17:35:29 +0100
Subject: [PATCH 005/137] Make a very simple change to an easystack file so we
 are able to trigger the bot build and test procedures and see if this PR
 actually works

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 82190071ab..4c73b5887a 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -15,3 +15,4 @@ easyconfigs:
       options:
         from-pr: 19573
   - scikit-learn-1.3.1-gfbf-2023a.eb
+  - patchelf-0.18.0-GCCcore-12.3.0.eb

From 3faae9de814b96cd07896bd4042ae7b5b4b50f63 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 10:00:25 +0100
Subject: [PATCH 006/137] Make scripts executable

---
 bot/check-test.sh | 0
 run_tests.sh      | 0
 test_suite.sh     | 0
 3 files changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 bot/check-test.sh
 mode change 100644 => 100755 run_tests.sh
 mode change 100644 => 100755 test_suite.sh

diff --git a/bot/check-test.sh b/bot/check-test.sh
old mode 100644
new mode 100755
diff --git a/run_tests.sh b/run_tests.sh
old mode 100644
new mode 100755
diff --git a/test_suite.sh b/test_suite.sh
old mode 100644
new mode 100755

From 081e2b195b1dd948ae86d4f8e1634d368db3b467 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 10:30:25 +0100
Subject: [PATCH 007/137] Check for existence of SLURM output first

---
 bot/check-test.sh | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 9fc783aa99..9308a87e8e 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -13,6 +13,17 @@ job_dir=${PWD}
 job_out="slurm-${SLURM_JOB_ID}.out"
 job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
 
+# Check that job output file is found
+[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
+if  [[ -f ${job_out} ]]; then
+    SLURM=1
+    [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
+else
+    SLURM=0
+    [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
+fi
+
+
 # ReFrame prints e.g.
 #[----------] start processing checks
 #[ RUN      ] GROMACS_EESSI %benchmark_info=HECBioSim/Crambin %nb_impl=cpu %scale=2_nodes %module_name=GROMACS/2021.3-foss-2021a /d597cff4 @snellius:rome+default
@@ -39,7 +50,6 @@ if [[ ${SLURM} -eq 1 ]]; then
   [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
 fi
 
-
 # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
 # I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests
 ERROR=-1
@@ -53,11 +63,13 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 echo "[TEST]" > ${job_test_result_file}
-if [[ ${ERROR} -eq 1 ]]; then
-    echo "comment_description = Failure to execute test step" >> ${job_test_result_file}
+if [[ ${SLURM} -eq 0 ]]; then
+    echo "comment_description = FAILED (job output file not found)" >> ${job_test_result_file}
+elif [[ ${ERROR} -eq 1 ]]; then
+    echo "comment_description = FAILED (test step failed to execute)" >> ${job_test_result_file}
     echo "status = FAILURE" >> ${job_test_result_file}
 elif [[ ${FAILED} -eq 1 ]]; then
-    echo "comment_description = EESSI test suite produced failures" >> ${job_test_result_file}
+    echo "comment_description = FAILED (EESSI test suite produced failures)" >> ${job_test_result_file}
 else
     echo "comment_description = Test step run succesfully" >> ${job_test_result_file}
     echo "status = SUCCESS" >> ${job_test_result_file}

From 9b6fa731d6ac55e3e57538eda01c9c47d73300b1 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 10:43:20 +0100
Subject: [PATCH 008/137] Clarify return messages from the bot

---
 bot/check-test.sh |  8 ++++----
 test_suite.sh     | 20 --------------------
 2 files changed, 4 insertions(+), 24 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 9308a87e8e..05c7589640 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -64,14 +64,14 @@ fi
 
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
-    echo "comment_description = FAILED (job output file not found)" >> ${job_test_result_file}
+    echo "comment_description = :cry: FAILED (job output file not found, cannot check test results)" >> ${job_test_result_file}
 elif [[ ${ERROR} -eq 1 ]]; then
-    echo "comment_description = FAILED (test step failed to execute)" >> ${job_test_result_file}
+    echo "comment_description = :cry: FAILED (EESSI test suite was not run, test step itself failed to execute)" >> ${job_test_result_file}
     echo "status = FAILURE" >> ${job_test_result_file}
 elif [[ ${FAILED} -eq 1 ]]; then
-    echo "comment_description = FAILED (EESSI test suite produced failures)" >> ${job_test_result_file}
+    echo "comment_description = :cry: FAILED (EESSI test suite produced failures)" >> ${job_test_result_file}
 else
-    echo "comment_description = Test step run succesfully" >> ${job_test_result_file}
+    echo "comment_description = :grin: SUCCESS" >> ${job_test_result_file}
     echo "status = SUCCESS" >> ${job_test_result_file}
 fi
 
diff --git a/test_suite.sh b/test_suite.sh
index 36262a6cbf..7995f8aa3a 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -78,26 +78,9 @@ else
     fatal_error "$EESSI_CVMFS_REPO is not available!"
 fi
 
-# make sure we're in Prefix environment by checking $SHELL
-if [[ ${SHELL} = ${EPREFIX}/bin/bash ]]; then
-    echo_green ">> It looks like we're in a Gentoo Prefix environment, good!"
-else
-    fatal_error "Not running in Gentoo Prefix environment, run '${EPREFIX}/startprefix' first!"
-fi
-
 # avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
 export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
 
-DETECTION_PARAMETERS=''
-GENERIC=0
-EB='eb'
-if [[ "$EASYBUILD_OPTARCH" == "GENERIC" ]]; then
-    echo_yellow ">> GENERIC build/test requested, taking appropriate measures!"
-    DETECTION_PARAMETERS="$DETECTION_PARAMETERS --generic"
-    GENERIC=1
-    EB='eb --optarch=GENERIC'
-fi
-
 echo ">> Determining software subdirectory to use for current build/test host..."
 if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
   export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
@@ -129,9 +112,6 @@ else
     fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
 fi
 
-echo ">> Configuring EasyBuild..."
-source $TOPDIR/configure_easybuild
-
 echo ">> Setting up \$MODULEPATH..."
 # make sure no modules are loaded
 module --force purge

From cb48b36e590f1d36d7a8d08df22eb538b716c70b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 11:32:15 +0100
Subject: [PATCH 009/137] Use EESSI_SOFTWARE_PATH instead of
 EASYBUILD_INSTALLPATH to set the modulepath. It should be the same, and we no
 longer configure EasyBuild, since we don't use it when running the test suite

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 7995f8aa3a..1464a3a5d4 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -117,7 +117,7 @@ echo ">> Setting up \$MODULEPATH..."
 module --force purge
 # ignore current $MODULEPATH entirely
 module unuse $MODULEPATH
-module use $EASYBUILD_INSTALLPATH/modules/all
+module use ${EESSI_SOFTWARE_PATH}/modules/all
 if [[ -z ${MODULEPATH} ]]; then
     fatal_error "Failed to set up \$MODULEPATH?!"
 else

From d1a0219486fe4a4c1ad94039b4b74b41ebf7da1f Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 12:02:12 +0100
Subject: [PATCH 010/137] Do git clone in a seperate script, so that that can
 be run in the prefix layer

---
 clone_eessi_test_suite.sh | 1 +
 run_tests.sh              | 3 ++-
 test_suite.sh             | 9 +++++++--
 3 files changed, 10 insertions(+), 3 deletions(-)
 create mode 100755 clone_eessi_test_suite.sh

diff --git a/clone_eessi_test_suite.sh b/clone_eessi_test_suite.sh
new file mode 100755
index 0000000000..f7684c8cfd
--- /dev/null
+++ b/clone_eessi_test_suite.sh
@@ -0,0 +1 @@
+git clone https://github.com/EESSI/test-suite EESSI-test-suite
diff --git a/run_tests.sh b/run_tests.sh
index 69672f18f3..90185fea17 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -9,4 +9,5 @@ source ${base_dir}/init/eessi_defaults
 # TODO: in the future we may implement this as a light first check.
 
 # Run the test suite
-./test_suite.sh "$@"
+./run_in_compat_layer_env.sh clone_eessi_test_suite.sh
+./test_suite.sh
diff --git a/test_suite.sh b/test_suite.sh
index 1464a3a5d4..6177f0a253 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -141,9 +141,14 @@ else
     fatal_error "Failed to load hpctestlib"
 fi
 
-# Clone the EESSI test suite
-git clone https://github.com/EESSI/test-suite EESSI-test-suite
+# Cloning should already be done by clone_eessi_test_suite.sh, which runs in compat layer to have 'git' available
+# git clone https://github.com/EESSI/test-suite EESSI-test-suite
 export TESTSUITEPREFIX=$PWD/EESSI-test-suite
+if [ -d $TESTSUITEPREFIX ]; then
+    echo_green "Clone of the test suite $TESTSUITEPREFIX available, OK!"
+else
+    fatal_error "Clone of the test suite $TESTSUITEPREFIX is not available!"
+fi
 export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH
 
 # Check that we can import from the testsuite

From 86d5d3d47d076fbdf0a61c57ca83fee8a1c87123 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 12:06:09 +0100
Subject: [PATCH 011/137] Should use the one from current dir

---
 run_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/run_tests.sh b/run_tests.sh
index 90185fea17..bcca1ee417 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -9,5 +9,5 @@ source ${base_dir}/init/eessi_defaults
 # TODO: in the future we may implement this as a light first check.
 
 # Run the test suite
-./run_in_compat_layer_env.sh clone_eessi_test_suite.sh
+./run_in_compat_layer_env.sh ./clone_eessi_test_suite.sh
 ./test_suite.sh

From 7861aec18c0081c8ff24e7ed46d34f9ea52c2dae Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 12:10:56 +0100
Subject: [PATCH 012/137] Remove quotes, see if that helps

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 6177f0a253..1ac82a5e66 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -175,7 +175,7 @@ fi
 
 # List the tests we want to run
 export REFRAME_ARGS='--tag CI --tag 1_nodes'
-reframe "${REFRAME_ARGS}" --list
+reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
 else

From a7b5ee13c841e940ab136ee2d014b4426a23baa7 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 12:52:41 +0100
Subject: [PATCH 013/137] Apparently, we need to remove curly braces too

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 1ac82a5e66..46c3493f41 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -175,7 +175,7 @@ fi
 
 # List the tests we want to run
 export REFRAME_ARGS='--tag CI --tag 1_nodes'
-reframe ${REFRAME_ARGS} --list
+reframe $REFRAME_ARGS --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
 else

From f50e463e9d0d0f5d91b161805279b6a8a764b990 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 13:03:31 +0100
Subject: [PATCH 014/137] Make sure this actually gets reported as failure, as
 the ReFrame runtime fails to run the test suite (it is not an indication of
 tests failing themselves).

---
 test_suite.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 46c3493f41..8a69bdee8d 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -175,7 +175,7 @@ fi
 
 # List the tests we want to run
 export REFRAME_ARGS='--tag CI --tag 1_nodes'
-reframe $REFRAME_ARGS --list
+reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
 else
@@ -184,6 +184,12 @@ fi
 
 # Run all tests
 reframe "${REFRAME_ARGS}" --run
+if [[ $? -eq 0 ]]; then
+    echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."
+else
+    fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run."
+fi
+
 
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}

From f67df9bc1319053901aa506668533cbb8b99716f Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 13:04:49 +0100
Subject: [PATCH 015/137] Fix the actual issue: make sure that two arguments
 are interpreted seperately by not quoting them

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 8a69bdee8d..633333890c 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -183,7 +183,7 @@ else
 fi
 
 # Run all tests
-reframe "${REFRAME_ARGS}" --run
+reframe ${REFRAME_ARGS} --run
 if [[ $? -eq 0 ]]; then
     echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."
 else

From 3394851c5347a045431d7826f579ad5f35f01edf Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 13:32:27 +0100
Subject: [PATCH 016/137] See if we can make check-test formatting more fancy

---
 bot/check-test.sh | 84 +++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 78 insertions(+), 6 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 05c7589640..3fa7c1d694 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -64,15 +64,87 @@ fi
 
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
-    echo "comment_description = :cry: FAILED (job output file not found, cannot check test results)" >> ${job_test_result_file}
+    summary=":cry: FAILURE"
+    summary_details="(job output file not found, cannot check test results)"# >> ${job_test_result_file}
+    status="FAILURE"
 elif [[ ${ERROR} -eq 1 ]]; then
-    echo "comment_description = :cry: FAILED (EESSI test suite was not run, test step itself failed to execute)" >> ${job_test_result_file}
-    echo "status = FAILURE" >> ${job_test_result_file}
+    summary=":cry: FAILURE"
+    summary_details="(EESSI test suite was not run, test step itself failed to execute)"# >> ${job_test_result_file}
+    status="FAILURE"
+#    echo "status = FAILURE" >> ${job_test_result_file}
 elif [[ ${FAILED} -eq 1 ]]; then
-    echo "comment_description = :cry: FAILED (EESSI test suite produced failures)" >> ${job_test_result_file}
+    summary=":cry: FAILURE"
+    summary_details="(EESSI test suite produced failures)"# >> ${job_test_result_file}
+    status="FAILURE"
 else
-    echo "comment_description = :grin: SUCCESS" >> ${job_test_result_file}
-    echo "status = SUCCESS" >> ${job_test_result_file}
+    summary=":grin: SUCCESS"# >> ${job_test_result_file}
+    summary_details=""
+    status="SUCCESS"
+#    echo "status = SUCCESS" >> ${job_test_result_file}
 fi
 
+function add_detail() {
+    actual=${1}
+    expected=${2}
+    success_msg="${3}"
+    failure_msg="${4}"
+    if [[ ${actual} -eq ${expected} ]]; then
+        success "${success_msg}"
+    else
+        failure "${failure_msg}"
+    fi
+}
+
+echo "[TEST]" > ${job_result_file}
+echo -n "comment_description = " >> ${job_result_file}
+
+# Use template for writing PR comment with details
+# construct and write complete PR comment details: implements third alternative
+comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT____ARTEFACTS_FMT__</dl></details>"
+comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
+comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
+comment_success_item_fmt=":white_check_mark: __ITEM__"
+comment_failure_item_fmt=":x: __ITEM__"
+comment_artefacts_fmt="<dt>_Artefacts_</dt><dd>__ARTEFACTS_LIST__</dd>"
+comment_artefact_details_fmt="<details>__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__</details>"
+
+comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+
+# first construct comment_details_list, abbreviated CoDeList
+# then use it to set comment_details
+CoDeList=""
+
+success_msg="job output file <code>${job_out}</code>"
+failure_msg="no job output file <code>${job_out}</code>"
+CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching <code>${GP_error}</code>"
+failure_msg="found message matching <code>${GP_error}</code>"
+CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+
+success_msg="no message matching <code>${GP_failed}</code>"
+failure_msg="found message matching <code>${GP_failed}</code>"
+CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+
+# Should not be needed for testing, I think? Maybe for loading ReFrame module...
+# success_msg="no message matching <code>${GP_req_missing}</code>"
+# failure_msg="found message matching <code>${GP_req_missing}</code>"
+# CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+# 
+# success_msg="found message(s) matching <code>${GP_no_missing}</code>"
+# failure_msg="no message matching <code>${GP_no_missing}</code>"
+# CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+# 
+# success_msg="found message matching <code>${GP_tgz_created}</code>"
+# failure_msg="no message matching <code>${GP_tgz_created}</code>"
+# CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+
+comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
+comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
+
+# Actually writing the comment description to the result file
+echo "${comment_description}" >> ${job_result_file}
+
 exit 0

From 8e5682a47a589a2e251852ed674b71f16980ffc7 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 14:51:46 +0100
Subject: [PATCH 017/137] Replace the env var for the result file

---
 bot/check-test.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 3fa7c1d694..89d3d6a3c6 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -95,8 +95,8 @@ function add_detail() {
     fi
 }
 
-echo "[TEST]" > ${job_result_file}
-echo -n "comment_description = " >> ${job_result_file}
+echo "[TEST]" > ${job_test_result_file}
+echo -n "comment_description = " >> ${job_test_result_file}
 
 # Use template for writing PR comment with details
 # construct and write complete PR comment details: implements third alternative
@@ -145,6 +145,6 @@ comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
 
 # Actually writing the comment description to the result file
-echo "${comment_description}" >> ${job_result_file}
+echo "${comment_description}" >> ${job_test_result_file}
 
 exit 0

From 6cf0cf959844081b004595bd3a6c21329ccf256d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 14:53:21 +0100
Subject: [PATCH 018/137] Add missing functions

---
 bot/check-test.sh | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 89d3d6a3c6..87c79b3b30 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -83,6 +83,24 @@ else
 #    echo "status = SUCCESS" >> ${job_test_result_file}
 fi
 
+function print_br_item() {
+    format="${1}"
+    item="${2}"
+    echo -n "${format//__ITEM__/${item}}<br/>"
+}
+
+function success() {
+    format="${comment_success_item_fmt}"
+    item="$1"
+    print_br_item "${format}" "${item}"
+}
+
+function failure() {
+    format="${comment_failure_item_fmt}"
+    item="$1"
+    print_br_item "${format}" "${item}"
+}
+
 function add_detail() {
     actual=${1}
     expected=${2}

From 7fe24e13cdef3f703b51930423a2296e3dc07d59 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 15:14:50 +0100
Subject: [PATCH 019/137] Remove artefacts from reporting

---
 bot/check-test.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 87c79b3b30..da06cd630b 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -118,13 +118,11 @@ echo -n "comment_description = " >> ${job_test_result_file}
 
 # Use template for writing PR comment with details
 # construct and write complete PR comment details: implements third alternative
-comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT____ARTEFACTS_FMT__</dl></details>"
+comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT__</dl></details>"
 comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_success_item_fmt=":white_check_mark: __ITEM__"
 comment_failure_item_fmt=":x: __ITEM__"
-comment_artefacts_fmt="<dt>_Artefacts_</dt><dd>__ARTEFACTS_LIST__</dd>"
-comment_artefact_details_fmt="<details>__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__</details>"
 
 comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
 

From afc309a168048e54b625ec857a5d09a14ca1d585 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 15:25:33 +0100
Subject: [PATCH 020/137] Add a reason for the failure

---
 bot/check-test.sh | 42 ++++++++++++++++++++++--------------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index da06cd630b..38e86efccd 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -65,24 +65,37 @@ fi
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
     summary=":cry: FAILURE"
-    summary_details="(job output file not found, cannot check test results)"# >> ${job_test_result_file}
+    summary_details="Reason: job output file not found, cannot check test results."
     status="FAILURE"
 elif [[ ${ERROR} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    summary_details="(EESSI test suite was not run, test step itself failed to execute)"# >> ${job_test_result_file}
+    summary_details="Reason: EESSI test suite was not run, test step itself failed to execute."
     status="FAILURE"
-#    echo "status = FAILURE" >> ${job_test_result_file}
 elif [[ ${FAILED} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    summary_details="(EESSI test suite produced failures)"# >> ${job_test_result_file}
+    summary_details="Reason: EESSI test suite produced failures."
     status="FAILURE"
 else
-    summary=":grin: SUCCESS"# >> ${job_test_result_file}
+    summary=":grin: SUCCESS"
     summary_details=""
     status="SUCCESS"
-#    echo "status = SUCCESS" >> ${job_test_result_file}
 fi
 
+
+echo "[TEST]" > ${job_test_result_file}
+echo -n "comment_description = " >> ${job_test_result_file}
+
+# Use template for writing PR comment with details
+# construct and write complete PR comment details: implements third alternative
+comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT__</dl></details>"
+comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
+comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
+comment_success_item_fmt=":white_check_mark: __ITEM__"
+comment_failure_item_fmt=":x: __ITEM__"
+
+comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+
+# Declare functions
 function print_br_item() {
     format="${1}"
     item="${2}"
@@ -113,22 +126,11 @@ function add_detail() {
     fi
 }
 
-echo "[TEST]" > ${job_test_result_file}
-echo -n "comment_description = " >> ${job_test_result_file}
-
-# Use template for writing PR comment with details
-# construct and write complete PR comment details: implements third alternative
-comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT__</dl></details>"
-comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
-comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
-comment_success_item_fmt=":white_check_mark: __ITEM__"
-comment_failure_item_fmt=":x: __ITEM__"
-
-comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
-
 # first construct comment_details_list, abbreviated CoDeList
 # then use it to set comment_details
-CoDeList=""
+
+# Initialize with summary_details, which elaborates on the reason for failure
+CoDeList=$(print_br_item "__ITEM__" "${summary_details}"
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"

From b84e48755dfb582a5a4bae3480e8865df179331e Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 15:32:07 +0100
Subject: [PATCH 021/137] Fixed missing bracket

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 38e86efccd..e340907bb3 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -130,7 +130,7 @@ function add_detail() {
 # then use it to set comment_details
 
 # Initialize with summary_details, which elaborates on the reason for failure
-CoDeList=$(print_br_item "__ITEM__" "${summary_details}"
+CoDeList=$(print_br_item "__ITEM__" "${summary_details}")
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"

From ef7bc0142f0ff518e82121371403c4da9bc0444d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 15:48:44 +0100
Subject: [PATCH 022/137] Add reporting of the ReFrame result

---
 bot/check-test.sh | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index e340907bb3..78c153e5dc 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -43,11 +43,11 @@ fi
 FAILED=-1
 if [[ ${SLURM} -eq 1 ]]; then
   GP_failed='\[\s*FAILED\s*\]'
-  grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
+  grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
-  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
 fi
 
 # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
@@ -62,6 +62,15 @@ if [[ ${SLURM} -eq 1 ]]; then
   [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
 fi
 
+# Grep for the success pattern, so we can report the amount of tests run
+if [[ ${SLURM} -eq 1 ]]; then
+  GP_success='\[\s*PASSED\s*\]'
+  grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
+fi
+
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
     summary=":cry: FAILURE"
@@ -132,6 +141,11 @@ function add_detail() {
 # Initialize with summary_details, which elaborates on the reason for failure
 CoDeList=$(print_br_item "__ITEM__" "${summary_details}")
 
+# Add final ReFrame line as line
+if [[ ! -z ${grep_reframe_result} ]]; then
+    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}"
+fi
+
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
 CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
@@ -140,8 +154,8 @@ success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
 CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
-success_msg="no message matching <code>${GP_failed}</code>"
-failure_msg="found message matching <code>${GP_failed}</code>"
+success_msg="no message matching <code>""${GP_failed}""</code>"
+failure_msg="found message matching <code>""${GP_failed}""</code>"
 CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
 # Should not be needed for testing, I think? Maybe for loading ReFrame module...

From 867681a2ed7413b8ecdbf646d05a20d7d2553adc Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 16:01:40 +0100
Subject: [PATCH 023/137] Add missing bracket. Again

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 78c153e5dc..7ff5f8a3ea 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -143,7 +143,7 @@ CoDeList=$(print_br_item "__ITEM__" "${summary_details}")
 
 # Add final ReFrame line as line
 if [[ ! -z ${grep_reframe_result} ]]; then
-    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}"
+    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
 fi
 
 success_msg="job output file <code>${job_out}</code>"

From 47e30cac825362f85138e1d6cb3fe8ba02f13c6e Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 16:17:25 +0100
Subject: [PATCH 024/137] Add some echo for debugging

---
 bot/check-test.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 7ff5f8a3ea..1a2d8b5d5d 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -171,6 +171,9 @@ CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 # failure_msg="no message matching <code>${GP_tgz_created}</code>"
 # CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
 
+echo "Finale CoDeList is:"
+echo ${CoDeList}
+
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
 
 comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}

From 1bdadf884e8f18ec911f8961a894125a6c5cccce Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 16:24:34 +0100
Subject: [PATCH 025/137] more debugging output

---
 bot/check-test.sh | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 1a2d8b5d5d..841b8ed01b 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -137,14 +137,20 @@ function add_detail() {
 
 # first construct comment_details_list, abbreviated CoDeList
 # then use it to set comment_details
+CoDeList=""
 
 # Initialize with summary_details, which elaborates on the reason for failure
-CoDeList=$(print_br_item "__ITEM__" "${summary_details}")
+if [[ ! -z ${summary_details} ]]; then
+    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}")
+fi
 
 # Add final ReFrame line as line
 if [[ ! -z ${grep_reframe_result} ]]; then
     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
 fi
+echo "CoDeList up here is"
+echo ${CoDeList}
+
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"

From 5152f5e47380c462c234566f5fc3c29922694ef1 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 16:54:02 +0100
Subject: [PATCH 026/137] Lets try some nicer formatting with headers etc. See
 if this actually prints something.

---
 bot/check-test.sh | 62 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 42 insertions(+), 20 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 841b8ed01b..8f8eb66f5e 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -74,19 +74,19 @@ fi
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
     summary=":cry: FAILURE"
-    summary_details="Reason: job output file not found, cannot check test results."
+    reason="Reason: job output file not found, cannot check test results."
     status="FAILURE"
 elif [[ ${ERROR} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    summary_details="Reason: EESSI test suite was not run, test step itself failed to execute."
+    reason="Reason: EESSI test suite was not run, test step itself failed to execute."
     status="FAILURE"
 elif [[ ${FAILED} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    summary_details="Reason: EESSI test suite produced failures."
+    reason="Reason: EESSI test suite produced failures."
     status="FAILURE"
 else
     summary=":grin: SUCCESS"
-    summary_details=""
+    reason=""
     status="SUCCESS"
 fi
 
@@ -96,13 +96,36 @@ echo -n "comment_description = " >> ${job_test_result_file}
 
 # Use template for writing PR comment with details
 # construct and write complete PR comment details: implements third alternative
-comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT__</dl></details>"
-comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
-comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
+comment_template="<details>__SUMMARY_FMT__<dl>__REASON_FMT____REFRAME_FMT____DETAILS_FMT__</dl></details>"
 comment_success_item_fmt=":white_check_mark: __ITEM__"
 comment_failure_item_fmt=":x: __ITEM__"
 
+# Initialize comment_description
+comment_description=${comment_template}
+
+# Now, start replacing template items one by one
+comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
 comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+comment_description=${comment_description/__SUMMARY_FMT__/${comment_summary}}
+
+
+# Omit this if there is no reason (e.g. because it was succesful)
+if [[ -z ${reason} ]]; then
+    comment_reason_fmt="<dt>_Reason_</dt><dd>__REASONS__</dd>"
+    reason_details="${comment_reason_fmt/__REASONS__/${reason}}"
+    comment_description=${comment_description/__REASON_FMT__/${reason_details}}
+else
+    comment_description=${comment_description/__REASON_FMT__/""}
+fi
+
+# Omit this if there is no reframe summary (i.e. the workflow didn't run succesfully)
+if [[ -z ${grep_reframe_result} ]]; then
+    comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
+    reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}}
+    comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}}
+else
+    comment_description=${comment_description/__REFRAME_FMT__/""}
+fi
 
 # Declare functions
 function print_br_item() {
@@ -139,17 +162,17 @@ function add_detail() {
 # then use it to set comment_details
 CoDeList=""
 
-# Initialize with summary_details, which elaborates on the reason for failure
-if [[ ! -z ${summary_details} ]]; then
-    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}")
-fi
-
-# Add final ReFrame line as line
-if [[ ! -z ${grep_reframe_result} ]]; then
-    CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
-fi
-echo "CoDeList up here is"
-echo ${CoDeList}
+# # Initialize with summary_details, which elaborates on the reason for failure
+# if [[ ! -z ${summary_details} ]]; then
+#     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}")
+# fi
+# 
+# # Add final ReFrame line as line
+# if [[ ! -z ${grep_reframe_result} ]]; then
+#     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
+# fi
+# echo "CoDeList up here is"
+# echo ${CoDeList}
 
 
 success_msg="job output file <code>${job_out}</code>"
@@ -180,9 +203,8 @@ CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 echo "Finale CoDeList is:"
 echo ${CoDeList}
 
+comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
-
-comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
 
 # Actually writing the comment description to the result file

From ea38c66114af601071e1edc626cf9952219a7a18 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:07:28 +0100
Subject: [PATCH 027/137] Inverse check, it was incorrect

---
 bot/check-test.sh | 21 ++++-----------------
 1 file changed, 4 insertions(+), 17 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 8f8eb66f5e..01769bbac3 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -109,8 +109,8 @@ comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
 comment_description=${comment_description/__SUMMARY_FMT__/${comment_summary}}
 
 
-# Omit this if there is no reason (e.g. because it was succesful)
-if [[ -z ${reason} ]]; then
+# Only add if there is a reason (e.g. no reason for successful runs)
+if [[ ! -z ${reason} ]]; then
     comment_reason_fmt="<dt>_Reason_</dt><dd>__REASONS__</dd>"
     reason_details="${comment_reason_fmt/__REASONS__/${reason}}"
     comment_description=${comment_description/__REASON_FMT__/${reason_details}}
@@ -118,8 +118,8 @@ else
     comment_description=${comment_description/__REASON_FMT__/""}
 fi
 
-# Omit this if there is no reframe summary (i.e. the workflow didn't run succesfully)
-if [[ -z ${grep_reframe_result} ]]; then
+# Only add if there is a reframe summary (e.g. no reframe summary if reframe wasn't launched succesfully)
+if [[ ! -z ${grep_reframe_result} ]]; then
     comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
     reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}}
     comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}}
@@ -162,19 +162,6 @@ function add_detail() {
 # then use it to set comment_details
 CoDeList=""
 
-# # Initialize with summary_details, which elaborates on the reason for failure
-# if [[ ! -z ${summary_details} ]]; then
-#     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${summary_details}")
-# fi
-# 
-# # Add final ReFrame line as line
-# if [[ ! -z ${grep_reframe_result} ]]; then
-#     CoDeList=${CoDeList}$(print_br_item "__ITEM__" "${grep_reframe_result}")
-# fi
-# echo "CoDeList up here is"
-# echo ${CoDeList}
-
-
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
 CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")

From 8edb6fffb401f0e74e0b975b5585d3f9257f88f8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:11:41 +0100
Subject: [PATCH 028/137] Remove debugging output

---
 bot/check-test.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 01769bbac3..7a663f35d1 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -187,9 +187,6 @@ CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 # failure_msg="no message matching <code>${GP_tgz_created}</code>"
 # CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
 
-echo "Finale CoDeList is:"
-echo ${CoDeList}
-
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}

From 26bd2cb04af381af580805b094f69c096ea40410 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:33:12 +0100
Subject: [PATCH 029/137] Correct the regex pattern to account for the fact
 that the slurm output file contains special characters in that pattern that
 are NOT whitespace

---
 bot/check-test.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 7a663f35d1..aebd56d333 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -42,7 +42,7 @@ fi
 # Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
 FAILED=-1
 if [[ ${SLURM} -eq 1 ]]; then
-  GP_failed='\[\s*FAILED\s*\]'
+  GP_failed='\[.*FAILED.*\].*Ran .* test case'
   grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
@@ -64,7 +64,7 @@ fi
 
 # Grep for the success pattern, so we can report the amount of tests run
 if [[ ${SLURM} -eq 1 ]]; then
-  GP_success='\[\s*PASSED\s*\]'
+  GP_success='\[.*PASSED.*\].*Ran .* test case'
   grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
@@ -119,6 +119,8 @@ else
 fi
 
 # Only add if there is a reframe summary (e.g. no reframe summary if reframe wasn't launched succesfully)
+echo "ReFrame result:"
+echo "${grep_reframe_result}"
 if [[ ! -z ${grep_reframe_result} ]]; then
     comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
     reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}}

From 416b72d3cb18d6150e0ef8760cd5b66867e29496 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:45:30 +0100
Subject: [PATCH 030/137] Corrected typo

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index aebd56d333..702add848d 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -123,7 +123,7 @@ echo "ReFrame result:"
 echo "${grep_reframe_result}"
 if [[ ! -z ${grep_reframe_result} ]]; then
     comment_reframe_fmt="<dt>_ReFrame Summary_</dt><dd>__REFRAME_SUMMARY__</dd>"
-    reframe_summary=${comment_reframe_ftm/__REFRAME_SUMMARY__/${grep_reframe_result}}
+    reframe_summary=${comment_reframe_fmt/__REFRAME_SUMMARY__/${grep_reframe_result}}
     comment_description=${comment_description/__REFRAME_FMT__/${reframe_summary}}
 else
     comment_description=${comment_description/__REFRAME_FMT__/""}

From 19900937361e16259dce714c9e4acda4b09e0a7b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 17:51:27 +0100
Subject: [PATCH 031/137] Trigger failure in the workflow on purpose, to show
 what that looks like

---
 test_suite.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index 633333890c..4439821d56 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -166,7 +166,7 @@ export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs
 
 # Check we can run reframe
-reframe --version
+reframe --version BLABLABLA
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully ran reframe --version"
 else
@@ -174,7 +174,7 @@ else
 fi
 
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_nodes'
+export REFRAME_ARGS='--tag CI --tag 1_node'
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"

From 4b9885612fa13d2d1f45522ca1aa04ec333863b8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 6 Feb 2024 18:11:30 +0100
Subject: [PATCH 032/137] Change order of checking error codes. If the ReFrame
 tests ran and we find that pattern, that should take priority. Other errors
 might have come from errors in the build phase...

---
 bot/check-test.sh | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 702add848d..7b7ada98e7 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -62,10 +62,12 @@ if [[ ${SLURM} -eq 1 ]]; then
   [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}"
 fi
 
+SUCCESS=-1
 # Grep for the success pattern, so we can report the amount of tests run
 if [[ ${SLURM} -eq 1 ]]; then
   GP_success='\[.*PASSED.*\].*Ran .* test case'
   grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
+  [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
   [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
@@ -74,19 +76,27 @@ fi
 echo "[TEST]" > ${job_test_result_file}
 if [[ ${SLURM} -eq 0 ]]; then
     summary=":cry: FAILURE"
-    reason="Reason: job output file not found, cannot check test results."
+    reason="Job output file not found, cannot check test results."
     status="FAILURE"
-elif [[ ${ERROR} -eq 1 ]]; then
+# Should come before general errors: if SUCCESS==1, it indicates the test suite ran succesfully
+# regardless of other things that might have gone wrong
+elif [[ ${SUCCESS} -eq 1 ]]; then
+    summary=":grin: SUCCESS"
+    reason=""
+    status="SUCCESS"
+# Should come before general errors: if FAILED==1, it indicates the test suite ran
+# otherwise the pattern wouldn't have been there
+elif [[ ${FAILED} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    reason="Reason: EESSI test suite was not run, test step itself failed to execute."
+    reason="EESSI test suite produced failures."
     status="FAILURE"
-elif [[ ${FAILED} -eq 1 ]]; then
+elif [[ ${ERROR} -eq 1 ]]; then
     summary=":cry: FAILURE"
-    reason="Reason: EESSI test suite produced failures."
+    reason="EESSI test suite was not run, test step itself failed to execute."
     status="FAILURE"
 else
-    summary=":grin: SUCCESS"
-    reason=""
+    summary=":grin: FAILURE"
+    reason="Failed for unknown reason"
     status="SUCCESS"
 fi
 

From 0f261420106862049091494d6a640a6c46de5dd8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 12:11:25 +0100
Subject: [PATCH 033/137] Make sure we don't overwrite the grepped result from
 failures

---
 bot/check-test.sh | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 7b7ada98e7..8af45d26a6 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -43,11 +43,11 @@ fi
 FAILED=-1
 if [[ ${SLURM} -eq 1 ]]; then
   GP_failed='\[.*FAILED.*\].*Ran .* test case'
-  grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
+  grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'"
-  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_failed}"
 fi
 
 # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
@@ -66,11 +66,17 @@ SUCCESS=-1
 # Grep for the success pattern, so we can report the amount of tests run
 if [[ ${SLURM} -eq 1 ]]; then
   GP_success='\[.*PASSED.*\].*Ran .* test case'
-  grep_reframe_result=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
+  grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
   [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
   # have to be careful to not add searched for pattern into slurm out file
   [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_success}"'"
-  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_result}"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_reframe_success}"
+fi
+
+if [[ ! -z ${grep_reframe_failed} ]]; then
+    grep_reframe_results=${grep_reframe_failed}
+else
+    grep_reframe_results=${grep_reframe_success}
 fi
 
 echo "[TEST]" > ${job_test_result_file}

From 94e0bb5e3b886192508d95485b221b3ec7a6ff5e Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 13:09:31 +0100
Subject: [PATCH 034/137] Fix plural/singular typo

---
 bot/check-test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 8af45d26a6..39901f0691 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -74,9 +74,9 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 if [[ ! -z ${grep_reframe_failed} ]]; then
-    grep_reframe_results=${grep_reframe_failed}
+    grep_reframe_result=${grep_reframe_failed}
 else
-    grep_reframe_results=${grep_reframe_success}
+    grep_reframe_result=${grep_reframe_success}
 fi
 
 echo "[TEST]" > ${job_test_result_file}

From 9e7ac1c7c4f2f64250ef504810d057e0cb3e5225 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 13:25:45 +0100
Subject: [PATCH 035/137] Be a bit more expressive about what we are doing

---
 test_suite.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index 4439821d56..1610d87738 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -165,8 +165,11 @@ export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
 export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs
 
+echo "Configured reframe with the following environment variables:"
+env | grep "RFM_"
+
 # Check we can run reframe
-reframe --version BLABLABLA
+reframe --version
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully ran reframe --version"
 else
@@ -175,6 +178,7 @@ fi
 
 # List the tests we want to run
 export REFRAME_ARGS='--tag CI --tag 1_node'
+echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully listed ReFrame tests with command: reframe ${REFRAME_ARGS} --list"
@@ -183,6 +187,7 @@ else
 fi
 
 # Run all tests
+echo "Running tests: reframe ${REFRAME_ARGS} --run"
 reframe ${REFRAME_ARGS} --run
 if [[ $? -eq 0 ]]; then
     echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."
@@ -190,7 +195,6 @@ else
     fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run."
 fi
 
-
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}
 

From 3a52bde23e5de6cfe4d5da170060c07d8dfc0009 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 14:28:08 +0100
Subject: [PATCH 036/137] Change to dedicated config file for testing by the
 bot in the build job

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 1610d87738..d6699878f9 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -160,7 +160,7 @@ else
 fi
 
 # Configure ReFrame
-export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/github_actions.py
+export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/software_layer_bot.py
 export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
 export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs

From 159e384bdef117265e09a0a420cce3fa6d8842c8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 14:47:54 +0100
Subject: [PATCH 037/137] Make it verbose, see if we can see why autodetect
 fails

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index d6699878f9..a8e1b2d97a 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -177,7 +177,7 @@ else
 fi
 
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_node'
+export REFRAME_ARGS='--tag CI --tag 1_node -vvv'
 echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then

From b92044e5f913ce0a6c7082c0ebfb4e238d47f95d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 16:23:53 +0100
Subject: [PATCH 038/137] Add ReFrame config template. Not that four template
 variables need to be replaced for this to be a valid ReFrame config file

---
 reframe_config_bot.py | 58 +++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 58 insertions(+)
 create mode 100644 reframe_config_bot.py

diff --git a/reframe_config_bot.py b/reframe_config_bot.py
new file mode 100644
index 0000000000..65ccf9f43e
--- /dev/null
+++ b/reframe_config_bot.py
@@ -0,0 +1,58 @@
+# WARNING: this file is intended as template and the __X__ template variables need to be replaced
+# before it can act as a configuration file
+# Once replaced, this is a config file for running tests after the build phase, by the bot
+
+from eessi.testsuite.common_config import common_logging_config
+from eessi.testsuite.constants import *  # noqa: F403
+
+
+site_configuration = {
+    'systems': [
+        {
+            'name': 'Testing in bot Build jobs for EESSI software Layer',
+            'descr': 'Software-layer bot',
+            'hostnames': ['.*'],
+            'modules_system': 'lmod',
+            'partitions': [
+                {
+                    'name': 'default',
+                    'scheduler': 'local',
+                    'launcher': 'mpirun',
+                    'environs': ['default'],
+                    'features': [FEATURES[CPU]],
+                    'processor': {
+                        'num_cpus': __NUM_CPUS__,
+                        'num_sockets': __NUM_SOCKETS__,
+                        'num_cpus_per_core': __NUM_CPUS_PER_CORE__,
+                        'num_cpus_per_socket': __NUM_CPUS_PER_SOCKET__,
+                    }
+                    'resources': [
+                        {
+                            'name': 'memory',
+                            'options': ['--mem={size}'],
+                        }
+                    ],
+                    'max_jobs': 1
+                    }
+                ]
+            }
+        ],
+    'environments': [
+        {
+            'name': 'default',
+            'cc': 'cc',
+            'cxx': '',
+            'ftn': ''
+            }
+        ],
+    'general': [
+        {
+            'purge_environment': True,
+            'resolve_module_conflicts': False,  # avoid loading the module before submitting the job
+            # Enable automatic detection of CPU architecture
+            # See https://reframe-hpc.readthedocs.io/en/stable/configure.html#auto-detecting-processor-information
+            'remote_detect': True,
+        }
+    ],
+    'logging': common_logging_config(),
+}

From 02b0e3115dfc81387554f0cb9fe2c292e60098ab Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 16:30:18 +0100
Subject: [PATCH 039/137] Use template config file and make replacements based
 on lscpu

---
 test_suite.sh | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index a8e1b2d97a..f5cc7fabfd 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -156,11 +156,11 @@ python3 -c 'import eessi.testsuite'
 if [[ $? -eq 0 ]]; then
     echo_green "Succesfully found and imported eessi.testsuite"
 else
-    fatal_error "FAILED to import from eessi.testsuite in Python"
+    fatal_error "Failed to import from eessi.testsuite in Python"
 fi
 
 # Configure ReFrame
-export RFM_CONFIG_FILES=$TESTSUITEPREFIX/config/software_layer_bot.py
+export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py
 export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
 export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs
@@ -168,6 +168,33 @@ export RFM_PREFIX=$PWD/reframe_runs
 echo "Configured reframe with the following environment variables:"
 env | grep "RFM_"
 
+# Inject correct CPU properties into the ReFrame config file
+cpuinfo=$(lscpu)
+if [[ "${cpuinfo}" =~ CPU\(s\):[^0-9]*([0-9]+) ]]; then
+    cpu_count=${BASH_REMATCH[1]}
+else
+    fatal_error "Failed to get the number of CPUs for the current test hardware with lscpu."
+fi
+if [[ "${text}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then
+    socket_count=${BASH_REMATCH[1]}
+else
+    fatal_error "Failed to get the number of sockets for the current test hardware with lscpu."
+fi
+if [[ "${text}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then
+    threads_per_core=${BASH_REMATCH[2]}
+else
+    fatal_error "Failed to get the number of threads per core for the current test hardware with lscpu."
+fi
+if [[ "${text}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
+    cores_per_socket=${BASH_REMATCH[2]}
+else
+    fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu."
+fi
+sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_SOCKETS__/${cores_per_socket}/g" $RFM_CONFIG_FILES
+
 # Check we can run reframe
 reframe --version
 if [[ $? -eq 0 ]]; then

From 58c4cfa772454405c456503c29a6fc8d59956910 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 16:39:46 +0100
Subject: [PATCH 040/137] Fix incorrect variable name

---
 test_suite.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index f5cc7fabfd..7ba5282327 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -175,17 +175,17 @@ if [[ "${cpuinfo}" =~ CPU\(s\):[^0-9]*([0-9]+) ]]; then
 else
     fatal_error "Failed to get the number of CPUs for the current test hardware with lscpu."
 fi
-if [[ "${text}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then
+if [[ "${cpuinfo}" =~ Socket\(s\):[^0-9]*([0-9]+) ]]; then
     socket_count=${BASH_REMATCH[1]}
 else
     fatal_error "Failed to get the number of sockets for the current test hardware with lscpu."
 fi
-if [[ "${text}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then
+if [[ "${cpuinfo}" =~ (Thread\(s\) per core:[^0-9]*([0-9]+)) ]]; then
     threads_per_core=${BASH_REMATCH[2]}
 else
     fatal_error "Failed to get the number of threads per core for the current test hardware with lscpu."
 fi
-if [[ "${text}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
+if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
     cores_per_socket=${BASH_REMATCH[2]}
 else
     fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu."

From 2fc121e86399b7c7c3cab6092b609a91a88db43f Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 16:57:48 +0100
Subject: [PATCH 041/137] Missing comma

---
 reframe_config_bot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe_config_bot.py b/reframe_config_bot.py
index 65ccf9f43e..bbe8a7e2df 100644
--- a/reframe_config_bot.py
+++ b/reframe_config_bot.py
@@ -25,7 +25,7 @@
                         'num_sockets': __NUM_SOCKETS__,
                         'num_cpus_per_core': __NUM_CPUS_PER_CORE__,
                         'num_cpus_per_socket': __NUM_CPUS_PER_SOCKET__,
-                    }
+                    },
                     'resources': [
                         {
                             'name': 'memory',

From db08f4f39a618450c70f1d52f43c46f244011545 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 21:30:43 +0100
Subject: [PATCH 042/137] Corrected typo

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 7ba5282327..9bd95c6749 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -193,7 +193,7 @@ fi
 sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES
 sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
 sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
-sed -i "s/__NUM_CPUS_PER_SOCKETS__/${cores_per_socket}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES
 
 # Check we can run reframe
 reframe --version

From 6b3a6ccc985314e5c66c8097e8eb26e1c1fb3546 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 21:47:43 +0100
Subject: [PATCH 043/137] Remove verbose flags

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 9bd95c6749..99643c42b8 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -204,7 +204,7 @@ else
 fi
 
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_node -vvv'
+export REFRAME_ARGS='--tag CI --tag 1_node'
 echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then

From a70b4f132e36f078cb3b4f0d802f96a877699050 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 7 Feb 2024 21:55:16 +0100
Subject: [PATCH 044/137] Pick name without spaces, to avoid issues with
 directories

---
 reframe_config_bot.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/reframe_config_bot.py b/reframe_config_bot.py
index bbe8a7e2df..ad66a72041 100644
--- a/reframe_config_bot.py
+++ b/reframe_config_bot.py
@@ -9,7 +9,7 @@
 site_configuration = {
     'systems': [
         {
-            'name': 'Testing in bot Build jobs for EESSI software Layer',
+            'name': 'BotBuildTests',
             'descr': 'Software-layer bot',
             'hostnames': ['.*'],
             'modules_system': 'lmod',

From ce24e6d6de196ca191371b186b449b7a9d4aa5c9 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 17:50:39 +0100
Subject: [PATCH 045/137] Pick more meaningful name for SLURM variable, namely
 SLURM_OUTPUT_FOUND

---
 bot/check-build.sh | 22 +++++++++++-----------
 bot/check-test.sh  | 18 +++++++++---------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index ec1ca56bba..42dca46a08 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -95,18 +95,18 @@ job_dir=${PWD}
 
 [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}"
 
-job_out="slurm-${SLURM_JOB_ID}.out"
+job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out"
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
 if  [[ -f ${job_out} ]]; then
-    SLURM=1
+    SLURM_OUTPUT_FOUND=1
     [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
 else
-    SLURM=0
+    SLURM_OUTPUT_FOUND=0
     [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
 fi
 
 ERROR=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_error='ERROR: '
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
   [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
@@ -116,7 +116,7 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 FAILED=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_failed='FAILED: '
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
@@ -126,7 +126,7 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 MISSING=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_req_missing=' required modules missing:'
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}")
   [[ $? -eq 0 ]] && MISSING=1 || MISSING=0
@@ -136,7 +136,7 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 NO_MISSING=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_no_missing='No missing installations'
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}")
   [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0
@@ -147,7 +147,7 @@ fi
 
 TGZ=-1
 TARBALL=
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_tgz_created="\.tar\.gz created!"
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
   if [[ $? -eq 0 ]]; then
@@ -169,9 +169,9 @@ fi
 [[ ${VERBOSE} -ne 0 ]] && echo "  NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 [[ ${VERBOSE} -ne 0 ]] && echo "  TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 
-job_result_file=_bot_job${SLURM_JOB_ID}.result
+job_result_file=_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.result
 
-if [[ ${SLURM} -eq 1 ]] && \
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
    [[ ${ERROR} -eq 0 ]] && \
    [[ ${FAILED} -eq 0 ]] && \
    [[ ${MISSING} -eq 0 ]] && \
@@ -340,7 +340,7 @@ CoDeList=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
diff --git a/bot/check-test.sh b/bot/check-test.sh
index 39901f0691..549afd2f65 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -10,16 +10,16 @@
 # license: GPLv2
 #
 job_dir=${PWD}
-job_out="slurm-${SLURM_JOB_ID}.out"
-job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
+job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out"
+job_test_result_file="_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.test"
 
 # Check that job output file is found
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
 if  [[ -f ${job_out} ]]; then
-    SLURM=1
+    SLURM_OUTPUT_FOUND=1
     [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
 else
-    SLURM=0
+    SLURM_OUTPUT_FOUND=0
     [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
 fi
 
@@ -41,7 +41,7 @@ fi
 # We will grep for the last and final line, since this reflects the overall result
 # Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
 FAILED=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_failed='\[.*FAILED.*\].*Ran .* test case'
   grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
@@ -53,7 +53,7 @@ fi
 # Here, we grep for 'ERROR:', which is printed if a fatal_error is encountered when executing the test step
 # I.e. this is an error in execution of the run_tests.sh itself, NOT in running the actual tests
 ERROR=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_error='ERROR: '
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
   [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
@@ -64,7 +64,7 @@ fi
 
 SUCCESS=-1
 # Grep for the success pattern, so we can report the amount of tests run
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_success='\[.*PASSED.*\].*Ran .* test case'
   grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
   [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
@@ -80,7 +80,7 @@ else
 fi
 
 echo "[TEST]" > ${job_test_result_file}
-if [[ ${SLURM} -eq 0 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 0 ]]; then
     summary=":cry: FAILURE"
     reason="Job output file not found, cannot check test results."
     status="FAILURE"
@@ -182,7 +182,7 @@ CoDeList=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"

From 9881c2ad5f93c617c075e8df81d770033ec2a121 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 17:51:36 +0100
Subject: [PATCH 046/137] Correct erroneous status

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 549afd2f65..9724f45832 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -103,7 +103,7 @@ elif [[ ${ERROR} -eq 1 ]]; then
 else
     summary=":grin: FAILURE"
     reason="Failed for unknown reason"
-    status="SUCCESS"
+    status="FAILURE"
 fi
 
 

From 1446055cb5cd4306d61809c67b5dcda4ce930c8b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 17:53:09 +0100
Subject: [PATCH 047/137] Rename CoDeList variable to comment_details_list

---
 bot/check-build.sh | 18 +++++++++---------
 bot/check-test.sh  | 18 +++++++++---------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index 42dca46a08..584d4ff828 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -334,35 +334,35 @@ echo -n "comment_description = " >> ${job_result_file}
 
 comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
 
-# first construct comment_details_list, abbreviated CoDeList
+# first construct comment_details_list, abbreviated comment_details_list
 # then use it to set comment_details
-CoDeList=""
+comment_details_list=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
-CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_failed}</code>"
 failure_msg="found message matching <code>${GP_failed}</code>"
-CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_req_missing}</code>"
 failure_msg="found message matching <code>${GP_req_missing}</code>"
-CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="found message(s) matching <code>${GP_no_missing}</code>"
 failure_msg="no message matching <code>${GP_no_missing}</code>"
-CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="found message matching <code>${GP_tgz_created}</code>"
 failure_msg="no message matching <code>${GP_tgz_created}</code>"
-CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
 
-comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
 
 
 # first construct comment_artefacts_list, abbreviated CoArList
diff --git a/bot/check-test.sh b/bot/check-test.sh
index 9724f45832..fe1fcfe85d 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -176,37 +176,37 @@ function add_detail() {
     fi
 }
 
-# first construct comment_details_list, abbreviated CoDeList
+# first construct comment_details_list, abbreviated comment_details_list
 # then use it to set comment_details
-CoDeList=""
+comment_details_list=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
-CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>""${GP_failed}""</code>"
 failure_msg="found message matching <code>""${GP_failed}""</code>"
-CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
 # Should not be needed for testing, I think? Maybe for loading ReFrame module...
 # success_msg="no message matching <code>${GP_req_missing}</code>"
 # failure_msg="found message matching <code>${GP_req_missing}</code>"
-# CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+# comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
 # 
 # success_msg="found message(s) matching <code>${GP_no_missing}</code>"
 # failure_msg="no message matching <code>${GP_no_missing}</code>"
-# CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+# comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
 # 
 # success_msg="found message matching <code>${GP_tgz_created}</code>"
 # failure_msg="no message matching <code>${GP_tgz_created}</code>"
-# CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+# comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
 
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
-comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
 
 # Actually writing the comment description to the result file

From f80c0eb42f891c686cd1818ccbfc353e8bb29f05 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 17:54:00 +0100
Subject: [PATCH 048/137] Remove commented section, it is not relevant

---
 bot/check-test.sh | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index fe1fcfe85d..9e7a524607 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -192,19 +192,6 @@ success_msg="no message matching <code>""${GP_failed}""</code>"
 failure_msg="found message matching <code>""${GP_failed}""</code>"
 comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
-# Should not be needed for testing, I think? Maybe for loading ReFrame module...
-# success_msg="no message matching <code>${GP_req_missing}</code>"
-# failure_msg="found message matching <code>${GP_req_missing}</code>"
-# comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
-# 
-# success_msg="found message(s) matching <code>${GP_no_missing}</code>"
-# failure_msg="no message matching <code>${GP_no_missing}</code>"
-# comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
-# 
-# success_msg="found message matching <code>${GP_tgz_created}</code>"
-# failure_msg="no message matching <code>${GP_tgz_created}</code>"
-# comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
-
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}

From ff5baace136c589becaf42adab32b4a0eed7afc6 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com>
Date: Thu, 8 Feb 2024 17:57:36 +0100
Subject: [PATCH 049/137] Update run_tests.sh

Avoid a one-line script (if possible)

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 run_tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/run_tests.sh b/run_tests.sh
index bcca1ee417..5e9fcb543d 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -9,5 +9,5 @@ source ${base_dir}/init/eessi_defaults
 # TODO: in the future we may implement this as a light first check.
 
 # Run the test suite
-./run_in_compat_layer_env.sh ./clone_eessi_test_suite.sh
+./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 ./test_suite.sh

From 4adae1838edd1146d1a4a8c4322bffeb5e49db64 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 18:42:47 +0100
Subject: [PATCH 050/137] Fix mistake in replacing SLURM with
 SLURM_OUTPUT_FOUND

---
 bot/check-build.sh | 4 ++--
 bot/check-test.sh  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index 584d4ff828..1b46652049 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -95,7 +95,7 @@ job_dir=${PWD}
 
 [[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}"
 
-job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out"
+job_out="slurm-${SLURM_JOB_ID}.out"
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
 if  [[ -f ${job_out} ]]; then
     SLURM_OUTPUT_FOUND=1
@@ -169,7 +169,7 @@ fi
 [[ ${VERBOSE} -ne 0 ]] && echo "  NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 [[ ${VERBOSE} -ne 0 ]] && echo "  TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 
-job_result_file=_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.result
+job_result_file=_bot_job${SLURM_JOB_ID}.result
 
 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
    [[ ${ERROR} -eq 0 ]] && \
diff --git a/bot/check-test.sh b/bot/check-test.sh
index 9e7a524607..faf6e440fa 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -10,8 +10,8 @@
 # license: GPLv2
 #
 job_dir=${PWD}
-job_out="slurm-${SLURM_OUTPUT_FOUND_JOB_ID}.out"
-job_test_result_file="_bot_job${SLURM_OUTPUT_FOUND_JOB_ID}.test"
+job_out="slurm-${SLURM_JOB_ID}.out"
+job_test_result_file="_bot_job${SLURM__JOB_ID}.test"
 
 # Check that job output file is found
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"

From 9e6955f53e70d3848ca2199756150993c36212ec Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 18:52:44 +0100
Subject: [PATCH 051/137] Change name of the template for the ReFrame config
 file to make clear it is a template. Then, alter the sed commands so that
 they output to a new file: the actual config file

---
 reframe_config_bot.py => reframe_config_bot.py.tmpl | 0
 test_suite.sh                                       | 9 +++++----
 2 files changed, 5 insertions(+), 4 deletions(-)
 rename reframe_config_bot.py => reframe_config_bot.py.tmpl (100%)

diff --git a/reframe_config_bot.py b/reframe_config_bot.py.tmpl
similarity index 100%
rename from reframe_config_bot.py
rename to reframe_config_bot.py.tmpl
diff --git a/test_suite.sh b/test_suite.sh
index 99643c42b8..b3884c3b81 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -161,6 +161,7 @@ fi
 
 # Configure ReFrame
 export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py
+export RFM_CONFIG_FILE_TEMPLATE=$TOPDIR/reframe_config_bot.py.tmpl
 export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests
 export RFM_CHECK_SEARCH_RECURSIVE=1
 export RFM_PREFIX=$PWD/reframe_runs
@@ -190,10 +191,10 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
 else
     fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu."
 fi
-sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES
-sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
-sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
-sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES
+sed "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
+sed "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
+sed "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
+sed "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
 
 # Check we can run reframe
 reframe --version

From 08a7fd8eaa64dce298624e1d4f6054d2938a7fcc Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 18:55:20 +0100
Subject: [PATCH 052/137] Fix typo in variable

---
 bot/check-test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index faf6e440fa..ef52206346 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -11,7 +11,7 @@
 #
 job_dir=${PWD}
 job_out="slurm-${SLURM_JOB_ID}.out"
-job_test_result_file="_bot_job${SLURM__JOB_ID}.test"
+job_test_result_file="_bot_job${SLURM_JOB_ID}.test"
 
 # Check that job output file is found
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"

From 8b5bde5413553c03f58ce788ef03e78ae87f343b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 18:56:07 +0100
Subject: [PATCH 053/137] No longer needed

---
 clone_eessi_test_suite.sh | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100755 clone_eessi_test_suite.sh

diff --git a/clone_eessi_test_suite.sh b/clone_eessi_test_suite.sh
deleted file mode 100755
index f7684c8cfd..0000000000
--- a/clone_eessi_test_suite.sh
+++ /dev/null
@@ -1 +0,0 @@
-git clone https://github.com/EESSI/test-suite EESSI-test-suite

From 5d383eb8bc4c729cfa85d1aeae9ea10d82000c5c Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:03:29 +0100
Subject: [PATCH 054/137] Added header, rephrased some comments

---
 run_tests.sh | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

diff --git a/run_tests.sh b/run_tests.sh
index 5e9fcb543d..0c51a14807 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -1,13 +1,24 @@
 #!/bin/bash
+#
+# This script gets invoked by the bot/test.sh script to run within the EESSI container
+# Thus, this script defines all of the steps that should run for the tests.
+# Note that, unless we have good reason, we don't run test steps in the prefix environment:
+# users also typically don't run in the prefix environment, and we want to check if the
+# software works well in that specific setup.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Caspar van Leeuwe (@casparvl)
+#
+# license: GPLv2
+#
+
 base_dir=$(dirname $(realpath $0))
 source ${base_dir}/init/eessi_defaults
 
-# Note: for these tests, we _don't_ run in the compat layer env
-# These tests should mimic what users do, and they are typically not in a prefix environment
-
-# Run eb --sanity-check-only on changed easyconfigs
-# TODO: in the future we may implement this as a light first check.
+# Git clone has to be run in compat layer, to make the git command available
+./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 
 # Run the test suite
-./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 ./test_suite.sh

From cdf79712184e89a31afaaff3880fc7cf0387b2e5 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:13:11 +0100
Subject: [PATCH 055/137] Fix headers

---
 run_tests.sh  |  2 +-
 test_suite.sh | 13 +++++++++----
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/run_tests.sh b/run_tests.sh
index 0c51a14807..de7fd8c2e1 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -9,7 +9,7 @@
 # This script is part of the EESSI software layer, see
 # https://github.com/EESSI/software-layer.git
 #
-# author: Caspar van Leeuwe (@casparvl)
+# author: Caspar van Leeuwen (@casparvl)
 #
 # license: GPLv2
 #
diff --git a/test_suite.sh b/test_suite.sh
index b3884c3b81..ffe8955f1e 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -1,9 +1,14 @@
 #!/bin/bash
 #
-# Run sanity check for all requested modules (and built dependencies)?
-# get ec from diff
-# set up EasyBuild
-# run `eb --sanity-only ec`
+# This script creates a ReFrame config file from a template, in which CPU properties get replaced
+# based on where this script is run (typically: a build node). Then, it runs the EESSI test suite.
+#
+# This script is part of the EESSI software layer, see
+# https://github.com/EESSI/software-layer.git
+#
+# author: Caspar van Leeuwen (@casparvl)
+#
+# license: GPLv2
 
 display_help() {
   echo "usage: $0 [OPTIONS]"

From 79a9ad320aec47ac368ed96f72e0f59a0cfbe9f5 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:16:10 +0100
Subject: [PATCH 056/137] Load default ReFrame version

---
 test_suite.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index ffe8955f1e..1f6d8ae44d 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -129,11 +129,11 @@ else
     echo_green ">> MODULEPATH set up: ${MODULEPATH}"
 fi
 
-# TODO: this should not be hardcoded. Ideally, we put some logic in place to discover the newest version
-# of the ReFrame module available in the current environment
-module load ReFrame/4.3.3
+# Load the ReFrame module
+# Currently, we load the default version. Maybe we should somehow make this configurable in the future?
+module load ReFrame
 if [[ $? -eq 0 ]]; then
-    echo_green ">> Loaded ReFrame/4.3.3"
+    echo_green ">> Loaded ReFrame module"
 else
     fatal_error "Failed to load the ReFrame module"
 fi

From 2f280e4304b520de295fdb408218e3125b153853 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:21:24 +0100
Subject: [PATCH 057/137] Point to docs for ReFrame config

---
 test_suite.sh | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index 1f6d8ae44d..d2403b91d8 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -130,7 +130,7 @@ else
 fi
 
 # Load the ReFrame module
-# Currently, we load the default version. Maybe we should somehow make this configurable in the future?
+ Currently, we load the default version. Maybe we should somehow make this configurable in the future?
 module load ReFrame
 if [[ $? -eq 0 ]]; then
     echo_green ">> Loaded ReFrame module"
@@ -139,15 +139,16 @@ else
 fi
 
 # Check ReFrame came with the hpctestlib and we can import it
-python3 -c 'import hpctestlib.sciapps.gromacs'
+reframe_import="hpctestlib.sciapps.gromacs"
+python3 -c "import ${reframe_import}"
 if [[ $? -eq 0 ]]; then
-    echo_green "Succesfully found and imported hpctestlib.sciapps.gromas"
+    echo_green "Succesfully found and imported ${reframe_import}"
 else
-    fatal_error "Failed to load hpctestlib"
+    fatal_error "Failed to import ${reframe_import}"
 fi
 
-# Cloning should already be done by clone_eessi_test_suite.sh, which runs in compat layer to have 'git' available
-# git clone https://github.com/EESSI/test-suite EESSI-test-suite
+# Cloning should already be done in run_tests.sh before test_suite.sh is invoked
+# Check if that succeeded
 export TESTSUITEPREFIX=$PWD/EESSI-test-suite
 if [ -d $TESTSUITEPREFIX ]; then
     echo_green "Clone of the test suite $TESTSUITEPREFIX available, OK!"
@@ -157,14 +158,15 @@ fi
 export PYTHONPATH=$TESTSUITEPREFIX:$PYTHONPATH
 
 # Check that we can import from the testsuite
-python3 -c 'import eessi.testsuite'
+testsuite_import="eessi.testsuite"
+python3 -c "import ${testsuite_import}"
 if [[ $? -eq 0 ]]; then
-    echo_green "Succesfully found and imported eessi.testsuite"
+    echo_green "Succesfully found and imported ${testsuite_import}"
 else
-    fatal_error "Failed to import from eessi.testsuite in Python"
+    fatal_error "Failed to import ${testsuite_import}"
 fi
 
-# Configure ReFrame
+# Configure ReFrame, see https://www.eessi.io/docs/test-suite/installation-configuration
 export RFM_CONFIG_FILES=$TOPDIR/reframe_config_bot.py
 export RFM_CONFIG_FILE_TEMPLATE=$TOPDIR/reframe_config_bot.py.tmpl
 export RFM_CHECK_SEARCH_PATH=$TESTSUITEPREFIX/eessi/testsuite/tests

From e5d60b9c6bd2b3a39a32454a9ee6707320693c11 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:26:17 +0100
Subject: [PATCH 058/137] Fix silly mistake: I was overwriting the ReFrame
 config file with every sed call

---
 test_suite.sh | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index d2403b91d8..755102e6bc 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -198,17 +198,18 @@ if [[ "${cpuinfo}" =~ (Core\(s\) per socket:[^0-9]*([0-9]+)) ]]; then
 else
     fatal_error "Failed to get the number of cores per socket for the current test hardware with lscpu."
 fi
-sed "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
-sed "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
-sed "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
-sed "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILE_TEMPLATE > $RFM_CONFIG_FILES
+cp ${RFM_CONFIG_FILE_TEMPLATE} ${RFM_CONFIG_FILES}
+sed -i "s/__NUM_CPUS__/${cpu_count}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
+sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES
 
 # Check we can run reframe
 reframe --version
 if [[ $? -eq 0 ]]; then
-    echo_green "Succesfully ran reframe --version"
+    echo_green "Succesfully ran 'reframe --version'"
 else
-    fatal_error "Failed to run ReFrame --version"
+    fatal_error "Failed to run 'reframe --version'"
 fi
 
 # List the tests we want to run
@@ -224,7 +225,8 @@ fi
 # Run all tests
 echo "Running tests: reframe ${REFRAME_ARGS} --run"
 reframe ${REFRAME_ARGS} --run
-if [[ $? -eq 0 ]]; then
+reframe_exit_code=$?
+if [[ ${reframe_exit_code} -eq 0 ]]; then
     echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."
 else
     fatal_error "ReFrame runtime failed to run with command: reframe ${REFRAME_ARGS} --run."
@@ -233,4 +235,4 @@ fi
 echo ">> Cleaning up ${TMPDIR}..."
 rm -r ${TMPDIR}
 
-exit 0
+exit ${reframe_exit_code}

From b5c60ccfc483602eb6cebb9e1f580c9d99cd964b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 8 Feb 2024 19:28:32 +0100
Subject: [PATCH 059/137] See if @boegel is correct and if we don't actually
 need ANY software change to have the bot run...

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 4c73b5887a..82190071ab 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -15,4 +15,3 @@ easyconfigs:
       options:
         from-pr: 19573
   - scikit-learn-1.3.1-gfbf-2023a.eb
-  - patchelf-0.18.0-GCCcore-12.3.0.eb

From 7fc04d830d372abbebcf9ec3701e38d7b3fec4b8 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 11 Feb 2024 21:46:02 +0100
Subject: [PATCH 060/137] {2023.06}[foss/2022b] ImageMagick v7.1.0-53

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
index 9e92c79062..b8fddd02bb 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
@@ -1,3 +1,4 @@
 easyconfigs:
   - SciPy-bundle-2023.02-gfbf-2022b.eb
   - GDAL-3.6.2-foss-2022b.eb
+  - ImageMagick-7.1.0-53-GCCcore-12.2.0.eb

From 69d6d2e227c6f2ceb0ad39068d6fe4a99d7fd24e Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 14 Feb 2024 14:55:14 +0100
Subject: [PATCH 061/137] Add escape character to star, so that the regex
 pattern prints correctly on GitHub in the report

---
 bot/check-test.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index ef52206346..bc5ba84101 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -188,6 +188,8 @@ success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
 comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
+# Add an escape character to every *, for it to be printed correctly in the comment on GitHub
+GP_failed="${GP_failed//\*/\\*}"
 success_msg="no message matching <code>""${GP_failed}""</code>"
 failure_msg="found message matching <code>""${GP_failed}""</code>"
 comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")

From 8491f9f090f5a55441904cd1d3b4c7ba5dacab4f Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Thu, 15 Feb 2024 11:58:49 +0100
Subject: [PATCH 062/137] Always use LMOD_RC file, and make sure loading CUDA
 apps always works when building

---
 eessi_container.sh               |  6 ++++++
 init/eessi_environment_variables | 16 +++++++++-------
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/eessi_container.sh b/eessi_container.sh
index d6e9558202..143cbcc6c8 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -457,6 +457,12 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then
         mkdir -p ${EESSI_USR_LOCAL_CUDA}
         BIND_PATHS="${BIND_PATHS},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda"
         [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
+        if [[ "${NVIDIA_MODE}" == "install" ]] ; then
+            # We need to "trick" our LMOD_RC file to allow us to load CUDA modules even without a CUDA driver
+            # (this works because we build within a container and the LMOD_RC recognises that)
+            touch ${EESSI_TMPDIR}/libcuda.so
+            export SINGULARITY_CONTAINLIBS="${EESSI_TMPDIR}/libcuda.so"
+        fi
     fi
 fi
 
diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables
index af5222e7b9..e042e8575a 100644
--- a/init/eessi_environment_variables
+++ b/init/eessi_environment_variables
@@ -50,6 +50,15 @@ if [ -d $EESSI_PREFIX ]; then
 
         show_msg "Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory."
         export EESSI_SOFTWARE_PATH=$EESSI_PREFIX/software/$EESSI_OS_TYPE/$EESSI_SOFTWARE_SUBDIR
+
+        # Configure our LMOD_RC file
+        export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua"
+        if [ -f $LMOD_RC ]; then
+          show_msg "Found Lmod configuration file at $LMOD_RC"
+        else
+          error "Lmod configuration file not found at $LMOD_RC"
+        fi
+
         if [ ! -z $EESSI_BASIC_ENV ]; then
           show_msg "Only setting up basic environment, so we're done"
         elif [ -d $EESSI_SOFTWARE_PATH ]; then
@@ -76,13 +85,6 @@ if [ -d $EESSI_PREFIX ]; then
             false
           fi
 
-          export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua"
-          if [ -f $LMOD_RC ]; then
-            show_msg "Found Lmod configuration file at $LMOD_RC"
-          else
-            error "Lmod configuration file not found at $LMOD_RC"
-          fi
-
         else
           error "EESSI software layer at $EESSI_SOFTWARE_PATH not found!"
         fi

From 2dd048e96cef85b203b76a9ca330c5c72813dbbc Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 15 Feb 2024 15:44:15 +0100
Subject: [PATCH 063/137] Add the list(SCALES.keys()) feature (though
 technically we could limit to singlenode already here...). Also, make sure
 color is not used for printing the ReFrame results, to avoid color-characters
 making it into the slurm output file

---
 reframe_config_bot.py.tmpl | 4 +++-
 test_suite.sh              | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl
index ad66a72041..b2f7916e79 100644
--- a/reframe_config_bot.py.tmpl
+++ b/reframe_config_bot.py.tmpl
@@ -19,7 +19,9 @@ site_configuration = {
                     'scheduler': 'local',
                     'launcher': 'mpirun',
                     'environs': ['default'],
-                    'features': [FEATURES[CPU]],
+                    'features': [
+                        FEATURES[CPU]
+                    ] + list(SCALES.keys()),
                     'processor': {
                         'num_cpus': __NUM_CPUS__,
                         'num_sockets': __NUM_SOCKETS__,
diff --git a/test_suite.sh b/test_suite.sh
index 755102e6bc..f8c46ef764 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -213,7 +213,7 @@ else
 fi
 
 # List the tests we want to run
-export REFRAME_ARGS='--tag CI --tag 1_node'
+export REFRAME_ARGS='--tag CI --tag 1_node --nocolor'
 echo "Listing tests: reframe ${REFRAME_ARGS} --list"
 reframe ${REFRAME_ARGS} --list
 if [[ $? -eq 0 ]]; then

From 0492b9e6d836ef75126ca5e8f613c7ad7487dc2b Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 15 Feb 2024 15:45:25 +0100
Subject: [PATCH 064/137] Try the original regex patterns, capturing spaces
 more specifically, now that we got rid of color characters

---
 bot/check-test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index bc5ba84101..4b5f7575e5 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -42,7 +42,7 @@ fi
 # Specifically, we grep for FAILED, since this is also what we print if a step in the test script itself fails
 FAILED=-1
 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
-  GP_failed='\[.*FAILED.*\].*Ran .* test case'
+  GP_failed='\[\s*FAILED\s*\].*Ran .* test case'
   grep_reframe_failed=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
   # have to be careful to not add searched for pattern into slurm out file
@@ -65,7 +65,7 @@ fi
 SUCCESS=-1
 # Grep for the success pattern, so we can report the amount of tests run
 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
-  GP_success='\[.*PASSED.*\].*Ran .* test case'
+  GP_success='\[\s*PASSED\s*\].*Ran .* test case'
   grep_reframe_success=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_success}")
   [[ $? -eq 0 ]] && SUCCESS=1 || SUCCESS=0
   # have to be careful to not add searched for pattern into slurm out file

From 0f6dc766c4770e160f193aac35be546094dc7c73 Mon Sep 17 00:00:00 2001
From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com>
Date: Fri, 16 Feb 2024 20:37:53 +0100
Subject: [PATCH 065/137] Update eessi-2023.06-eb-4.9.0-2022b.yml

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
index 9e92c79062..903364a289 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
@@ -1,3 +1,6 @@
 easyconfigs:
   - SciPy-bundle-2023.02-gfbf-2022b.eb
   - GDAL-3.6.2-foss-2022b.eb
+  - waLBerla-6.1-foss-2022b.eb 19324
+      options:
+        from-pr: 19252

From 3bac7114115028c6d08b808d4201da010671c854 Mon Sep 17 00:00:00 2001
From: Pedro Santos Neves <10762799+Neves-P@users.noreply.github.com>
Date: Fri, 16 Feb 2024 20:41:27 +0100
Subject: [PATCH 066/137] Add waLBerla v6.1 foss2022b

Fix easybuild PR number
---
 .../2023.06/eessi-2023.06-eb-4.9.0-2022b.yml                  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
index 903364a289..3bfb3588d8 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
@@ -1,6 +1,6 @@
 easyconfigs:
   - SciPy-bundle-2023.02-gfbf-2022b.eb
   - GDAL-3.6.2-foss-2022b.eb
-  - waLBerla-6.1-foss-2022b.eb 19324
+  - waLBerla-6.1-foss-2022b.eb
       options:
-        from-pr: 19252
+        from-pr: 19324

From d6e4cecd1b791633443073e6d357ae35e9f26cc6 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Sat, 17 Feb 2024 09:42:30 +0100
Subject: [PATCH 067/137] add missing ':' for waLBerla entry

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
index 3bfb3588d8..2a07133770 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
@@ -1,6 +1,6 @@
 easyconfigs:
   - SciPy-bundle-2023.02-gfbf-2022b.eb
   - GDAL-3.6.2-foss-2022b.eb
-  - waLBerla-6.1-foss-2022b.eb
+  - waLBerla-6.1-foss-2022b.eb:
       options:
         from-pr: 19324

From ec61759dc7b8a2a39613aff55f6ae74a55dd2620 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Mon, 19 Feb 2024 12:11:54 +0100
Subject: [PATCH 068/137] Remove manual setup of MODULEPATH etc, and source the
 regular EESSI init script to do this

---
 test_suite.sh | 68 +++++++++++----------------------------------------
 1 file changed, 14 insertions(+), 54 deletions(-)

diff --git a/test_suite.sh b/test_suite.sh
index f8c46ef764..8a98fd59a8 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -74,63 +74,14 @@ fi
 TMPDIR=$(mktemp -d)
 
 echo ">> Setting up environment..."
-
-source $TOPDIR/init/minimal_eessi_env
-
-if [ -d $EESSI_CVMFS_REPO ]; then
-    echo_green "$EESSI_CVMFS_REPO available, OK!"
-else
-    fatal_error "$EESSI_CVMFS_REPO is not available!"
-fi
-
-# avoid that pyc files for EasyBuild are stored in EasyBuild installation directory
-export PYTHONPYCACHEPREFIX=$TMPDIR/pycache
-
-echo ">> Determining software subdirectory to use for current build/test host..."
-if [ -z $EESSI_SOFTWARE_SUBDIR_OVERRIDE ]; then
-  export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
-  echo ">> Determined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE via 'eessi_software_subdir.py $DETECTION_PARAMETERS' script"
-else
-  echo ">> Picking up pre-defined \$EESSI_SOFTWARE_SUBDIR_OVERRIDE: ${EESSI_SOFTWARE_SUBDIR_OVERRIDE}"
-fi
-
-# Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE)
-# $EESSI_SILENT - don't print any messages
-# $EESSI_BASIC_ENV - give a basic set of environment variables
-EESSI_SILENT=1 EESSI_BASIC_ENV=1 source $TOPDIR/init/eessi_environment_variables
-
-if [[ -z ${EESSI_SOFTWARE_SUBDIR} ]]; then
-    fatal_error "Failed to determine software subdirectory?!"
-elif [[ "${EESSI_SOFTWARE_SUBDIR}" != "${EESSI_SOFTWARE_SUBDIR_OVERRIDE}" ]]; then
-    fatal_error "Values for EESSI_SOFTWARE_SUBDIR_OVERRIDE (${EESSI_SOFTWARE_SUBDIR_OVERRIDE}) and EESSI_SOFTWARE_SUBDIR (${EESSI_SOFTWARE_SUBDIR}) differ!"
-else
-    echo_green ">> Using ${EESSI_SOFTWARE_SUBDIR} as software subdirectory!"
-fi
-
-echo ">> Initializing Lmod..."
-source $EPREFIX/usr/share/Lmod/init/bash
-ml_version_out=$TMPDIR/ml.out
-ml --version &> $ml_version_out
-if [[ $? -eq 0 ]]; then
-    echo_green ">> Found Lmod ${LMOD_VERSION}"
-else
-    fatal_error "Failed to initialize Lmod?! (see output in ${ml_version_out}"
-fi
-
-echo ">> Setting up \$MODULEPATH..."
-# make sure no modules are loaded
 module --force purge
-# ignore current $MODULEPATH entirely
-module unuse $MODULEPATH
-module use ${EESSI_SOFTWARE_PATH}/modules/all
-if [[ -z ${MODULEPATH} ]]; then
-    fatal_error "Failed to set up \$MODULEPATH?!"
-else
-    echo_green ">> MODULEPATH set up: ${MODULEPATH}"
-fi
+# Make sure defaults are set for EESSI_CVMFS_REPO and EESSI_VERSION, before initializing EESSI
+source $TOPDIR/init/eessi_defaults
+# Initialize EESSI
+source ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/bash
 
 # Load the ReFrame module
- Currently, we load the default version. Maybe we should somehow make this configurable in the future?
+# Currently, we load the default version. Maybe we should somehow make this configurable in the future?
 module load ReFrame
 if [[ $? -eq 0 ]]; then
     echo_green ">> Loaded ReFrame module"
@@ -138,6 +89,15 @@ else
     fatal_error "Failed to load the ReFrame module"
 fi
 
+# Check that a system python3 is available
+python3_found=$(which python3)
+if [ -z ${python3_found} ]; then
+    fatal_error "No system python3 found"
+else
+    echo_green "System python3 found:"
+    python3 -V
+fi
+
 # Check ReFrame came with the hpctestlib and we can import it
 reframe_import="hpctestlib.sciapps.gromacs"
 python3 -c "import ${reframe_import}"

From f3f9721dac3f590768cb9dcb6fbcf26535f049d8 Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Tue, 20 Feb 2024 15:12:53 +0100
Subject: [PATCH 069/137] {2023.06}[foss/2023b] WSClean 3.4 + DP3 6.0

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml  | 30 +++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 4dd31dbd5d..fea9107dc0 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -8,3 +8,33 @@ easyconfigs:
   - matplotlib-3.8.2-gfbf-2023b.eb:
       options:
         from-pr: 19552
+  - AOFlagger-3.4.0-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        # Exclude Lua from `filter-deps` as the compat layer version is too old for the software
+        filter-deps:
+          - Autoconf
+          - Automake
+          - Autotools
+          - binutils
+          - bzip2
+          - DBus
+          - flex
+          - gettext
+          - gperf
+          - help2man
+          - intltool
+          - libreadline
+          - libtool
+          - ncurses
+          - M4
+          - makeinfo
+          - util-linux
+          - XZ
+          - zlib
+  - DP3-6.0-foss-2023b.eb:
+      options:
+        from-pr: 19840
+  - WSClean-3.4-foss-2023b.eb:
+      options:
+        from-pr: 19840

From f8cd2aa673187d6c1b346d53658932368118bd2a Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Tue, 20 Feb 2024 15:41:51 +0100
Subject: [PATCH 070/137] Remove AOFlagger (Lua is no longer part of
 filter-deps)

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml  | 24 -------------------
 1 file changed, 24 deletions(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index fea9107dc0..bf7e34932b 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -8,30 +8,6 @@ easyconfigs:
   - matplotlib-3.8.2-gfbf-2023b.eb:
       options:
         from-pr: 19552
-  - AOFlagger-3.4.0-foss-2023b.eb:
-      options:
-        from-pr: 19840
-        # Exclude Lua from `filter-deps` as the compat layer version is too old for the software
-        filter-deps:
-          - Autoconf
-          - Automake
-          - Autotools
-          - binutils
-          - bzip2
-          - DBus
-          - flex
-          - gettext
-          - gperf
-          - help2man
-          - intltool
-          - libreadline
-          - libtool
-          - ncurses
-          - M4
-          - makeinfo
-          - util-linux
-          - XZ
-          - zlib
   - DP3-6.0-foss-2023b.eb:
       options:
         from-pr: 19840

From eafbb6b757320ee003d678989c05bba6eb28ce2f Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Tue, 20 Feb 2024 17:29:55 +0100
Subject: [PATCH 071/137] Include easyblocks from pr 3088 to fix Python path

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml   | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index bf7e34932b..e8496f371a 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -8,9 +8,27 @@ easyconfigs:
   - matplotlib-3.8.2-gfbf-2023b.eb:
       options:
         from-pr: 19552
+  - AOFlagger-3.4.0-foss-2023b.eb:
+      options:
+        from:pr: 19840
+        include-easyblocks-from-pr: 3088
+  - casacore-3.5.0-foss-2023b.eb:
+      options:
+        from:pr: 19840
+        include-easyblocks-from-pr: 3088
   - DP3-6.0-foss-2023b.eb:
       options:
         from-pr: 19840
+        include-easyblocks-from-pr: 3088
+  - IDG-1.2.0-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
+  - EveryBeam-0.5.2-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
   - WSClean-3.4-foss-2023b.eb:
       options:
         from-pr: 19840
+        include-easyblocks-from-pr: 3088

From df089de7c137ecb3232eca5f4266ca7d7cf577ed Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Tue, 20 Feb 2024 17:42:44 +0100
Subject: [PATCH 072/137] Fix typo

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml                  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index e8496f371a..c874d7753f 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -10,11 +10,11 @@ easyconfigs:
         from-pr: 19552
   - AOFlagger-3.4.0-foss-2023b.eb:
       options:
-        from:pr: 19840
+        from-pr: 19840
         include-easyblocks-from-pr: 3088
   - casacore-3.5.0-foss-2023b.eb:
       options:
-        from:pr: 19840
+        from-pr: 19840
         include-easyblocks-from-pr: 3088
   - DP3-6.0-foss-2023b.eb:
       options:

From 2ec41e17b8e6701323e2a1a2070bc7b82c417980 Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Wed, 21 Feb 2024 07:49:52 +0100
Subject: [PATCH 073/137] List dependencies first

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml                | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index c874d7753f..d89bf6b1b0 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -16,15 +16,15 @@ easyconfigs:
       options:
         from-pr: 19840
         include-easyblocks-from-pr: 3088
-  - DP3-6.0-foss-2023b.eb:
+  - IDG-1.2.e0-foss-2023b.eb:
       options:
         from-pr: 19840
         include-easyblocks-from-pr: 3088
-  - IDG-1.2.0-foss-2023b.eb:
+  - EveryBeam-0.5.2-foss-2023b.eb:
       options:
         from-pr: 19840
         include-easyblocks-from-pr: 3088
-  - EveryBeam-0.5.2-foss-2023b.eb:
+  - DP3-6.0-foss-2023b.eb:
       options:
         from-pr: 19840
         include-easyblocks-from-pr: 3088

From d0ba0b724fb8ac8a4516d40731023e31fb190b02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Wed, 21 Feb 2024 09:42:52 +0100
Subject: [PATCH 074/137] fix typo in IDG version

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index d89bf6b1b0..33acff2e23 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -16,7 +16,7 @@ easyconfigs:
       options:
         from-pr: 19840
         include-easyblocks-from-pr: 3088
-  - IDG-1.2.e0-foss-2023b.eb:
+  - IDG-1.2.0-foss-2023b.eb:
       options:
         from-pr: 19840
         include-easyblocks-from-pr: 3088

From 5d797a93f21bf7e898898af0e15bc063849c0bd8 Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Wed, 21 Feb 2024 11:24:06 +0100
Subject: [PATCH 075/137] Include Armadillo-12.8.0-foss-2023b.eb

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml                  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 33acff2e23..7d5ce34a83 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -12,6 +12,10 @@ easyconfigs:
       options:
         from-pr: 19840
         include-easyblocks-from-pr: 3088
+  - Armadillo-12.8.0-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
   - casacore-3.5.0-foss-2023b.eb:
       options:
         from-pr: 19840

From a55826ddc0f4175e6fb94b7eb355a263d7a4b759 Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Wed, 21 Feb 2024 11:24:41 +0100
Subject: [PATCH 076/137] Revert "Include Armadillo-12.8.0-foss-2023b.eb"

This reverts commit b72730eecd5265829b58e8b09c7476d460bb011a.
---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml                  | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 7d5ce34a83..33acff2e23 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -12,10 +12,6 @@ easyconfigs:
       options:
         from-pr: 19840
         include-easyblocks-from-pr: 3088
-  - Armadillo-12.8.0-foss-2023b.eb:
-      options:
-        from-pr: 19840
-        include-easyblocks-from-pr: 3088
   - casacore-3.5.0-foss-2023b.eb:
       options:
         from-pr: 19840

From f63a4d83c0f307f7a7928f8deaab28f09f3ecf21 Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Wed, 21 Feb 2024 11:24:06 +0100
Subject: [PATCH 077/137] Include Armadillo-12.8.0-foss-2023b.eb

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml                  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 33acff2e23..7d5ce34a83 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -12,6 +12,10 @@ easyconfigs:
       options:
         from-pr: 19840
         include-easyblocks-from-pr: 3088
+  - Armadillo-12.8.0-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
   - casacore-3.5.0-foss-2023b.eb:
       options:
         from-pr: 19840

From 7a0acebeb3826ba3bbb6172da344d906c60f2a1c Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Wed, 21 Feb 2024 15:35:54 +0100
Subject: [PATCH 078/137] Include arpack-ng-3.9.0-foss-2023b.eb

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml                  | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 7d5ce34a83..07c2af56a7 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -12,6 +12,10 @@ easyconfigs:
       options:
         from-pr: 19840
         include-easyblocks-from-pr: 3088
+  - arpack-ng-3.9.0-foss-2023b.eb:
+      options:
+        from-pr: 19840
+        include-easyblocks-from-pr: 3088
   - Armadillo-12.8.0-foss-2023b.eb:
       options:
         from-pr: 19840

From 5c546ea8bff9692f4f6fbd68c0951bc29ff78f05 Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Thu, 22 Feb 2024 18:05:46 +0100
Subject: [PATCH 079/137] Disable 'vectorize' toolchain option for casacore
 3.5.0 on aarch64/neoverse_v1 with eb_hook

---
 eb_hooks.py | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index d29a837339..42f67278a8 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -21,7 +21,7 @@
 
 
 CPU_TARGET_NEOVERSE_V1 = 'aarch64/neoverse_v1'
-CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic' 
+CPU_TARGET_AARCH64_GENERIC = 'aarch64/generic'
 
 EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs'
 
@@ -159,6 +159,28 @@ def post_prepare_hook(self, *args, **kwargs):
     if self.name in POST_PREPARE_HOOKS:
         POST_PREPARE_HOOKS[self.name](self, *args, **kwargs)
 
+def parse_hook_casacore_disable_vectorize(ec, eprefix):
+    """
+    Disable 'vectorize' toolchain option for casacore 3.5.0 on aarch64/neoverse_v1
+    Compiling casacore 3.5.0 with GCC 13.2.0 (foss-2023) gives an error when building for aarch64/neoverse_v1.
+    See also, https://github.com/EESSI/software-layer/pull/479
+    """
+    if ec.name == 'casacore':
+        tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version']
+        if (
+            LooseVersion(ec.version) == LooseVersion('3.5.0') and
+            tcname == 'foss' and tcversion == '2023b'
+        ):
+            if get_cpu_architecture() == CPU_TARGET_NEOVERSE_V1:
+                ec['toolchainopts']['vectorize'] = False
+                print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts'])
+            else:
+                print_msg("Not changing option vectorize for %s on non-AARCH64", ec.name)
+        else:
+            print_msg("Not changing option vectorize for %s %s %s", ec.name, ec.version, ec.toolchain)
+
+    else:
+        raise EasyBuildError("casacore-specific hook triggered for non-casacore easyconfig?!")
 
 def parse_hook_cgal_toolchainopts_precise(ec, eprefix):
     """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER."""
@@ -327,7 +349,7 @@ def pre_configure_hook_wrf_aarch64(self, *args, **kwargs):
             if LooseVersion(self.version) <= LooseVersion('3.9.0'):
                     self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure_new.defaults && " % (pattern, repl))
                     print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts'])
-                    
+
             if LooseVersion('4.0.0') <= LooseVersion(self.version) <= LooseVersion('4.2.1'):
                     self.cfg.update('preconfigopts', "sed -i 's/%s/%s/g' arch/configure.defaults && " % (pattern, repl))
                     print_msg("Using custom preconfigopts for %s: %s", self.name, self.cfg['preconfigopts'])
@@ -414,7 +436,7 @@ def pre_test_hook_ignore_failing_tests_netCDF(self, *args, **kwargs):
     """
     cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
     if self.name == 'netCDF' and self.version == '4.9.2' and cpu_target == CPU_TARGET_NEOVERSE_V1:
-        self.cfg['testopts'] = "|| echo ignoring failing tests" 
+        self.cfg['testopts'] = "|| echo ignoring failing tests"
 
 def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs):
     """
@@ -579,6 +601,7 @@ def inject_gpu_property(ec):
 
 
 PARSE_HOOKS = {
+    'casacore': parse_hook_casacore_disable_vectorize,
     'CGAL': parse_hook_cgal_toolchainopts_precise,
     'fontconfig': parse_hook_fontconfig_add_fonts,
     'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors,

From 5b822a3e2292e80077956b0a3f7c9e453a1d25cd Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Thu, 22 Feb 2024 18:09:02 +0100
Subject: [PATCH 080/137] Remove empty line

---
 eb_hooks.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 42f67278a8..2cc9f63ad4 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -178,7 +178,6 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix):
                 print_msg("Not changing option vectorize for %s on non-AARCH64", ec.name)
         else:
             print_msg("Not changing option vectorize for %s %s %s", ec.name, ec.version, ec.toolchain)
-
     else:
         raise EasyBuildError("casacore-specific hook triggered for non-casacore easyconfig?!")
 

From 889243779f41ee43c68079c2bd98133857d75e98 Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Thu, 22 Feb 2024 18:11:03 +0100
Subject: [PATCH 081/137] Cleanup code

---
 eb_hooks.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 2cc9f63ad4..448d6fe62c 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -159,10 +159,11 @@ def post_prepare_hook(self, *args, **kwargs):
     if self.name in POST_PREPARE_HOOKS:
         POST_PREPARE_HOOKS[self.name](self, *args, **kwargs)
 
+
 def parse_hook_casacore_disable_vectorize(ec, eprefix):
     """
     Disable 'vectorize' toolchain option for casacore 3.5.0 on aarch64/neoverse_v1
-    Compiling casacore 3.5.0 with GCC 13.2.0 (foss-2023) gives an error when building for aarch64/neoverse_v1.
+    Compiling casacore 3.5.0 with GCC 13.2.0 (foss-2023b) gives an error when building for aarch64/neoverse_v1.
     See also, https://github.com/EESSI/software-layer/pull/479
     """
     if ec.name == 'casacore':
@@ -181,6 +182,7 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix):
     else:
         raise EasyBuildError("casacore-specific hook triggered for non-casacore easyconfig?!")
 
+
 def parse_hook_cgal_toolchainopts_precise(ec, eprefix):
     """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER."""
     if ec.name == 'CGAL':

From 3dcf5fae07e154858eec0f7ff866a27d00268e2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Thu, 22 Feb 2024 22:38:15 +0100
Subject: [PATCH 082/137] add WRF-4.4.1-foss-2022b-dmpar.eb

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
index 2a07133770..ac50c77b67 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2022b.yml
@@ -4,3 +4,4 @@ easyconfigs:
   - waLBerla-6.1-foss-2022b.eb:
       options:
         from-pr: 19324
+  - WRF-4.4.1-foss-2022b-dmpar.eb

From 4e5f2db05384b43af4ebbe4bca3b787660b21875 Mon Sep 17 00:00:00 2001
From: t1mk1k <96469032+t1mk1k@users.noreply.github.com>
Date: Fri, 23 Feb 2024 15:26:14 +0100
Subject: [PATCH 083/137] Retrieve cpu_target from EESSI env vars
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Bob Dröge <b.e.droge@rug.nl>
---
 eb_hooks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 448d6fe62c..747ca5f601 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -172,7 +172,8 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix):
             LooseVersion(ec.version) == LooseVersion('3.5.0') and
             tcname == 'foss' and tcversion == '2023b'
         ):
-            if get_cpu_architecture() == CPU_TARGET_NEOVERSE_V1:
+            cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
+            if cpu_target == CPU_TARGET_NEOVERSE_V1:
                 ec['toolchainopts']['vectorize'] = False
                 print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts'])
             else:

From 354407375cc65087c5850f162077b7fa22f785f2 Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Fri, 23 Feb 2024 15:30:27 +0100
Subject: [PATCH 084/137] Specify print message for skipping non-neoverse_v1

---
 eb_hooks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 747ca5f601..15813ec354 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -177,7 +177,7 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix):
                 ec['toolchainopts']['vectorize'] = False
                 print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts'])
             else:
-                print_msg("Not changing option vectorize for %s on non-AARCH64", ec.name)
+                print_msg("Not changing option vectorize for %s on non-neoverse_v1", ec.name)
         else:
             print_msg("Not changing option vectorize for %s %s %s", ec.name, ec.version, ec.toolchain)
     else:

From 0aea4b44691d237d9359e0a35991b3874b2a092b Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Mon, 26 Feb 2024 19:33:34 +0100
Subject: [PATCH 085/137] Create toolchainopts dict if it does not exist

---
 eb_hooks.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/eb_hooks.py b/eb_hooks.py
index 15813ec354..d93ee37067 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -174,6 +174,8 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix):
         ):
             cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
             if cpu_target == CPU_TARGET_NEOVERSE_V1:
+                if not hasattr(ec, 'toolchainopts'):
+                    ec['toolchainopts'] = {}
                 ec['toolchainopts']['vectorize'] = False
                 print_msg("Changed toochainopts for %s: %s", ec.name, ec['toolchainopts'])
             else:

From 07296d3589587cf1090e574513185ad6e50b3d69 Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Tue, 27 Feb 2024 11:17:33 +0100
Subject: [PATCH 086/137] Enable lax vector conversion for DP3

---
 eb_hooks.py | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/eb_hooks.py b/eb_hooks.py
index d93ee37067..1478f1a508 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -186,6 +186,32 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix):
         raise EasyBuildError("casacore-specific hook triggered for non-casacore easyconfig?!")
 
 
+def parse_hook_dp3_enable_relaxed_vector_conversions(ec, eprefix):
+    """
+    Enable lax vector conversion for DP3 on aarch64/neoverse_v1
+    Compiling DP3 6.0 with GCC 13.2.0 (foss-2023b) gives a conversion error when building for aarch64/neoverse_v1.
+    See also, https://github.com/EESSI/software-layer/pull/479
+    """
+    if ec.name == 'DP3':
+        tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version']
+        if (
+            LooseVersion(ec.version) == LooseVersion('6.0') and
+            tcname == 'foss' and tcversion == '2023b'
+        ):
+            cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
+            if cpu_target == CPU_TARGET_NEOVERSE_V1:
+                if not hasattr(ec, 'configopts'):
+                    ec['configopts'] = ""
+                ec['configopts'] += ' -DCMAKE_CXX_FLAGS="$CXXFLAGS -flax-vector-conversions" '
+                print_msg("Changed configopts for %s: %s", ec.name, ec['configopts'])
+            else:
+                print_msg("Not changing configopts for %s on non-neoverse_v1", ec.name)
+        else:
+            print_msg("Not changing configopts for %s %s %s", ec.name, ec.version, ec.toolchain)
+    else:
+        raise EasyBuildError("DP3-specific hook triggered for non-DP3 easyconfig?!")
+
+
 def parse_hook_cgal_toolchainopts_precise(ec, eprefix):
     """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER."""
     if ec.name == 'CGAL':
@@ -607,6 +633,7 @@ def inject_gpu_property(ec):
 PARSE_HOOKS = {
     'casacore': parse_hook_casacore_disable_vectorize,
     'CGAL': parse_hook_cgal_toolchainopts_precise,
+    'DP3': parse_hook_dp3_enable_relaxed_vector_conversions,
     'fontconfig': parse_hook_fontconfig_add_fonts,
     'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors,
     'pybind11': parse_hook_pybind11_replace_catch2,

From 793e9f8ec925aa09b020f72d4acffa1ce0b400eb Mon Sep 17 00:00:00 2001
From: Richard Top <richard.topouchian@uib.no>
Date: Wed, 28 Feb 2024 13:20:54 +0000
Subject: [PATCH 087/137] {2023.06}[foss/2023a] dask v2023.9.2

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml       |  1 +
 eb_hooks.py                                        | 14 +++++++++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 157a47a49e..d444b1f8ae 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -29,3 +29,4 @@ easyconfigs:
       # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554
       options:
         from-pr: 19554
+  - dask-2023.9.2-foss-2023a.eb
diff --git a/eb_hooks.py b/eb_hooks.py
index d29a837339..4ed72a12ac 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -204,6 +204,18 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix):
         raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!")
 
 
+def parse_hook_Pillow_SIMD_harcoded_paths(ec, eprefix):
+    # patch setup.py to prefix hardcoded /usr/* and /lib paths with value of %(sysroot) template
+    # (which will be empty if EasyBuild is not configured to use an alternate sysroot);
+    # see also https://gitlab.com/eessi/support/-/issues/9
+    if ec.name == 'Pillow-SIMD':
+        ec.update('preinstallopts', """sed -i 's@"/usr/@"%(sysroot)s/usr/@g' setup.py && """)
+        ec.update('preinstallopts', """sed -i 's@"/lib@"%(sysroot)s/lib@g' setup.py && """)
+        print_msg("Using custom configure options for %s: %s", ec.name, ec['preinstallopts'])
+    else:
+        raise EasyBuildError("Pillow-SIMD-specific hook triggered for non-Pillow-SIMD easyconfig?!")
+
+
 def parse_hook_pybind11_replace_catch2(ec, eprefix):
     """
     Replace Catch2 build dependency in pybind11 easyconfigs with one that doesn't use system toolchain.
@@ -237,7 +249,6 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix):
 
 
 def parse_hook_ucx_eprefix(ec, eprefix):
-    """Make UCX aware of compatibility layer via additional configuration options."""
     if ec.name == 'UCX':
         ec.update('configopts', '--with-sysroot=%s' % eprefix)
         ec.update('configopts', '--with-rdmacm=%s' % os.path.join(eprefix, 'usr'))
@@ -582,6 +593,7 @@ def inject_gpu_property(ec):
     'CGAL': parse_hook_cgal_toolchainopts_precise,
     'fontconfig': parse_hook_fontconfig_add_fonts,
     'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors,
+    'Pillow-SIMD' : parse_hook_Pillow_SIMD_harcoded_paths,
     'pybind11': parse_hook_pybind11_replace_catch2,
     'Qt5': parse_hook_qt5_check_qtwebengine_disable,
     'UCX': parse_hook_ucx_eprefix,

From 68a766fe8637c7c90b7c667aa0f738b8807f115f Mon Sep 17 00:00:00 2001
From: Richard Top <richard.topouchian@uib.no>
Date: Wed, 28 Feb 2024 13:27:38 +0000
Subject: [PATCH 088/137] fixed info

---
 eb_hooks.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/eb_hooks.py b/eb_hooks.py
index 4ed72a12ac..d30700267b 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -205,6 +205,7 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix):
 
 
 def parse_hook_Pillow_SIMD_harcoded_paths(ec, eprefix):
+    """Make Pillow-SIMD aware of sysroot."""
     # patch setup.py to prefix hardcoded /usr/* and /lib paths with value of %(sysroot) template
     # (which will be empty if EasyBuild is not configured to use an alternate sysroot);
     # see also https://gitlab.com/eessi/support/-/issues/9
@@ -249,6 +250,7 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix):
 
 
 def parse_hook_ucx_eprefix(ec, eprefix):
+    """Make UCX aware of compatibility layer via additional configuration options."""
     if ec.name == 'UCX':
         ec.update('configopts', '--with-sysroot=%s' % eprefix)
         ec.update('configopts', '--with-rdmacm=%s' % os.path.join(eprefix, 'usr'))

From 3cdf80390fa3150bcde82427537776c22bf7bb25 Mon Sep 17 00:00:00 2001
From: Richard Top <richard.topouchian@uib.no>
Date: Wed, 28 Feb 2024 17:33:43 +0000
Subject: [PATCH 089/137] removed the hook and added --from-pr

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml       |  5 ++++-
 eb_hooks.py                                        | 14 --------------
 2 files changed, 4 insertions(+), 15 deletions(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index d444b1f8ae..983d59cc04 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -29,4 +29,7 @@ easyconfigs:
       # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554
       options:
         from-pr: 19554
-  - dask-2023.9.2-foss-2023a.eb
+  - dask-2023.9.2-foss-2023a.eb:
+      options:
+        from-pr: 19996
+ 
diff --git a/eb_hooks.py b/eb_hooks.py
index d30700267b..d29a837339 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -204,19 +204,6 @@ def parse_hook_openblas_relax_lapack_tests_num_errors(ec, eprefix):
         raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!")
 
 
-def parse_hook_Pillow_SIMD_harcoded_paths(ec, eprefix):
-    """Make Pillow-SIMD aware of sysroot."""
-    # patch setup.py to prefix hardcoded /usr/* and /lib paths with value of %(sysroot) template
-    # (which will be empty if EasyBuild is not configured to use an alternate sysroot);
-    # see also https://gitlab.com/eessi/support/-/issues/9
-    if ec.name == 'Pillow-SIMD':
-        ec.update('preinstallopts', """sed -i 's@"/usr/@"%(sysroot)s/usr/@g' setup.py && """)
-        ec.update('preinstallopts', """sed -i 's@"/lib@"%(sysroot)s/lib@g' setup.py && """)
-        print_msg("Using custom configure options for %s: %s", ec.name, ec['preinstallopts'])
-    else:
-        raise EasyBuildError("Pillow-SIMD-specific hook triggered for non-Pillow-SIMD easyconfig?!")
-
-
 def parse_hook_pybind11_replace_catch2(ec, eprefix):
     """
     Replace Catch2 build dependency in pybind11 easyconfigs with one that doesn't use system toolchain.
@@ -595,7 +582,6 @@ def inject_gpu_property(ec):
     'CGAL': parse_hook_cgal_toolchainopts_precise,
     'fontconfig': parse_hook_fontconfig_add_fonts,
     'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors,
-    'Pillow-SIMD' : parse_hook_Pillow_SIMD_harcoded_paths,
     'pybind11': parse_hook_pybind11_replace_catch2,
     'Qt5': parse_hook_qt5_check_qtwebengine_disable,
     'UCX': parse_hook_ucx_eprefix,

From 117081843c09ae69e9b25eb18ecc54f6736cc16f Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Thu, 29 Feb 2024 10:01:17 +0100
Subject: [PATCH 090/137] Temporary fix for CI by pinning archspec version

---
 .github/workflows/tests.yml      | 2 +-
 .github/workflows/tests_init.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 8e74a4e844..faa7eb82ff 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -22,7 +22,7 @@ jobs:
 
     - name: install Python packages
       run: |
-        pip install archspec
+        pip install archspec==0.2.2
 
     - name: test eessi_software_subdir.py script
       run: |
diff --git a/.github/workflows/tests_init.yml b/.github/workflows/tests_init.yml
index 38ccbbad31..053acb9730 100644
--- a/.github/workflows/tests_init.yml
+++ b/.github/workflows/tests_init.yml
@@ -22,7 +22,7 @@ jobs:
 
     - name: install Python packages
       run: |
-        pip install archspec pytest
+        pip install archspec==0.2.2 pytest
 
     - name: unit tests for eessi_software_subdir_for_host.py script
       run:

From cf7705643865f85b0e73f5185952a400f41c95f6 Mon Sep 17 00:00:00 2001
From: TopRichard <121792457+TopRichard@users.noreply.github.com>
Date: Thu, 29 Feb 2024 13:39:14 +0100
Subject: [PATCH 091/137] remove-blanks

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml   | 1 -
 1 file changed, 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 983d59cc04..7d9c331fcf 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -32,4 +32,3 @@ easyconfigs:
   - dask-2023.9.2-foss-2023a.eb:
       options:
         from-pr: 19996
- 

From 887e102631ef5fbfadffef744c7724e28a01217f Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Thu, 29 Feb 2024 13:50:41 +0100
Subject: [PATCH 092/137] Update
 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 7d9c331fcf..d187d8e92c 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -29,6 +29,7 @@ easyconfigs:
       # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19554
       options:
         from-pr: 19554
-  - dask-2023.9.2-foss-2023a.eb:
+  - Pillow-SIMD-9.5.0-GCCcore-12.3.0.eb:
       options:
         from-pr: 19996
+  - dask-2023.9.2-foss-2023a.eb

From 938e52e66b529458fad9984f79bce156dc43d023 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Thu, 29 Feb 2024 14:16:17 +0100
Subject: [PATCH 093/137] Update
 easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index d187d8e92c..0a0a625c35 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -31,5 +31,6 @@ easyconfigs:
         from-pr: 19554
   - Pillow-SIMD-9.5.0-GCCcore-12.3.0.eb:
       options:
+        # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19996
         from-pr: 19996
   - dask-2023.9.2-foss-2023a.eb

From 301ab593867335700b576cc9949fe58c408325e7 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Thu, 29 Feb 2024 18:12:22 +0100
Subject: [PATCH 094/137] Make check_missing_installations.sh check against
 develop branch of EasyBuild

---
 check_missing_installations.sh | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 5ea7c5a4f5..8845adb161 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -18,6 +18,13 @@ easystack=$1
 
 LOCAL_TMPDIR=$(mktemp -d)
 
+# Clone the develop branch of EasyBuild and use that to search for easyconfigs
+git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs
+EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
+
+# All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr
+grep -v from-pr ${easystack} > ${LOCAL_TMPDIR}/${easystack} 
+
 source $TOPDIR/scripts/utils.sh
 
 source $TOPDIR/configure_easybuild
@@ -27,7 +34,7 @@ ${EB:-eb} --show-config
 
 echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
 eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
-${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out}
+${EB:-eb} --easystack ${LOCAL_TMPDIR}/${easystack} --missing 2>&1 | tee ${eb_missing_out}
 exit_code=${PIPESTATUS[0]}
 
 ok_msg="Command 'eb --missing ...' succeeded, analysing output..."

From 93d1d7df3c0659e9886bac3cd0d4c994e7a82d09 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Thu, 29 Feb 2024 18:19:46 +0100
Subject: [PATCH 095/137] Use filename rather than full path

---
 check_missing_installations.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 8845adb161..6c38e8e37f 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -23,7 +23,8 @@ git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $
 EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
 
 # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr
-grep -v from-pr ${easystack} > ${LOCAL_TMPDIR}/${easystack} 
+tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack})
+grep -v from-pr ${easystack} > ${tmp_easystack} 
 
 source $TOPDIR/scripts/utils.sh
 
@@ -34,7 +35,7 @@ ${EB:-eb} --show-config
 
 echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..."
 eb_missing_out=$LOCAL_TMPDIR/eb_missing.out
-${EB:-eb} --easystack ${LOCAL_TMPDIR}/${easystack} --missing 2>&1 | tee ${eb_missing_out}
+${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out}
 exit_code=${PIPESTATUS[0]}
 
 ok_msg="Command 'eb --missing ...' succeeded, analysing output..."

From f6259dc563bcb1cad3ee962f56501872e53c0cba Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Thu, 29 Feb 2024 18:26:16 +0100
Subject: [PATCH 096/137] export envvar

---
 check_missing_installations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 6c38e8e37f..c902fa8184 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -20,7 +20,7 @@ LOCAL_TMPDIR=$(mktemp -d)
 
 # Clone the develop branch of EasyBuild and use that to search for easyconfigs
 git clone -b develop https://github.com/easybuilders/easybuild-easyconfigs.git $LOCAL_TMPDIR/easyconfigs
-EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
+export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
 
 # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr
 tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack})

From 78ba8d8e304ec0ae4c82721e3e5fae7e25402f91 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Fri, 1 Mar 2024 12:53:07 +0100
Subject: [PATCH 097/137] Adding OSU microbenchmarks with GPU support for
 2023a, and one without GPU support for 2023b

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 0a0a625c35..0d8e71e86c 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -34,3 +34,5 @@ easyconfigs:
         # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19996
         from-pr: 19996
   - dask-2023.9.2-foss-2023a.eb
+  - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb
+  - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb

From 1fd1609ec960e3b713e940076241f118677a3181 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Fri, 1 Mar 2024 14:00:03 +0100
Subject: [PATCH 098/137] 'which' is not working in the container, command -v
 is

---
 test_suite.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test_suite.sh b/test_suite.sh
index 8a98fd59a8..e9168184bc 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -90,7 +90,7 @@ else
 fi
 
 # Check that a system python3 is available
-python3_found=$(which python3)
+python3_found=$(command -v python3)
 if [ -z ${python3_found} ]; then
     fatal_error "No system python3 found"
 else

From 314178937a1876eff79300fc0b562031ffc410a8 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Fri, 1 Mar 2024 15:58:01 +0100
Subject: [PATCH 099/137] Workaround for 'PSM3 can't open nic unit' error

---
 test_suite.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test_suite.sh b/test_suite.sh
index e9168184bc..e3bab04aec 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -164,6 +164,9 @@ sed -i "s/__NUM_SOCKETS__/${socket_count}/g" $RFM_CONFIG_FILES
 sed -i "s/__NUM_CPUS_PER_CORE__/${threads_per_core}/g" $RFM_CONFIG_FILES
 sed -i "s/__NUM_CPUS_PER_SOCKET__/${cores_per_socket}/g" $RFM_CONFIG_FILES
 
+# Workaround for https://github.com/EESSI/software-layer/pull/467#issuecomment-1973341966
+export PSM3_DEVICES='self,shm'  # this is enough, since we only run single node for now
+
 # Check we can run reframe
 reframe --version
 if [[ $? -eq 0 ]]; then

From efbeb561740f8ef5c066e1b064c16f45087c668f Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Mon, 4 Mar 2024 16:59:00 +0100
Subject: [PATCH 100/137] Use LMOD_CONFIG_DIR instead of LMOD_RC to allow user
 overrides of what we specify in our RC file, see search order on
 https://lmod.readthedocs.io/en/latest/145_properties.html?highlight=lmod_rc#the-properties-file-lmodrc-lua
 . Also, prefix our hooks with eessi_ to avoid unintentional clashes in
 namespace between site lmod configuration and eessi lmod configuration

---
 create_lmodrc.py                 | 8 ++++----
 init/eessi_environment_variables | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/create_lmodrc.py b/create_lmodrc.py
index bc69dd4396..ff34906c0b 100755
--- a/create_lmodrc.py
+++ b/create_lmodrc.py
@@ -29,7 +29,7 @@
     return content
 end
 
-local function cuda_enabled_load_hook(t)
+local function eessi_cuda_enabled_load_hook(t)
     local frameStk  = require("FrameStk"):singleton()
     local mt        = frameStk:mt()
     local simpleName = string.match(t.modFullName, "(.-)/")
@@ -94,7 +94,7 @@
     end
 end
 
-local function openmpi_load_hook(t)
+local function eessi_openmpi_load_hook(t)
     -- disable smcuda BTL when loading OpenMPI module for aarch64/neoverse_v1,
     -- to work around hang/crash due to bug in OpenMPI;
     -- see https://gitlab.com/eessi/support/-/issues/41
@@ -114,8 +114,8 @@
     end
 end
 
-hook.register("load", cuda_enabled_load_hook)
-hook.register("load", openmpi_load_hook)
+hook.register("load", eessi_cuda_enabled_load_hook)
+hook.register("load", eessi_openmpi_load_hook)
 """
 
 def error(msg):
diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables
index af5222e7b9..d3a209630e 100644
--- a/init/eessi_environment_variables
+++ b/init/eessi_environment_variables
@@ -76,8 +76,8 @@ if [ -d $EESSI_PREFIX ]; then
             false
           fi
 
-          export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua"
-          if [ -f $LMOD_RC ]; then
+          export LMOD_CONFIG_DIR="$EESSI_SOFTWARE_PATH/.lmod"
+          if [ -f $LMOD_CONFIG_DIR/lmodrc.lua ]; then
             show_msg "Found Lmod configuration file at $LMOD_RC"
           else
             error "Lmod configuration file not found at $LMOD_RC"

From 590cabd7c068a9986f0eba33792005a7b9ce883a Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 5 Mar 2024 10:42:27 +0100
Subject: [PATCH 101/137] Combine functions into a single function, before
 registering this as a hook. Lmod only permits the registration of a single
 function, trying to register a second function will just overwrite the first,
 as can be seen from
 https://github.com/TACC/Lmod/blob/a97d68193a6f32ebca4b4bd70dac0f6ac8fddefe/src/Hook.lua#L87

---
 create_lmodrc.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/create_lmodrc.py b/create_lmodrc.py
index ff34906c0b..621c8e271a 100755
--- a/create_lmodrc.py
+++ b/create_lmodrc.py
@@ -114,8 +114,15 @@
     end
 end
 
-hook.register("load", eessi_cuda_enabled_load_hook)
-hook.register("load", eessi_openmpi_load_hook)
+-- Combine both functions into a single one, as we can only register one function as load hook in lmod
+-- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed
+function eessi_load_hook(t)
+    eessi_cuda_enabled_load_hook(t)
+    eessi_openmpi_load_hook(t)
+end
+
+
+hook.register("load", eessi_load_hook)
 """
 
 def error(msg):

From 0de249e32f6cb2cd1cbb694cbddf37fb93987561 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Tue, 5 Mar 2024 18:15:10 +0100
Subject: [PATCH 102/137] Look for site-specific lmodrc file in
 host_injections. If present, set LMOD_RC

---
 init/eessi_environment_variables | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables
index d3a209630e..1c8a68b1d5 100644
--- a/init/eessi_environment_variables
+++ b/init/eessi_environment_variables
@@ -82,6 +82,11 @@ if [ -d $EESSI_PREFIX ]; then
           else
             error "Lmod configuration file not found at $LMOD_RC"
           fi
+          # Allow for site-specific lmod configuration
+          host_lmod_rc="$EESSI_CVMFS_REPO"/host_injections/.lmod/lmodrc.lua
+          if [ -f $host_lmod_rc ]; then
+              export LMOD_RC="$host_lmod_rc"
+          fi
 
         else
           error "EESSI software layer at $EESSI_SOFTWARE_PATH not found!"

From c1840dd55f306b20e6d26201990313ad4dc18a3a Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Wed, 6 Mar 2024 15:05:07 +0100
Subject: [PATCH 103/137] Next the local lmodrc within the versioned directory,
 e.g. /cvmfs/software.eessi.io/host_injections/2023.06/.lmod/lmodrc.lua

---
 init/eessi_environment_variables | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables
index 1c8a68b1d5..5407034191 100644
--- a/init/eessi_environment_variables
+++ b/init/eessi_environment_variables
@@ -83,7 +83,7 @@ if [ -d $EESSI_PREFIX ]; then
             error "Lmod configuration file not found at $LMOD_RC"
           fi
           # Allow for site-specific lmod configuration
-          host_lmod_rc="$EESSI_CVMFS_REPO"/host_injections/.lmod/lmodrc.lua
+          host_lmod_rc="${EESSI_PREFIX/versions/host_injections}/.lmod/lmodrc.lua
           if [ -f $host_lmod_rc ]; then
               export LMOD_RC="$host_lmod_rc"
           fi

From df4140a74bad2f0af022c10a2d8ada12fa1627df Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Wed, 6 Mar 2024 15:54:28 +0100
Subject: [PATCH 104/137] Update init/eessi_environment_variables

---
 init/eessi_environment_variables | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables
index 5407034191..f9ca6de408 100644
--- a/init/eessi_environment_variables
+++ b/init/eessi_environment_variables
@@ -83,7 +83,7 @@ if [ -d $EESSI_PREFIX ]; then
             error "Lmod configuration file not found at $LMOD_RC"
           fi
           # Allow for site-specific lmod configuration
-          host_lmod_rc="${EESSI_PREFIX/versions/host_injections}/.lmod/lmodrc.lua
+          host_lmod_rc="${EESSI_PREFIX/versions/host_injections}"/.lmod/lmodrc.lua
           if [ -f $host_lmod_rc ]; then
               export LMOD_RC="$host_lmod_rc"
           fi

From 78e582ba2c5fd5de813836151a3d256fc6ba5c94 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Wed, 6 Mar 2024 15:59:31 +0100
Subject: [PATCH 105/137] Update init/eessi_environment_variables

---
 init/eessi_environment_variables | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables
index f9ca6de408..b41dd60924 100644
--- a/init/eessi_environment_variables
+++ b/init/eessi_environment_variables
@@ -78,9 +78,9 @@ if [ -d $EESSI_PREFIX ]; then
 
           export LMOD_CONFIG_DIR="$EESSI_SOFTWARE_PATH/.lmod"
           if [ -f $LMOD_CONFIG_DIR/lmodrc.lua ]; then
-            show_msg "Found Lmod configuration file at $LMOD_RC"
+            show_msg "Found Lmod configuration file at $LMOD_CONFIG_DIR/lmodrc.lua"
           else
-            error "Lmod configuration file not found at $LMOD_RC"
+            error "Lmod configuration file not found at $LMOD_CONFIG_DIR/lmodrc.lua"
           fi
           # Allow for site-specific lmod configuration
           host_lmod_rc="${EESSI_PREFIX/versions/host_injections}"/.lmod/lmodrc.lua

From 77c97235f0291dff7b98f2f1958e345a7afb48ee Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Wed, 6 Mar 2024 16:00:42 +0100
Subject: [PATCH 106/137] Update init/eessi_environment_variables

---
 init/eessi_environment_variables | 1 +
 1 file changed, 1 insertion(+)

diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables
index b41dd60924..d4c113c67a 100644
--- a/init/eessi_environment_variables
+++ b/init/eessi_environment_variables
@@ -85,6 +85,7 @@ if [ -d $EESSI_PREFIX ]; then
           # Allow for site-specific lmod configuration
           host_lmod_rc="${EESSI_PREFIX/versions/host_injections}"/.lmod/lmodrc.lua
           if [ -f $host_lmod_rc ]; then
+              show_msg "Found site Lmod configuration file at $LMOD_RC"
               export LMOD_RC="$host_lmod_rc"
           fi
 

From 276a64c66347e24b94ace71914547cc0abd75381 Mon Sep 17 00:00:00 2001
From: Richard Top <richard.topouchian@uib.no>
Date: Thu, 7 Mar 2024 07:54:23 +0000
Subject: [PATCH 107/137] {2023.06}[gompi/2023b] CDO v2.2.2

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 4dd31dbd5d..2664d8f417 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -8,3 +8,6 @@ easyconfigs:
   - matplotlib-3.8.2-gfbf-2023b.eb:
       options:
         from-pr: 19552
+  - CDO-2.2.2-gompi-2023b.eb:
+      options:
+        from-pr: 19792   

From e33938f1a8424d28966827c4a8942f82b8043bed Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Thu, 7 Mar 2024 14:52:25 +0100
Subject: [PATCH 108/137] Revert changes in eessi_environment_variables. Limit
 this PR to the bug fix only. We can follow up properly using SitePackage.lua
 instead

---
 init/eessi_environment_variables | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/init/eessi_environment_variables b/init/eessi_environment_variables
index d4c113c67a..af5222e7b9 100644
--- a/init/eessi_environment_variables
+++ b/init/eessi_environment_variables
@@ -76,17 +76,11 @@ if [ -d $EESSI_PREFIX ]; then
             false
           fi
 
-          export LMOD_CONFIG_DIR="$EESSI_SOFTWARE_PATH/.lmod"
-          if [ -f $LMOD_CONFIG_DIR/lmodrc.lua ]; then
-            show_msg "Found Lmod configuration file at $LMOD_CONFIG_DIR/lmodrc.lua"
+          export LMOD_RC="$EESSI_SOFTWARE_PATH/.lmod/lmodrc.lua"
+          if [ -f $LMOD_RC ]; then
+            show_msg "Found Lmod configuration file at $LMOD_RC"
           else
-            error "Lmod configuration file not found at $LMOD_CONFIG_DIR/lmodrc.lua"
-          fi
-          # Allow for site-specific lmod configuration
-          host_lmod_rc="${EESSI_PREFIX/versions/host_injections}"/.lmod/lmodrc.lua
-          if [ -f $host_lmod_rc ]; then
-              show_msg "Found site Lmod configuration file at $LMOD_RC"
-              export LMOD_RC="$host_lmod_rc"
+            error "Lmod configuration file not found at $LMOD_RC"
           fi
 
         else

From 45022b6b47c3046ea3e2d4033197fdd80dc63850 Mon Sep 17 00:00:00 2001
From: Alan O'Cais <alan.ocais@cecam.org>
Date: Fri, 8 Mar 2024 17:30:55 +0100
Subject: [PATCH 109/137] Allow for open PRs when checking missing
 installations in build script

---
 EESSI-install-software.sh      |  2 +-
 check_missing_installations.sh | 20 +++++++++++++++++---
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 69de9d1997..75b4f71178 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -237,7 +237,7 @@ else
                 copy_build_log "${eb_last_log}" "${build_logs_dir}"
             fi
     
-            $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file}
+            $TOPDIR/check_missing_installations.sh ${TOPDIR}/${easystack_file} ${TOPDIR}/${pr_diff}
         else
             fatal_error "Easystack file ${easystack_file} not found!"
         fi
diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index c902fa8184..623c46c74f 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -10,8 +10,15 @@
 
 TOPDIR=$(dirname $(realpath $0))
 
-if [ $# -ne 1 ]; then
-    echo "ERROR: Usage: $0 <path to easystack file>" >&2
+if [ "$#" -eq 1 ]; then
+    true
+elif [ "$#" -eq 2 ]; then
+    echo "Using $2 to give create exceptions for PR filtering of easystack"
+    # Find lines that are added and use from-pr, make them unique, grab the
+    # PR numbers and use them to construct something we can use within awk
+    pr_exceptions=$(grep ^+ $2 | grep from-pr | uniq | awk '{print $3}' | xargs -i echo " || /'{}'/")
+else
+    echo "ERROR: Usage: $0 <path to easystack file> (<optional path to PR diff>)" >&2
     exit 1
 fi
 easystack=$1
@@ -24,7 +31,10 @@ export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
 
 # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr
 tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack})
-grep -v from-pr ${easystack} > ${tmp_easystack} 
+# Let's use awk so we can allow for exceptions if we are given a PR diff file
+awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack"
+awk_command=${awk_command/\\/}  # Strip out the backslash we needed for !
+eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack}
 
 source $TOPDIR/scripts/utils.sh
 
@@ -40,6 +50,10 @@ exit_code=${PIPESTATUS[0]}
 
 ok_msg="Command 'eb --missing ...' succeeded, analysing output..."
 fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'"
+if [ "$#" -eq 1 ]; then
+    fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)"
+fi
+
 check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}"
 
 # the above assesses the installed software for each easyconfig provided in

From 723fce82b272cccd75a1d1841ef301be419461e6 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Sat, 9 Mar 2024 11:39:33 +0100
Subject: [PATCH 110/137] Check for unmerged EasyConfig output pattern

---
 bot/check-build.sh | 143 ++++++++++++++++++++++++++++-----------------
 1 file changed, 90 insertions(+), 53 deletions(-)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index ec1ca56bba..e737363f35 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -98,15 +98,15 @@ job_dir=${PWD}
 job_out="slurm-${SLURM_JOB_ID}.out"
 [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'"
 if  [[ -f ${job_out} ]]; then
-    SLURM=1
+    SLURM_OUTPUT_FOUND=1
     [[ ${VERBOSE} -ne 0 ]] && echo "   found slurm output file '"${job_out}"'"
 else
-    SLURM=0
+    SLURM_OUTPUT_FOUND=0
     [[ ${VERBOSE} -ne 0 ]] && echo "   Slurm output file '"${job_out}"' NOT found"
 fi
 
 ERROR=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_error='ERROR: '
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}")
   [[ $? -eq 0 ]] && ERROR=1 || ERROR=0
@@ -116,7 +116,7 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 FAILED=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_failed='FAILED: '
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}")
   [[ $? -eq 0 ]] && FAILED=1 || FAILED=0
@@ -126,7 +126,7 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 MISSING=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_req_missing=' required modules missing:'
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}")
   [[ $? -eq 0 ]] && MISSING=1 || MISSING=0
@@ -136,7 +136,7 @@ if [[ ${SLURM} -eq 1 ]]; then
 fi
 
 NO_MISSING=-1
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_no_missing='No missing installations'
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}")
   [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0
@@ -147,7 +147,7 @@ fi
 
 TGZ=-1
 TARBALL=
-if [[ ${SLURM} -eq 1 ]]; then
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
   GP_tgz_created="\.tar\.gz created!"
   grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u)
   if [[ $? -eq 0 ]]; then
@@ -169,9 +169,27 @@ fi
 [[ ${VERBOSE} -ne 0 ]] && echo "  NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 [[ ${VERBOSE} -ne 0 ]] && echo "  TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)"
 
+# Here, we try to do some additional analysis on the output file
+# to see if we can print a more clear 'reason' for the failure
+# For now, we only analyse unmerged EasyConfigs as potential cause, but we can easily add checks for other
+# specific scenarios below
+
+# Check for the pattern being added here https://github.com/EESSI/software-layer/pull/493 to the output to
+# see if EasyConfigs might have been unmerged, and that's causing a failure
+UNMERGED_EASYCONFIG=-1
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then
+  gp_unmerged="are you sure all PRs referenced have been merged in EasyBuild"
+  grep_unmerged=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${gp_unmerged}")
+  [[ $? -eq 0 ]] && UNMERGED_EASYCONFIG=1 || UNMERGED_EASYCONFIG=0
+  # have to be careful to not add searched for pattern into slurm out file
+  [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${gp_unmerged}"'"
+  [[ ${VERBOSE} -ne 0 ]] && echo "${grep_unmerged}"
+fi
+
 job_result_file=_bot_job${SLURM_JOB_ID}.result
 
-if [[ ${SLURM} -eq 1 ]] && \
+# Default reason:
+if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]] && \
    [[ ${ERROR} -eq 0 ]] && \
    [[ ${FAILED} -eq 0 ]] && \
    [[ ${MISSING} -eq 0 ]] && \
@@ -180,7 +198,12 @@ if [[ ${SLURM} -eq 1 ]] && \
    [[ ! -z ${TARBALL} ]]; then
     # SUCCESS
     status="SUCCESS"
+    reason=""
     summary=":grin: SUCCESS"
+elif [[ ${UNMERGED_EASYCONFIG} -eq 1 ]]; then
+    status="FAILURE"
+    reason="EasyConfig not found during missing installation check. Are you sure all PRs referenced have been merged in EasyBuild?"
+    summary=":cry: FAILURE"
 else
     # FAILURE
     status="FAILURE"
@@ -253,14 +276,6 @@ fi
 # </details>
 ###
 
-# construct and write complete PR comment details: implements third alternative
-comment_template="<details>__SUMMARY_FMT__<dl>__DETAILS_FMT____ARTEFACTS_FMT__</dl></details>"
-comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
-comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
-comment_success_item_fmt=":white_check_mark: __ITEM__"
-comment_failure_item_fmt=":x: __ITEM__"
-comment_artefacts_fmt="<dt>_Artefacts_</dt><dd>__ARTEFACTS_LIST__</dd>"
-comment_artefact_details_fmt="<details>__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__</details>"
 
 function print_br_item() {
     format="${1}"
@@ -332,42 +347,65 @@ echo -n "comment_description = " >> ${job_result_file}
 # - __DETAILS_FMT__ -> variable $comment_details
 # - __ARTEFACTS_FMT__ -> variable $comment_artefacts
 
+# construct and write complete PR comment details: implements third alternative
+comment_template="<details>__SUMMARY_FMT__<dl>__REASON_FMT____DETAILS_FMT____ARTEFACTS_FMT__</dl></details>"
+comment_success_item_fmt=":white_check_mark: __ITEM__"
+comment_failure_item_fmt=":x: __ITEM__"
+
+# Initialize comment_description
+comment_description=${comment_template}
+
+# Now, start replacing template items one by one
+# Replace the summary template (__SUMMARY_FMT__)
+comment_summary_fmt="<summary>__SUMMARY__ _(click triangle for details)_</summary>"
 comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}"
+comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
+
+# Only add if there is a reason (e.g. no reason for successful runs)
+if [[ ! -z ${reason} ]]; then
+    comment_reason_fmt="<dt>_Reason_</dt><dd>__REASONS__</dd>"
+    reason_details="${comment_reason_fmt/__REASONS__/${reason}}"
+    comment_description=${comment_description/__REASON_FMT__/${reason_details}}
+else
+    comment_description=${comment_description/__REASON_FMT__/""}
+fi
 
-# first construct comment_details_list, abbreviated CoDeList
+# Replace the details template (__DETAILS_FMT__)
+# first construct comment_details_list, abbreviated comment_details_list
 # then use it to set comment_details
-CoDeList=""
+comment_details_list=""
 
 success_msg="job output file <code>${job_out}</code>"
 failure_msg="no job output file <code>${job_out}</code>"
-CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${SLURM_OUTPUT_FOUND} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_error}</code>"
 failure_msg="found message matching <code>${GP_error}</code>"
-CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_failed}</code>"
 failure_msg="found message matching <code>${GP_failed}</code>"
-CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="no message matching <code>${GP_req_missing}</code>"
 failure_msg="found message matching <code>${GP_req_missing}</code>"
-CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}")
 
 success_msg="found message(s) matching <code>${GP_no_missing}</code>"
 failure_msg="no message matching <code>${GP_no_missing}</code>"
-CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
+comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}")
 
 success_msg="found message matching <code>${GP_tgz_created}</code>"
 failure_msg="no message matching <code>${GP_tgz_created}</code>"
-CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
-
-comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}"
-
+comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
+# Now, do the actual repalcement of __DETAILS_FMT__
+comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
+comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
+comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
 
-# first construct comment_artefacts_list, abbreviated CoArList
+# first construct comment_artefacts_list
 # then use it to set comment_artefacts
-CoArList=""
+comment_artifacts_list=""
 
 # TARBALL should only contain a single tarball
 if [[ ! -z ${TARBALL} ]]; then
@@ -427,50 +465,49 @@ if [[ ! -z ${TARBALL} ]]; then
     software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u)
 
     artefact_summary="<summary>$(print_code_item '__ITEM__' ${TARBALL})</summary>"
-    CoArList=""
-    CoArList="${CoArList}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})"
-    CoArList="${CoArList}$(print_br_item 'entries: __ITEM__' ${entries})"
-    CoArList="${CoArList}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)"
-    CoArList="${CoArList}<pre>"
+    comment_artifacts_list=""
+    comment_artifacts_list="${comment_artifacts_list}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})"
+    comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'entries: __ITEM__' ${entries})"
+    comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)"
+    comment_artifacts_list="${comment_artifacts_list}<pre>"
     if [[ ! -z ${modules} ]]; then
         while IFS= read -r mod ; do
-            CoArList="${CoArList}$(print_br_item '<code>__ITEM__</code>' ${mod})"
+            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '<code>__ITEM__</code>' ${mod})"
         done <<< "${modules}"
     else
-        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no module files in tarball')"
+        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no module files in tarball')"
     fi
-    CoArList="${CoArList}</pre>"
-    CoArList="${CoArList}$(print_br_item 'software under ___ITEM___' ${prefix}/software)"
-    CoArList="${CoArList}<pre>"
+    comment_artifacts_list="${comment_artifacts_list}</pre>"
+    comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'software under ___ITEM___' ${prefix}/software)"
+    comment_artifacts_list="${comment_artifacts_list}<pre>"
     if [[ ! -z ${software_pkgs} ]]; then
         while IFS= read -r sw_pkg ; do
-            CoArList="${CoArList}$(print_br_item '<code>__ITEM__</code>' ${sw_pkg})"
+            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '<code>__ITEM__</code>' ${sw_pkg})"
         done <<< "${software_pkgs}"
     else
-        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no software packages in tarball')"
+        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no software packages in tarball')"
     fi
-    CoArList="${CoArList}</pre>"
-    CoArList="${CoArList}$(print_br_item 'other under ___ITEM___' ${prefix})"
-    CoArList="${CoArList}<pre>"
+    comment_artifacts_list="${comment_artifacts_list}</pre>"
+    comment_artifacts_list="${comment_artifacts_list}$(print_br_item 'other under ___ITEM___' ${prefix})"
+    comment_artifacts_list="${comment_artifacts_list}<pre>"
     if [[ ! -z ${other_shortened} ]]; then
         while IFS= read -r other ; do
-            CoArList="${CoArList}$(print_br_item '<code>__ITEM__</code>' ${other})"
+            comment_artifacts_list="${comment_artifacts_list}$(print_br_item '<code>__ITEM__</code>' ${other})"
         done <<< "${other_shortened}"
     else
-        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no other files in tarball')"
+        comment_artifacts_list="${comment_artifacts_list}$(print_br_item '__ITEM__' 'no other files in tarball')"
     fi
-    CoArList="${CoArList}</pre>"
+    comment_artifacts_list="${comment_artifacts_list}</pre>"
 else
-    CoArList="${CoArList}$(print_dd_item 'No artefacts were created or found.' '')"
+    comment_artifacts_list="${comment_artifacts_list}$(print_dd_item 'No artefacts were created or found.' '')"
 fi
 
+comment_artefact_details_fmt="<details>__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__</details>"
 comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}"
-comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${CoArList}}"
-comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}"
+comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${comment_artifacts_list}}"
 
-# now put all pieces together creating comment_details from comment_template
-comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}}
-comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
+comment_artefacts_fmt="<dt>_Artefacts_</dt><dd>__ARTEFACTS_LIST__</dd>"
+comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}"
 comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}}
 
 echo "${comment_description}" >> ${job_result_file}

From f4f277605a17aea824f940ae1de0727b190be47d Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Sat, 9 Mar 2024 13:22:18 +0100
Subject: [PATCH 111/137] First, try without --from-pr. If that fails, try with
 --from-pr, but only for items added to the EasyStack file in _this_ PR to
 software-layer. If that succeeds, the reason must be that one of the
 EasyBuild PRs wasn't merged yet, and we can add that information to the
 fail_msg to make it more explicit

---
 check_missing_installations.sh | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 623c46c74f..fd2c0d34bb 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -31,6 +31,8 @@ export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
 
 # All PRs used in EESSI are supposed to be merged, so we can strip out all cases of from-pr
 tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack})
+grep -v from-pr ${easystack} > ${tmp_easystack}
+
 # Let's use awk so we can allow for exceptions if we are given a PR diff file
 awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack"
 awk_command=${awk_command/\\/}  # Strip out the backslash we needed for !
@@ -51,7 +53,25 @@ exit_code=${PIPESTATUS[0]}
 ok_msg="Command 'eb --missing ...' succeeded, analysing output..."
 fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'"
 if [ "$#" -eq 1 ]; then
-    fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)"
+    # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR
+    # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg
+
+    # Let's use awk so we can allow for exceptions if we are given a PR diff file
+    awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack"
+    awk_command=${awk_command/\\/}  # Strip out the backslash we needed for !
+    eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack}
+
+    msg=">> Checking for missing installations in ${EASYBUILD_INSTALLPATH},"
+    msg="${msg} allowing for --from-pr's that were added in this PR..."
+    echo ${msg}
+    eb_missing_out=$LOCAL_TMPDIR/eb_missing_with_from_pr.out
+    ${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out}
+    exit_code=${PIPESTATUS[0]}
+
+    # If now we succeeded, the reason must be that we originally stripped the --from-pr's
+    if [ "$#" -eq 0 ]; then
+        fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)"
+    fi
 fi
 
 check_exit_code ${exit_code} "${ok_msg}" "${fail_msg}"

From b7891087072ab05b50d93295b0f8990e27b50edf Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Sat, 9 Mar 2024 13:24:08 +0100
Subject: [PATCH 112/137] Add CFITSIO as a test example, see if we can get a
 clear error message printed on a build failure here

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml                 | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 2664d8f417..51e96f1fac 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -10,4 +10,7 @@ easyconfigs:
         from-pr: 19552
   - CDO-2.2.2-gompi-2023b.eb:
       options:
-        from-pr: 19792   
+        from-pr: 19792
+  - CFITSIO-4.3.1-GCCcore-13.2.0.eb:
+      options:
+        from-pr: 19840

From d178a8f14d6e16f5bcddcd7af0bb7978f647d823 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Sat, 9 Mar 2024 13:34:33 +0100
Subject: [PATCH 113/137] Added clarifying comment

---
 bot/check-build.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index e737363f35..24c608b04a 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -207,6 +207,8 @@ elif [[ ${UNMERGED_EASYCONFIG} -eq 1 ]]; then
 else
     # FAILURE
     status="FAILURE"
+    # General failure, we don't know a more specific reason
+    reason=""
     summary=":cry: FAILURE"
 fi
 

From 9123c9c8f483cf5f6b1e931cfe5a4484fc978faa Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Sat, 9 Mar 2024 13:37:08 +0100
Subject: [PATCH 114/137] Correct mistake: I meant to _move_ the awk part, not
 duplicate it

---
 check_missing_installations.sh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index fd2c0d34bb..4781ecc72a 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -33,11 +33,6 @@ export EASYBUILD_ROBOT_PATHS=$LOCAL_TMPDIR/easyconfigs/easybuild/easyconfigs
 tmp_easystack=${LOCAL_TMPDIR}/$(basename ${easystack})
 grep -v from-pr ${easystack} > ${tmp_easystack}
 
-# Let's use awk so we can allow for exceptions if we are given a PR diff file
-awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack"
-awk_command=${awk_command/\\/}  # Strip out the backslash we needed for !
-eval ${awk_command/EXCEPTIONS/$pr_exceptions} > ${tmp_easystack}
-
 source $TOPDIR/scripts/utils.sh
 
 source $TOPDIR/configure_easybuild

From 7e4422335f1c2f3ad682362a060906866b9b90f4 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Sat, 9 Mar 2024 13:54:07 +0100
Subject: [PATCH 115/137] Now actually check against the exit code of the
 pipe...

---
 check_missing_installations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 4781ecc72a..51a328b50c 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -47,7 +47,7 @@ exit_code=${PIPESTATUS[0]}
 
 ok_msg="Command 'eb --missing ...' succeeded, analysing output..."
 fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'"
-if [ "$#" -eq 1 ]; then
+if [ "$exit_code" -eq 1 ]; then
     # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR
     # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg
 

From da749a111320720773d648d4a55caa2ba1a826e3 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Sat, 9 Mar 2024 14:08:27 +0100
Subject: [PATCH 116/137] Make seperate exit code for the run that allows
 --from-pr

---
 check_missing_installations.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 51a328b50c..6d84384fd7 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -61,10 +61,10 @@ if [ "$exit_code" -eq 1 ]; then
     echo ${msg}
     eb_missing_out=$LOCAL_TMPDIR/eb_missing_with_from_pr.out
     ${EB:-eb} --easystack ${tmp_easystack} --missing 2>&1 | tee ${eb_missing_out}
-    exit_code=${PIPESTATUS[0]}
+    exit_code_with_from_pr=${PIPESTATUS[0]}
 
     # If now we succeeded, the reason must be that we originally stripped the --from-pr's
-    if [ "$#" -eq 0 ]; then
+    if [ "$exit_code_with_from_pr" -eq 0 ]; then
         fail_msg="$fail_msg (are you sure all PRs referenced have been merged in EasyBuild?)"
     fi
 fi

From 86d8a36c52a9a3dfe9af25662bc6c95968804024 Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Mon, 11 Mar 2024 11:46:32 +0100
Subject: [PATCH 117/137] Remove ineffective DP3 hook

---
 eb_hooks.py | 27 ---------------------------
 1 file changed, 27 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index 1478f1a508..d93ee37067 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -186,32 +186,6 @@ def parse_hook_casacore_disable_vectorize(ec, eprefix):
         raise EasyBuildError("casacore-specific hook triggered for non-casacore easyconfig?!")
 
 
-def parse_hook_dp3_enable_relaxed_vector_conversions(ec, eprefix):
-    """
-    Enable lax vector conversion for DP3 on aarch64/neoverse_v1
-    Compiling DP3 6.0 with GCC 13.2.0 (foss-2023b) gives a conversion error when building for aarch64/neoverse_v1.
-    See also, https://github.com/EESSI/software-layer/pull/479
-    """
-    if ec.name == 'DP3':
-        tcname, tcversion = ec['toolchain']['name'], ec['toolchain']['version']
-        if (
-            LooseVersion(ec.version) == LooseVersion('6.0') and
-            tcname == 'foss' and tcversion == '2023b'
-        ):
-            cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
-            if cpu_target == CPU_TARGET_NEOVERSE_V1:
-                if not hasattr(ec, 'configopts'):
-                    ec['configopts'] = ""
-                ec['configopts'] += ' -DCMAKE_CXX_FLAGS="$CXXFLAGS -flax-vector-conversions" '
-                print_msg("Changed configopts for %s: %s", ec.name, ec['configopts'])
-            else:
-                print_msg("Not changing configopts for %s on non-neoverse_v1", ec.name)
-        else:
-            print_msg("Not changing configopts for %s %s %s", ec.name, ec.version, ec.toolchain)
-    else:
-        raise EasyBuildError("DP3-specific hook triggered for non-DP3 easyconfig?!")
-
-
 def parse_hook_cgal_toolchainopts_precise(ec, eprefix):
     """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER."""
     if ec.name == 'CGAL':
@@ -633,7 +607,6 @@ def inject_gpu_property(ec):
 PARSE_HOOKS = {
     'casacore': parse_hook_casacore_disable_vectorize,
     'CGAL': parse_hook_cgal_toolchainopts_precise,
-    'DP3': parse_hook_dp3_enable_relaxed_vector_conversions,
     'fontconfig': parse_hook_fontconfig_add_fonts,
     'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors,
     'pybind11': parse_hook_pybind11_replace_catch2,

From f8816acf3c0c0022bbba3e123f6d0239469728d3 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Mon, 11 Mar 2024 14:19:15 +0100
Subject: [PATCH 118/137] {2023.06}[2023a] JupyterNotebook v7.0.2

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 0d8e71e86c..cb207e50ea 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -36,3 +36,4 @@ easyconfigs:
   - dask-2023.9.2-foss-2023a.eb
   - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb
   - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb
+  - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb

From a8c2883f0c73af0537a834221c3ae439f44c1b2f Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <33718780+casparvl@users.noreply.github.com>
Date: Mon, 11 Mar 2024 15:59:32 +0100
Subject: [PATCH 119/137] Correct typo

Co-authored-by: ocaisa <alan.ocais@cecam.org>
---
 bot/check-build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index 24c608b04a..10b1ed6a11 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -400,7 +400,7 @@ comment_details_list=${comment_details_list}$(add_detail ${NO_MISSING} 1 "${succ
 success_msg="found message matching <code>${GP_tgz_created}</code>"
 failure_msg="no message matching <code>${GP_tgz_created}</code>"
 comment_details_list=${comment_details_list}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}")
-# Now, do the actual repalcement of __DETAILS_FMT__
+# Now, do the actual replacement of __DETAILS_FMT__
 comment_details_fmt="<dt>_Details_</dt><dd>__DETAILS_LIST__</dd>"
 comment_details="${comment_details_fmt/__DETAILS_LIST__/${comment_details_list}}"
 comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}

From 6553e294163fb9e117803a2433acba3c14e135f2 Mon Sep 17 00:00:00 2001
From: Caspar van Leeuwen <casparl@tcn1.local.snellius.surf.nl>
Date: Mon, 11 Mar 2024 16:05:16 +0100
Subject: [PATCH 120/137] Skip the if-condition also if pr_exceptions is empty.
 There is no point in running the eb --missing again in that case, since there
 were no --from-pr's in this PR to begin with, so unmerged PRs cannot be the
 reason for failing

---
 check_missing_installations.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 6d84384fd7..0ee6ec4385 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -47,9 +47,11 @@ exit_code=${PIPESTATUS[0]}
 
 ok_msg="Command 'eb --missing ...' succeeded, analysing output..."
 fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'"
-if [ "$exit_code" -eq 1 ]; then
+if [ "$exit_code" -eq 1 ] && [ ! -z $pr_exceptions ]; then
     # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR
     # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg
+    # Note that if no --from-pr's were used in this PR, $pr_exceptions will be empty and we might as
+    # well skip this check - unmerged PRs can not be the reason for the non-zero exit code in that scenario
 
     # Let's use awk so we can allow for exceptions if we are given a PR diff file
     awk_command="awk '\!/'from-pr'/ EXCEPTIONS' $easystack"

From 0343684cc8e7b73fc25e160007dc458c2c54e6d0 Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Tue, 12 Mar 2024 10:19:33 +0100
Subject: [PATCH 121/137] {2023.06}[foss/2023b] python-casacore 3.5.2 +
 libspatialindex 1.9.3

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023b.yml                | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 54011c262e..186f4f86b3 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -43,3 +43,9 @@ easyconfigs:
   - CDO-2.2.2-gompi-2023b.eb:
       options:
         from-pr: 19792
+  - python-casacore-3.5.2-foss-2023b.eb:
+      options:
+        from-pr: 20084
+  - libspatialindex-1.9.3-GCCcore-13.2.0.eb:
+      options:
+        from-pr: 19922

From 3e43b061314bdde685b585804a047802cb95f81a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Tue, 12 Mar 2024 14:00:38 +0100
Subject: [PATCH 122/137] add ImageMagick-7.1.1-15-GCCcore-12.3.0.eb

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index cb207e50ea..8165755865 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -37,3 +37,6 @@ easyconfigs:
   - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb
   - OSU-Micro-Benchmarks-7.2-gompi-2023b.eb
   - JupyterNotebook-7.0.2-GCCcore-12.3.0.eb
+  - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb:
+      options:
+        from-pr: 20086

From 02955711c5600305d8feedece3e85978626ff962 Mon Sep 17 00:00:00 2001
From: Tim Kok <tim.kok@surf.nl>
Date: Tue, 12 Mar 2024 18:33:07 +0100
Subject: [PATCH 123/137] Use PR that sets LD_LIBRARY_PATH for python-casacore

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 186f4f86b3..e2d35276df 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -45,7 +45,7 @@ easyconfigs:
         from-pr: 19792
   - python-casacore-3.5.2-foss-2023b.eb:
       options:
-        from-pr: 20084
+        from-pr: 20089
   - libspatialindex-1.9.3-GCCcore-13.2.0.eb:
       options:
         from-pr: 19922

From c08831ee060d153bb1da466cfaa00e4103c37e31 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Wed, 13 Mar 2024 15:46:24 +0100
Subject: [PATCH 124/137] Update bot/check-build.sh

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 bot/check-build.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/check-build.sh b/bot/check-build.sh
index 10b1ed6a11..e075518421 100755
--- a/bot/check-build.sh
+++ b/bot/check-build.sh
@@ -174,7 +174,7 @@ fi
 # For now, we only analyse unmerged EasyConfigs as potential cause, but we can easily add checks for other
 # specific scenarios below
 
-# Check for the pattern being added here https://github.com/EESSI/software-layer/pull/493 to the output to
+# Check for the pattern being added here by check_missing_installations.sh to the output to
 # see if EasyConfigs might have been unmerged, and that's causing a failure
 UNMERGED_EASYCONFIG=-1
 if [[ ${SLURM_OUTPUT_FOUND} -eq 1 ]]; then

From 992b11dc782d391899862f80033f08b1efba4634 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Wed, 13 Mar 2024 15:47:37 +0100
Subject: [PATCH 125/137] Remove PR to trigger comment

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
index 2b5844289d..e2d35276df 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023b.yml
@@ -49,6 +49,3 @@ easyconfigs:
   - libspatialindex-1.9.3-GCCcore-13.2.0.eb:
       options:
         from-pr: 19922
-  - CFITSIO-4.3.1-GCCcore-13.2.0.eb:
-      options:
-        from-pr: 19840

From 8d6493ea05422ee443f10bc9230c3ff3884af998 Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Wed, 13 Mar 2024 15:48:08 +0100
Subject: [PATCH 126/137] Trigger on any non-zero exit code

Co-authored-by: Kenneth Hoste <kenneth.hoste@ugent.be>
---
 check_missing_installations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 0ee6ec4385..7c29228416 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -47,7 +47,7 @@ exit_code=${PIPESTATUS[0]}
 
 ok_msg="Command 'eb --missing ...' succeeded, analysing output..."
 fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'"
-if [ "$exit_code" -eq 1 ] && [ ! -z $pr_exceptions ]; then
+if [ "$exit_code" -ne 0 ] && [ ! -z $pr_exceptions ]; then
     # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR
     # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg
     # Note that if no --from-pr's were used in this PR, $pr_exceptions will be empty and we might as

From 0fdbd3a3be714c116b76f72c694406ea03e513fe Mon Sep 17 00:00:00 2001
From: ocaisa <alan.ocais@cecam.org>
Date: Thu, 14 Mar 2024 11:35:35 +0100
Subject: [PATCH 127/137] Add Z3 with new module name

Via https://github.com/easybuilders/easybuild-easyconfigs/pull/20050 , the different Z3 easyconfigs have been consolidated. This means the version used as a dependency for PyTorch has been "corrected" so we need to dance around that for our CI purposes
---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml                | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 8165755865..5a46cf5fb2 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -40,3 +40,9 @@ easyconfigs:
   - ImageMagick-7.1.1-15-GCCcore-12.3.0.eb:
       options:
         from-pr: 20086
+  - Z3-4.12.2-GCCcore-12.3.0.eb:
+      options:
+        # The Z3 dependency of PyTorch had it's versionsuffix removed
+        # and we need to workaround the problem this creates,
+        # see https://github.com/EESSI/software-layer/pull/501 for details
+        from-pr: 20050

From d097189aa07f2d323779212e4451a2df2466193d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Thu, 14 Mar 2024 22:55:43 +0100
Subject: [PATCH 128/137] {2023.06}[2023a] PyOpenGL 3.1.7

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 5a46cf5fb2..1d3119e5c2 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -46,3 +46,4 @@ easyconfigs:
         # and we need to workaround the problem this creates,
         # see https://github.com/EESSI/software-layer/pull/501 for details
         from-pr: 20050
+  - PyOpenGL-3.1.7-GCCcore-12.3.0.eb

From 8245aff4b7f4dd178924cd42eda76c21eed146fa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Bob=20Dr=C3=B6ge?= <b.e.droge@rug.nl>
Date: Fri, 15 Mar 2024 00:02:09 +0100
Subject: [PATCH 129/137] add from-pr=20007

---
 .../2023.06/eessi-2023.06-eb-4.9.0-2023a.yml                  | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 1d3119e5c2..65b4265ff7 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -46,4 +46,6 @@ easyconfigs:
         # and we need to workaround the problem this creates,
         # see https://github.com/EESSI/software-layer/pull/501 for details
         from-pr: 20050
-  - PyOpenGL-3.1.7-GCCcore-12.3.0.eb
+  - PyOpenGL-3.1.7-GCCcore-12.3.0.eb:
+      options:
+        from-pr: 20007

From af58e1a5fc00ab311eab185a91302d0c735c8978 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 15 Mar 2024 20:48:12 +0100
Subject: [PATCH 130/137] {2023.06}[2023a] ipympl 0.9.3

---
 .../software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
index 65b4265ff7..384aa04a9c 100644
--- a/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
+++ b/easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.0-2023a.yml
@@ -49,3 +49,6 @@ easyconfigs:
   - PyOpenGL-3.1.7-GCCcore-12.3.0.eb:
       options:
         from-pr: 20007
+  - ipympl-0.9.3-foss-2023a.eb:
+      options:
+        from-pr: 20126

From a12b9a340e17953a8e6fe741245663c69ece813b Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Fri, 15 Mar 2024 21:13:32 +0100
Subject: [PATCH 131/137] fix check_missing_installations.sh when unmerged PR
 is used

---
 check_missing_installations.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/check_missing_installations.sh b/check_missing_installations.sh
index 7c29228416..d8135ea3cb 100755
--- a/check_missing_installations.sh
+++ b/check_missing_installations.sh
@@ -47,7 +47,7 @@ exit_code=${PIPESTATUS[0]}
 
 ok_msg="Command 'eb --missing ...' succeeded, analysing output..."
 fail_msg="Command 'eb --missing ...' failed, check log '${eb_missing_out}'"
-if [ "$exit_code" -ne 0 ] && [ ! -z $pr_exceptions ]; then
+if [ "$exit_code" -ne 0 ] && [ ! -z "$pr_exceptions" ]; then
     # We might have failed due to unmerged PRs. Try to make exceptions for --from-pr added in this PR
     # to software-layer, and see if then it passes. If so, we can report a more specific fail_msg
     # Note that if no --from-pr's were used in this PR, $pr_exceptions will be empty and we might as

From 330db7a02cf011c2115babf598991217f87bd022 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 10:32:49 +0100
Subject: [PATCH 132/137] also dump status to test result file

---
 bot/check-test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bot/check-test.sh b/bot/check-test.sh
index 4b5f7575e5..f045b9500a 100755
--- a/bot/check-test.sh
+++ b/bot/check-test.sh
@@ -200,5 +200,6 @@ comment_description=${comment_description/__DETAILS_FMT__/${comment_details}}
 
 # Actually writing the comment description to the result file
 echo "${comment_description}" >> ${job_test_result_file}
+echo "status = ${status}" >> ${job_test_result_file}
 
 exit 0

From f3c62077c40e299bb6a3f157adfd2c1f5a32d067 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 10:33:24 +0100
Subject: [PATCH 133/137] add Caspar as author in `bot/test.sh` script

---
 bot/test.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bot/test.sh b/bot/test.sh
index 89c1a6a8bf..66c2cef129 100755
--- a/bot/test.sh
+++ b/bot/test.sh
@@ -8,6 +8,7 @@
 # https://github.com/EESSI/software-layer.git
 #
 # author: Thomas Roeblitz (@trz42)
+# author: Caspar van Leeuwen (@casparvl)
 #
 # license: GPLv2
 #

From 13072df707b96cad40b4d21cbedd6666af756b0d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 10:34:09 +0100
Subject: [PATCH 134/137] use read-only mode to access software installations
 via container

---
 bot/test.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bot/test.sh b/bot/test.sh
index 66c2cef129..925b91d49a 100755
--- a/bot/test.sh
+++ b/bot/test.sh
@@ -180,7 +180,7 @@ echo "bot/test.sh: EESSI_OS_TYPE='${EESSI_OS_TYPE}'"
 # prepare arguments to eessi_container.sh common to build and tarball steps
 declare -a COMMON_ARGS=()
 COMMON_ARGS+=("--verbose")
-COMMON_ARGS+=("--access" "rw")
+COMMON_ARGS+=("--access" "ro")
 COMMON_ARGS+=("--mode" "run")
 [[ ! -z ${CONTAINER} ]] && COMMON_ARGS+=("--container" "${CONTAINER}")
 [[ ! -z ${HTTP_PROXY} ]] && COMMON_ARGS+=("--http-proxy" "${HTTP_PROXY}")

From 1fa323ada6d0fddc312ad102cba8619d073f9fb0 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 10:34:53 +0100
Subject: [PATCH 135/137] fix typo: build -> test

---
 bot/test.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bot/test.sh b/bot/test.sh
index 925b91d49a..4984340e6e 100755
--- a/bot/test.sh
+++ b/bot/test.sh
@@ -190,7 +190,7 @@ COMMON_ARGS+=("--mode" "run")
 # make sure to use the same parent dir for storing tarballs of tmp
 PREVIOUS_TMP_DIR=${PWD}/previous_tmp
 
-# prepare directory to store tarball of tmp for build step
+# prepare directory to store tarball of tmp for test step
 TARBALL_TMP_TEST_STEP_DIR=${PREVIOUS_TMP_DIR}/test_step
 mkdir -p ${TARBALL_TMP_TEST_STEP_DIR}
 
@@ -216,7 +216,7 @@ fi
 # create tmp file for output of build step
 test_outerr=$(mktemp test.outerr.XXXX)
 
-echo "Executing command to build software:"
+echo "Executing command to test software:"
 echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}"
 echo "                     -- ./run_tests.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}"
 ./eessi_container.sh "${COMMON_ARGS[@]}" "${TEST_STEP_ARGS[@]}" \

From 8d05473eb5eedf0cdaba75f6b4d4a40f38a3182d Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 10:35:30 +0100
Subject: [PATCH 136/137] disable CPU auto-detection in template ReFrame
 configuration script used for running test suite

---
 reframe_config_bot.py.tmpl | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/reframe_config_bot.py.tmpl b/reframe_config_bot.py.tmpl
index b2f7916e79..0cc3e9f530 100644
--- a/reframe_config_bot.py.tmpl
+++ b/reframe_config_bot.py.tmpl
@@ -51,9 +51,8 @@ site_configuration = {
         {
             'purge_environment': True,
             'resolve_module_conflicts': False,  # avoid loading the module before submitting the job
-            # Enable automatic detection of CPU architecture
-            # See https://reframe-hpc.readthedocs.io/en/stable/configure.html#auto-detecting-processor-information
-            'remote_detect': True,
+            # disable automatic detection of CPU architecture (since we're using local scheduler)
+            'remote_detect': False,
         }
     ],
     'logging': common_logging_config(),

From c53f578687e43ae0c7876c3a9d5c48fc2432ad49 Mon Sep 17 00:00:00 2001
From: Kenneth Hoste <kenneth.hoste@ugent.be>
Date: Tue, 26 Mar 2024 10:37:09 +0100
Subject: [PATCH 137/137] pass down --generic to script used for running test
 suite, so tests are run with generic software installations when intended

---
 run_tests.sh  | 2 +-
 test_suite.sh | 9 ++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/run_tests.sh b/run_tests.sh
index de7fd8c2e1..1dbb47db9d 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -21,4 +21,4 @@ source ${base_dir}/init/eessi_defaults
 ./run_in_compat_layer_env.sh "git clone https://github.com/EESSI/test-suite EESSI-test-suite"
 
 # Run the test suite
-./test_suite.sh
+./test_suite.sh "$@"
diff --git a/test_suite.sh b/test_suite.sh
index e3bab04aec..95eb9daa2a 100755
--- a/test_suite.sh
+++ b/test_suite.sh
@@ -23,7 +23,7 @@ POSITIONAL_ARGS=()
 while [[ $# -gt 0 ]]; do
   case $1 in
     -g|--generic)
-      EASYBUILD_OPTARCH="GENERIC"
+      DETECTION_PARAMETERS="--generic"
       shift
       ;;
     -h|--help)
@@ -75,10 +75,9 @@ TMPDIR=$(mktemp -d)
 
 echo ">> Setting up environment..."
 module --force purge
-# Make sure defaults are set for EESSI_CVMFS_REPO and EESSI_VERSION, before initializing EESSI
-source $TOPDIR/init/eessi_defaults
-# Initialize EESSI
-source ${EESSI_CVMFS_REPO}/versions/${EESSI_VERSION}/init/bash
+export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py $DETECTION_PARAMETERS)
+
+source $TOPDIR/init/bash
 
 # Load the ReFrame module
 # Currently, we load the default version. Maybe we should somehow make this configurable in the future?