From a55c6c9309d96a96f2a24221d24b0659c228f408 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Thu, 30 May 2024 13:47:50 +0200
Subject: [PATCH 01/17] missing changes provided in EESSI PR 579

---
 bot/build.sh               | 15 +++++++++------
 eessi_container.sh         |  5 +++++
 run_in_compat_layer_env.sh |  6 +++---
 scripts/utils.sh           |  5 +++++
 4 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 6e835cb6aa..7bd4179939 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -266,16 +266,19 @@ mkdir -p ${TARBALL_TMP_BUILD_STEP_DIR}
 BUILD_STEP_ARGS+=("--save" "${TARBALL_TMP_BUILD_STEP_DIR}")
 BUILD_STEP_ARGS+=("--storage" "${STORAGE}")
 # add options required to handle NVIDIA support
-BUILD_STEP_ARGS+=("--nvidia" "all")
+if command_exists "nvidia-smi"; then
+    echo "Command 'nvidia-smi' found, using available GPU"
+    BUILD_STEP_ARGS+=("--nvidia" "all")
+else
+    echo "No 'nvidia-smi' found, no available GPU but allowing overriding this check"
+    BUILD_STEP_ARGS+=("--nvidia" "install")
+fi
+# Retain location for host injections so we don't reinstall CUDA
+# (Always need to run the driver installation as available driver may change)
 if [[ ! -z ${SHARED_FS_PATH} ]]; then
     BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections")
 fi
 
-# Don't run the Lmod GPU driver check when doing builds (may not have a GPU, and it's not relevant for vanilla builds anyway)
-echo "EESSI_OVERRIDE_GPU_CHECK='${EESSI_OVERRIDE_GPU_CHECK}'"
-export EESSI_OVERRIDE_GPU_CHECK=1
-echo "EESSI_OVERRIDE_GPU_CHECK='${EESSI_OVERRIDE_GPU_CHECK}'"
-
 # create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)
 
diff --git a/eessi_container.sh b/eessi_container.sh
index a9405b6d8e..e6bb13cbe7 100755
--- a/eessi_container.sh
+++ b/eessi_container.sh
@@ -477,6 +477,11 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then
         mkdir -p ${EESSI_USR_LOCAL_CUDA}
         BIND_PATHS="${BIND_PATHS},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda"
         [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}"
+        if [[ "${NVIDIA_MODE}" == "install" ]] ; then
+            # No GPU so we need to "trick" Lmod to allow us to load CUDA modules even without a CUDA driver
+            # (this variable means EESSI_OVERRIDE_GPU_CHECK=1 will be set inside the container)
+            export SINGULARITYENV_EESSI_OVERRIDE_GPU_CHECK=1
+        fi
     fi
 fi
 
diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh
index 393956a0c1..cc2cdae034 100755
--- a/run_in_compat_layer_env.sh
+++ b/run_in_compat_layer_env.sh
@@ -26,12 +26,12 @@ fi
 if [ ! -z ${EESSI_VERSION_OVERRIDE} ]; then
     INPUT="export EESSI_VERSION_OVERRIDE=${EESSI_VERSION_OVERRIDE}; ${INPUT}"
 fi
-if [ ! -z ${http_proxy} ]; then
-    INPUT="export http_proxy=${http_proxy}; ${INPUT}"
-fi
 if [ ! -z ${EESSI_OVERRIDE_GPU_CHECK} ]; then
     INPUT="export EESSI_OVERRIDE_GPU_CHECK=${EESSI_OVERRIDE_GPU_CHECK}; ${INPUT}"
 fi
+if [ ! -z ${http_proxy} ]; then
+    INPUT="export http_proxy=${http_proxy}; ${INPUT}"
+fi
 if [ ! -z ${https_proxy} ]; then
     INPUT="export https_proxy=${https_proxy}; ${INPUT}"
 fi
diff --git a/scripts/utils.sh b/scripts/utils.sh
index b2be3f6221..962decd20e 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -78,6 +78,11 @@ function create_directory_structure() {
   return $return_code
 }
 
+# Function to check if a command exists
+function command_exists() {
+    command -v "$1" >/dev/null 2>&1
+}
+
 function get_path_for_tool {
     tool_name=$1
     tool_envvar_name=$2

From ee8b22de639984fddd39b4835bf05204e95fd434 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Thu, 30 May 2024 14:03:07 +0200
Subject: [PATCH 02/17] script to copy rather than link GPU driver libraries;
 needed when symlinking is run in a container

---
 .../nvidia/copy_nvidia_host_libraries.sh      | 145 ++++++++++++++++++
 1 file changed, 145 insertions(+)
 create mode 100755 scripts/gpu_support/nvidia/copy_nvidia_host_libraries.sh

diff --git a/scripts/gpu_support/nvidia/copy_nvidia_host_libraries.sh b/scripts/gpu_support/nvidia/copy_nvidia_host_libraries.sh
new file mode 100755
index 0000000000..ebc428a50d
--- /dev/null
+++ b/scripts/gpu_support/nvidia/copy_nvidia_host_libraries.sh
@@ -0,0 +1,145 @@
+#!/bin/bash
+
+# This script links host libraries related to GPU drivers to a location where
+# they can be found by the EESSI linker
+
+# Initialise our bash functions
+TOPDIR=$(dirname $(realpath $BASH_SOURCE))
+source "$TOPDIR"/../../utils.sh
+
+# We rely on ldconfig to give us the location of the libraries on the host
+command_name="ldconfig"
+# We cannot use a version of ldconfig that's being shipped under CVMFS
+exclude_prefix="/cvmfs"
+
+found_paths=()
+# Always attempt to use /sbin/ldconfig
+if [ -x "/sbin/$command_name" ]; then
+    found_paths+=("/sbin/$command_name")
+fi
+IFS=':' read -ra path_dirs <<< "$PATH"
+for dir in "${path_dirs[@]}"; do
+  if [ "$dir" = "/sbin" ]; then
+    continue  # we've already checked for $command_name in /sbin, don't need to do it twice
+  fi
+  if [[ ! "$dir" =~ ^$exclude_prefix ]]; then
+      if [ -x "$dir/$command_name" ]; then
+          found_paths+=("$dir/$command_name")
+      fi
+  fi
+done
+
+if [ ${#found_paths[@]} -gt 0 ]; then
+    echo "Found $command_name in the following locations:"
+    printf -- "- %s\n" "${found_paths[@]}"
+    echo "Using first version"
+    host_ldconfig=${found_paths[0]}
+else
+    error="$command_name not found in PATH or only found in paths starting with $exclude_prefix."
+    fatal_error "$error"
+fi
+
+# Make sure EESSI is initialised (doesn't matter what version)
+check_eessi_initialised
+
+# Find the CUDA version of the host CUDA drivers
+# (making sure that this can still work inside prefix environment inside a container)
+export LD_LIBRARY_PATH=/.singularity.d/libs:$LD_LIBRARY_PATH
+nvidia_smi_command="nvidia-smi --query-gpu=driver_version --format=csv,noheader"
+if $nvidia_smi_command > /dev/null; then
+  host_driver_version=$($nvidia_smi_command | tail -n1)
+  echo_green "Found NVIDIA GPU driver version ${host_driver_version}"
+  # If the first worked, this should work too
+  host_cuda_version=$(nvidia-smi  -q --display=COMPUTE | grep CUDA | awk 'NF>1{print $NF}')
+  echo_green "Found host CUDA version ${host_cuda_version}"
+else
+  error="Failed to successfully execute\n  $nvidia_smi_command\n"
+  fatal_error "$error"
+fi
+
+# Let's make sure the driver libraries are not already in place
+link_drivers=1
+
+# first make sure that target of host_injections variant symlink is an existing directory
+host_injections_target=$(realpath -m ${EESSI_CVMFS_REPO}/host_injections)
+if [ ! -d ${host_injections_target} ]; then
+    create_directory_structure ${host_injections_target}
+fi
+
+host_injections_nvidia_dir="${EESSI_CVMFS_REPO}/host_injections/nvidia/${EESSI_CPU_FAMILY}"
+host_injection_driver_dir="${host_injections_nvidia_dir}/host"
+host_injection_driver_version_file="$host_injection_driver_dir/driver_version.txt"
+if [ -e "$host_injection_driver_version_file" ]; then
+  if grep -q "$host_driver_version" "$host_injection_driver_version_file"; then
+    echo_green "The host GPU driver libraries (v${host_driver_version}) have already been linked! (based on ${host_injection_driver_version_file})"
+    link_drivers=0
+  else
+    # There's something there but it is out of date
+    echo_yellow "Cleaning out outdated symlinks"
+    rm $host_injection_driver_dir/*
+    if [ $? -ne 0 ]; then
+      error="Unable to remove files under '$host_injection_driver_dir'."
+      fatal_error "$error"
+    fi
+  fi
+fi
+
+drivers_linked=0
+if [ "$link_drivers" -eq 1 ]; then
+  if ! create_directory_structure "${host_injection_driver_dir}" ; then
+    fatal_error "No write permissions to directory ${host_injection_driver_dir}"
+  fi
+  cd ${host_injection_driver_dir}
+  # Need a small temporary space to hold a couple of files
+  temp_dir=$(mktemp -d)
+  echo "temp_dir: '${temp_dir}'"
+
+  # Gather libraries on the host (_must_ be host ldconfig)
+  $host_ldconfig -p | awk '{print $NF}' > "$temp_dir"/libs.txt
+  # Allow for the fact that we may be in a container so the CUDA libs might be in there
+  ls /.singularity.d/libs/* >> "$temp_dir"/libs.txt 2>/dev/null
+
+  # Leverage singularity to find the full list of libraries we should be linking to
+  echo_yellow "Downloading latest version of nvliblist.conf from Apptainer to ${temp_dir}/nvliblist.conf"
+  curl --silent --output "$temp_dir"/nvliblist.conf https://raw.githubusercontent.com/apptainer/apptainer/main/etc/nvliblist.conf
+
+  # Make symlinks to all the interesting libraries
+  grep '.so$' "$temp_dir"/nvliblist.conf | xargs -i grep {} "$temp_dir"/libs.txt | xargs -i cp -a {} ${host_injection_driver_dir}/.
+
+  # Inject driver and CUDA versions into dir
+  echo $host_driver_version > driver_version.txt
+  echo $host_cuda_version > cuda_version.txt
+  drivers_linked=1
+
+  # Remove the temporary directory when done
+  rm -r "$temp_dir"
+fi
+
+# Make latest symlink for NVIDIA drivers
+cd $host_injections_nvidia_dir
+symlink="latest"
+if [ -L "$symlink" ]; then
+    # Unless the drivers have been installed, leave the symlink alone
+    if [ "$drivers_linked" -eq 1 ]; then
+      ln -sf host latest
+    fi
+else
+    # No link exists yet
+    ln -s host latest
+fi
+
+# Make sure the libraries can be found by the EESSI linker
+host_injection_linker_dir=${EESSI_EPREFIX/versions/host_injections}
+if [ -L "$host_injection_linker_dir/lib" ]; then
+  target_path=$(readlink -f "$host_injection_linker_dir/lib")
+  if [ "$target_path" != "$$host_injections_nvidia_dir/latest" ]; then
+    cd $host_injection_linker_dir
+    ln -sf $host_injections_nvidia_dir/latest lib
+  fi
+else
+  create_directory_structure $host_injection_linker_dir
+  cd $host_injection_linker_dir
+  ln -s $host_injections_nvidia_dir/latest lib
+fi
+
+echo_green "Host NVIDIA GPU drivers linked successfully for EESSI"

From 43f51014af1c77b59f12b56b86459086d2450119 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Thu, 30 May 2024 22:56:14 +0200
Subject: [PATCH 03/17] tooling and data to replace ctypes/util.py in
 Python/3.11.3

---
 bot/build.sh                     |  46 ++++
 replace_files.txt                |   1 +
 replacement_files/ctypes/util.py | 379 +++++++++++++++++++++++++++++++
 3 files changed, 426 insertions(+)
 create mode 100644 replace_files.txt
 create mode 100644 replacement_files/ctypes/util.py

diff --git a/bot/build.sh b/bot/build.sh
index 7bd4179939..258766f802 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -279,6 +279,52 @@ if [[ ! -z ${SHARED_FS_PATH} ]]; then
     BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections")
 fi
 
+# replace some files using lower_dirs mechanism
+# - read replacements from replace_files.txt
+#   each line has the format __EESSI_SOFTWARE_PATH__/some_path relative_path
+#   /cvmfs/repo_name/versions/repo_version/software/os_type/software_dir/some_path
+# - for each replacement do
+#   - check if the target exists in the repository
+#   - create directory for replacement
+#   - copy target into directory
+ADD_LOWER_DIRS=0
+if [[ -f "replace_files.txt" ]]; then
+    LOWER_DIRS="${STORAGE}/lower_dirs"
+    mkdir -p "${LOWER_DIRS}"
+
+    repo_name=${EESSI_CVMFS_REPO_OVERRIDE}
+    repo_version=${EESSI_VERSION_OVERRIDE}
+    os_type=${EESSI_OS_TYPE}
+    software_subdir_override=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
+    software_path="/cvmfs/${repo_name}/versions/${repo_version}/software/${os_type}/${software_subdir_override}/software"
+
+    cat replace_files.txt | while read replace_spec; do
+        echo "replace_spec: '${replace_spec}'"
+        target=$(echo "${replace_spec}" | cut -f1 -d' ')
+        target_full_path=$(echo "${target}" | sed -e "s+__EESSI_SOFTWARE_PATH__+${software_path}+")
+        replace=$(echo "${replace_spec}" | cut -f2 -d' ')
+        echo "target: '${target}'"
+        echo "target_full_path: '${target_full_path}'"
+        echo "replace: '${replace}'"
+        if [[ -f ${replace} ]]; then
+            echo "replacement file exists"
+            target_lower_path=$(echo "${target_full_path}" | cut -f4- -d/)
+            echo "target_lower_path: '${target_lower_path}'"
+            target_lower_dir=$(dirname ${target_lower_path})
+            echo "target_lower_dir: '${target_lower_dir}'"
+            mkdir -p ${LOWER_DIRS}/${target_lower_dir}
+            cp -a ${replace} ${LOWER_DIRS}/${target_lower_dir}/.
+            ls -lisa ${LOWER_DIRS}/${target_lower_dir}
+            ADD_LOWER_DIRS=1
+        else
+            echo "replacement file does NOT exist; ignoring replacement"
+        fi
+    done
+fi
+if [[ ${ADD_LOWER_DIRS} -eq 1 ]]; then
+    BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
+fi
+
 # create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)
 
diff --git a/replace_files.txt b/replace_files.txt
new file mode 100644
index 0000000000..bb29d2ea26
--- /dev/null
+++ b/replace_files.txt
@@ -0,0 +1 @@
+__EESSI_SOFTWARE_PATH__/Python/3.11.3-GCCcore-12.3.0/lib/python3.11/ctypes/util.py replacement_files/ctypes/util.py
diff --git a/replacement_files/ctypes/util.py b/replacement_files/ctypes/util.py
new file mode 100644
index 0000000000..b4cb4becb0
--- /dev/null
+++ b/replacement_files/ctypes/util.py
@@ -0,0 +1,379 @@
+import os
+import shutil
+import subprocess
+import sys
+
+# find_library(name) returns the pathname of a library, or None.
+if os.name == "nt":
+
+    def _get_build_version():
+        """Return the version of MSVC that was used to build Python.
+
+        For Python 2.3 and up, the version number is included in
+        sys.version.  For earlier versions, assume the compiler is MSVC 6.
+        """
+        # This function was copied from Lib/distutils/msvccompiler.py
+        prefix = "MSC v."
+        i = sys.version.find(prefix)
+        if i == -1:
+            return 6
+        i = i + len(prefix)
+        s, rest = sys.version[i:].split(" ", 1)
+        majorVersion = int(s[:-2]) - 6
+        if majorVersion >= 13:
+            majorVersion += 1
+        minorVersion = int(s[2:3]) / 10.0
+        # I don't think paths are affected by minor version in version 6
+        if majorVersion == 6:
+            minorVersion = 0
+        if majorVersion >= 6:
+            return majorVersion + minorVersion
+        # else we don't know what version of the compiler this is
+        return None
+
+    def find_msvcrt():
+        """Return the name of the VC runtime dll"""
+        version = _get_build_version()
+        if version is None:
+            # better be safe than sorry
+            return None
+        if version <= 6:
+            clibname = 'msvcrt'
+        elif version <= 13:
+            clibname = 'msvcr%d' % (version * 10)
+        else:
+            # CRT is no longer directly loadable. See issue23606 for the
+            # discussion about alternative approaches.
+            return None
+
+        # If python was built with in debug mode
+        import importlib.machinery
+        if '_d.pyd' in importlib.machinery.EXTENSION_SUFFIXES:
+            clibname += 'd'
+        return clibname+'.dll'
+
+    def find_library(name):
+        if name in ('c', 'm'):
+            return find_msvcrt()
+        # See MSDN for the REAL search order.
+        for directory in os.environ['PATH'].split(os.pathsep):
+            fname = os.path.join(directory, name)
+            if os.path.isfile(fname):
+                return fname
+            if fname.lower().endswith(".dll"):
+                continue
+            fname = fname + ".dll"
+            if os.path.isfile(fname):
+                return fname
+        return None
+
+elif os.name == "posix" and sys.platform == "darwin":
+    from ctypes.macholib.dyld import dyld_find as _dyld_find
+    def find_library(name):
+        possible = ['lib%s.dylib' % name,
+                    '%s.dylib' % name,
+                    '%s.framework/%s' % (name, name)]
+        for name in possible:
+            try:
+                return _dyld_find(name)
+            except ValueError:
+                continue
+        return None
+
+elif sys.platform.startswith("aix"):
+    # AIX has two styles of storing shared libraries
+    # GNU auto_tools refer to these as svr4 and aix
+    # svr4 (System V Release 4) is a regular file, often with .so as suffix
+    # AIX style uses an archive (suffix .a) with members (e.g., shr.o, libssl.so)
+    # see issue#26439 and _aix.py for more details
+
+    from ctypes._aix import find_library
+
+elif os.name == "posix":
+    # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump
+    import re, tempfile
+
+    def _is_elf(filename):
+        "Return True if the given file is an ELF file"
+        elf_header = b'\x7fELF'
+        with open(filename, 'br') as thefile:
+            return thefile.read(4) == elf_header
+
+    def _findLib_gcc(name):
+        # Run GCC's linker with the -t (aka --trace) option and examine the
+        # library name it prints out. The GCC command will fail because we
+        # haven't supplied a proper program with main(), but that does not
+        # matter.
+        expr = os.fsencode(r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name))
+
+        c_compiler = shutil.which('gcc')
+        if not c_compiler:
+            c_compiler = shutil.which('cc')
+        if not c_compiler:
+            # No C compiler available, give up
+            return None
+
+        temp = tempfile.NamedTemporaryFile()
+        try:
+            args = [c_compiler, '-Wl,-t', '-o', temp.name, '-l' + name]
+
+            env = dict(os.environ)
+            env['LC_ALL'] = 'C'
+            env['LANG'] = 'C'
+            try:
+                proc = subprocess.Popen(args,
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.STDOUT,
+                                        env=env)
+            except OSError:  # E.g. bad executable
+                return None
+            with proc:
+                trace = proc.stdout.read()
+        finally:
+            try:
+                temp.close()
+            except FileNotFoundError:
+                # Raised if the file was already removed, which is the normal
+                # behaviour of GCC if linking fails
+                pass
+        res = re.findall(expr, trace)
+        if not res:
+            return None
+
+        for file in res:
+            # Check if the given file is an elf file: gcc can report
+            # some files that are linker scripts and not actual
+            # shared objects. See bpo-41976 for more details
+            if not _is_elf(file):
+                continue
+            return os.fsdecode(file)
+
+
+    if sys.platform == "sunos5":
+        # use /usr/ccs/bin/dump on solaris
+        def _get_soname(f):
+            if not f:
+                return None
+
+            try:
+                proc = subprocess.Popen(("/usr/ccs/bin/dump", "-Lpv", f),
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.DEVNULL)
+            except OSError:  # E.g. command not found
+                return None
+            with proc:
+                data = proc.stdout.read()
+            res = re.search(br'\[.*\]\sSONAME\s+([^\s]+)', data)
+            if not res:
+                return None
+            return os.fsdecode(res.group(1))
+    else:
+        def _get_soname(f):
+            # assuming GNU binutils / ELF
+            if not f:
+                return None
+            objdump = shutil.which('objdump')
+            if not objdump:
+                # objdump is not available, give up
+                return None
+
+            try:
+                proc = subprocess.Popen((objdump, '-p', '-j', '.dynamic', f),
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.DEVNULL)
+            except OSError:  # E.g. bad executable
+                return None
+            with proc:
+                dump = proc.stdout.read()
+            res = re.search(br'\sSONAME\s+([^\s]+)', dump)
+            if not res:
+                return None
+            return os.fsdecode(res.group(1))
+
+    if sys.platform.startswith(("freebsd", "openbsd", "dragonfly")):
+
+        def _num_version(libname):
+            # "libxyz.so.MAJOR.MINOR" => [ MAJOR, MINOR ]
+            parts = libname.split(b".")
+            nums = []
+            try:
+                while parts:
+                    nums.insert(0, int(parts.pop()))
+            except ValueError:
+                pass
+            return nums or [sys.maxsize]
+
+        def find_library(name):
+            ename = re.escape(name)
+            expr = r':-l%s\.\S+ => \S*/(lib%s\.\S+)' % (ename, ename)
+            expr = os.fsencode(expr)
+
+            try:
+                proc = subprocess.Popen(('/sbin/ldconfig', '-r'),
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.DEVNULL)
+            except OSError:  # E.g. command not found
+                data = b''
+            else:
+                with proc:
+                    data = proc.stdout.read()
+
+            res = re.findall(expr, data)
+            if not res:
+                return _get_soname(_findLib_gcc(name))
+            res.sort(key=_num_version)
+            return os.fsdecode(res[-1])
+
+    elif sys.platform == "sunos5":
+
+        def _findLib_crle(name, is64):
+            if not os.path.exists('/usr/bin/crle'):
+                return None
+
+            env = dict(os.environ)
+            env['LC_ALL'] = 'C'
+
+            if is64:
+                args = ('/usr/bin/crle', '-64')
+            else:
+                args = ('/usr/bin/crle',)
+
+            paths = None
+            try:
+                proc = subprocess.Popen(args,
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.DEVNULL,
+                                        env=env)
+            except OSError:  # E.g. bad executable
+                return None
+            with proc:
+                for line in proc.stdout:
+                    line = line.strip()
+                    if line.startswith(b'Default Library Path (ELF):'):
+                        paths = os.fsdecode(line).split()[4]
+
+            if not paths:
+                return None
+
+            for dir in paths.split(":"):
+                libfile = os.path.join(dir, "lib%s.so" % name)
+                if os.path.exists(libfile):
+                    return libfile
+
+            return None
+
+        def find_library(name, is64 = False):
+            return _get_soname(_findLib_crle(name, is64) or _findLib_gcc(name))
+
+    else:
+
+        def _findSoname_ldconfig(name):
+            import struct
+            if struct.calcsize('l') == 4:
+                machine = os.uname().machine + '-32'
+            else:
+                machine = os.uname().machine + '-64'
+            mach_map = {
+                'x86_64-64': 'libc6,x86-64',
+                'ppc64-64': 'libc6,64bit',
+                'sparc64-64': 'libc6,64bit',
+                's390x-64': 'libc6,64bit',
+                'ia64-64': 'libc6,IA-64',
+                }
+            abi_type = mach_map.get(machine, 'libc6')
+
+            # XXX assuming GLIBC's ldconfig (with option -p)
+            regex = r'\s+(lib%s\.[^\s]+)\s+\(%s'
+            regex = os.fsencode(regex % (re.escape(name), abi_type))
+            try:
+                with subprocess.Popen(['/sbin/ldconfig', '-p'],
+                                      stdin=subprocess.DEVNULL,
+                                      stderr=subprocess.DEVNULL,
+                                      stdout=subprocess.PIPE,
+                                      env={'LC_ALL': 'C', 'LANG': 'C'}) as p:
+                    res = re.search(regex, p.stdout.read())
+                    if res:
+                        return os.fsdecode(res.group(1))
+            except OSError:
+                pass
+
+        def _findLib_ld(name):
+            # See issue #9998 for why this is needed
+            expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)
+            cmd = ['ld', '-t']
+            libpath = os.environ.get('LD_LIBRARY_PATH')
+            if libpath:
+                for d in libpath.split(':'):
+                    cmd.extend(['-L', d])
+            cmd.extend(['-o', os.devnull, '-l%s' % name])
+            result = None
+            try:
+                p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+                                     stderr=subprocess.PIPE,
+                                     universal_newlines=True)
+                out, _ = p.communicate()
+                res = re.findall(expr, os.fsdecode(out))
+                for file in res:
+                    # Check if the given file is an elf file: gcc can report
+                    # some files that are linker scripts and not actual
+                    # shared objects. See bpo-41976 for more details
+                    if not _is_elf(file):
+                        continue
+                    return os.fsdecode(file)
+            except Exception:
+                pass  # result will be None
+            return result
+
+        def find_library(name):
+            # See issue #9998
+            lib = _findLib_gcc(name)
+            # return absolute path
+            return _findSoname_ldconfig(name) or \
+                    os.path.join(os.path.dirname(lib), _get_soname(lib)) or \
+                   _get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name))
+
+################################################################
+# test code
+
+def test():
+    from ctypes import cdll
+    if os.name == "nt":
+        print(cdll.msvcrt)
+        print(cdll.load("msvcrt"))
+        print(find_library("msvcrt"))
+
+    if os.name == "posix":
+        # find and load_version
+        print(find_library("m"))
+        print(find_library("c"))
+        print(find_library("bz2"))
+
+        # load
+        if sys.platform == "darwin":
+            print(cdll.LoadLibrary("libm.dylib"))
+            print(cdll.LoadLibrary("libcrypto.dylib"))
+            print(cdll.LoadLibrary("libSystem.dylib"))
+            print(cdll.LoadLibrary("System.framework/System"))
+        # issue-26439 - fix broken test call for AIX
+        elif sys.platform.startswith("aix"):
+            from ctypes import CDLL
+            if sys.maxsize < 2**32:
+                print(f"Using CDLL(name, os.RTLD_MEMBER): {CDLL('libc.a(shr.o)', os.RTLD_MEMBER)}")
+                print(f"Using cdll.LoadLibrary(): {cdll.LoadLibrary('libc.a(shr.o)')}")
+                # librpm.so is only available as 32-bit shared library
+                print(find_library("rpm"))
+                print(cdll.LoadLibrary("librpm.so"))
+            else:
+                print(f"Using CDLL(name, os.RTLD_MEMBER): {CDLL('libc.a(shr_64.o)', os.RTLD_MEMBER)}")
+                print(f"Using cdll.LoadLibrary(): {cdll.LoadLibrary('libc.a(shr_64.o)')}")
+            print(f"crypt\t:: {find_library('crypt')}")
+            print(f"crypt\t:: {cdll.LoadLibrary(find_library('crypt'))}")
+            print(f"crypto\t:: {find_library('crypto')}")
+            print(f"crypto\t:: {cdll.LoadLibrary(find_library('crypto'))}")
+        else:
+            print(cdll.LoadLibrary("libm.so"))
+            print(cdll.LoadLibrary("libcrypt.so"))
+            print(find_library("crypt"))
+
+if __name__ == "__main__":
+    test()

From cd474d8142b58522f4ba69dc03f87657f801930c Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Thu, 30 May 2024 22:58:54 +0200
Subject: [PATCH 04/17] {2023.06}[foss/2023a] Python-bundle v2.1.2 with
 CUDA/12.1.1

---
 .../pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml
index 4f31c4dd08..61fb568aaa 100644
--- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml
+++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml
@@ -55,3 +55,4 @@ easyconfigs:
   - PyTorch-2.1.2-foss-2023a-CUDA-12.1.1.eb:
       options:
         cuda-compute-capabilities: 6.0,6.1,7.0,7.5,8.0,8.6,8.9,9.0
+  - PyTorch-bundle-2.1.2-foss-2023a-CUDA-12.1.1.eb

From 1cfcd069a9f8384c29e721ece706d220bfe7541c Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Thu, 30 May 2024 23:40:05 +0200
Subject: [PATCH 05/17] tweak lower_dirs code and use from-pr

---
 bot/build.sh                                           | 10 +++++++---
 .../2023.06/eessi-2023.06-eb-4.9.1-2023a.yml           |  5 ++++-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index 258766f802..d182193caf 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -287,10 +287,11 @@ fi
 #   - check if the target exists in the repository
 #   - create directory for replacement
 #   - copy target into directory
-ADD_LOWER_DIRS=0
+ADD_LOWER_DIRS=
 if [[ -f "replace_files.txt" ]]; then
     LOWER_DIRS="${STORAGE}/lower_dirs"
     mkdir -p "${LOWER_DIRS}"
+    echo "LOWER_DIRS: '${LOWER_DIRS}'"
 
     repo_name=${EESSI_CVMFS_REPO_OVERRIDE}
     repo_version=${EESSI_VERSION_OVERRIDE}
@@ -315,14 +316,17 @@ if [[ -f "replace_files.txt" ]]; then
             mkdir -p ${LOWER_DIRS}/${target_lower_dir}
             cp -a ${replace} ${LOWER_DIRS}/${target_lower_dir}/.
             ls -lisa ${LOWER_DIRS}/${target_lower_dir}
-            ADD_LOWER_DIRS=1
+            ADD_LOWER_DIRS=${LOWER_DIRS}
         else
             echo "replacement file does NOT exist; ignoring replacement"
         fi
     done
 fi
-if [[ ${ADD_LOWER_DIRS} -eq 1 ]]; then
+if [[ ! -z ${ADD_LOWER_DIRS} ]]; then
     BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
+    echo "Added '--lower-dirs ${LOWER_DIRS}' to build step arguments"
+echo
+    echo "Nothing to be added for LOWER_DIRS (${ADD_LOWER_DIRS})"
 fi
 
 # create tmp file for output of build step
diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml
index 61fb568aaa..2ac97f3a36 100644
--- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml
+++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml
@@ -55,4 +55,7 @@ easyconfigs:
   - PyTorch-2.1.2-foss-2023a-CUDA-12.1.1.eb:
       options:
         cuda-compute-capabilities: 6.0,6.1,7.0,7.5,8.0,8.6,8.9,9.0
-  - PyTorch-bundle-2.1.2-foss-2023a-CUDA-12.1.1.eb
+  - PyTorch-bundle-2.1.2-foss-2023a-CUDA-12.1.1.eb:
+      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20484
+      options:
+        from-pr: 20484

From 6e459226a890f3f986a6fa840ba1bc019c6b9722 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 31 May 2024 00:00:40 +0200
Subject: [PATCH 06/17] fix syntax + more debug output

---
 bot/build.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/bot/build.sh b/bot/build.sh
index d182193caf..cde60b3d6c 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -322,10 +322,12 @@ if [[ -f "replace_files.txt" ]]; then
         fi
     done
 fi
+echo "LOWER_DIRS: '${LOWER_DIRS}'"
+echo "ADD_LOWER_DIRS: '${ADD_LOWER_DIRS}'"
 if [[ ! -z ${ADD_LOWER_DIRS} ]]; then
     BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
     echo "Added '--lower-dirs ${LOWER_DIRS}' to build step arguments"
-echo
+else
     echo "Nothing to be added for LOWER_DIRS (${ADD_LOWER_DIRS})"
 fi
 

From 7f6177140540c923f9c30278ee6f08493dd615e5 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 31 May 2024 00:11:56 +0200
Subject: [PATCH 07/17] change logic for adding lower dirs

---
 bot/build.sh | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index cde60b3d6c..9a48b5df16 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -287,7 +287,7 @@ fi
 #   - check if the target exists in the repository
 #   - create directory for replacement
 #   - copy target into directory
-ADD_LOWER_DIRS=
+rm -f ADD_LOWER_DIRS
 if [[ -f "replace_files.txt" ]]; then
     LOWER_DIRS="${STORAGE}/lower_dirs"
     mkdir -p "${LOWER_DIRS}"
@@ -316,19 +316,18 @@ if [[ -f "replace_files.txt" ]]; then
             mkdir -p ${LOWER_DIRS}/${target_lower_dir}
             cp -a ${replace} ${LOWER_DIRS}/${target_lower_dir}/.
             ls -lisa ${LOWER_DIRS}/${target_lower_dir}
-            ADD_LOWER_DIRS=${LOWER_DIRS}
+            touch ADD_LOWER_DIRS
         else
             echo "replacement file does NOT exist; ignoring replacement"
         fi
     done
 fi
 echo "LOWER_DIRS: '${LOWER_DIRS}'"
-echo "ADD_LOWER_DIRS: '${ADD_LOWER_DIRS}'"
-if [[ ! -z ${ADD_LOWER_DIRS} ]]; then
+if [[ -f ADD_LOWER_DIRS ]]; then
     BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
     echo "Added '--lower-dirs ${LOWER_DIRS}' to build step arguments"
 else
-    echo "Nothing to be added for LOWER_DIRS (${ADD_LOWER_DIRS})"
+    echo "Nothing to be added for LOWER_DIRS"
 fi
 
 # create tmp file for output of build step

From bfdf0ab4c47d257aeb2d5c801fe20864deda3e4d Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 31 May 2024 09:56:36 +0200
Subject: [PATCH 08/17] rename replace file to deactivate replacement

---
 replace_files.txt => _replace_files.txt | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename replace_files.txt => _replace_files.txt (100%)

diff --git a/replace_files.txt b/_replace_files.txt
similarity index 100%
rename from replace_files.txt
rename to _replace_files.txt

From f95918f635d1c4701469e5d485cf71d03f5f2e86 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 5 Jun 2024 00:38:55 +0200
Subject: [PATCH 09/17] add changes from librosa PR

---
 EESSI-install-software.sh              |  5 ++
 eb_hooks.py                            | 63 +++++++++++++++++
 install_scripts.sh                     |  6 ++
 scripts/extra/install_custom_ctypes.sh | 96 ++++++++++++++++++++++++++
 4 files changed, 170 insertions(+)
 create mode 100755 scripts/extra/install_custom_ctypes.sh

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index ca6fed71d5..7d6cad7add 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -256,6 +256,11 @@ if command_exists "nvidia-smi"; then
     ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh
 fi
 
+# Install extra software that is needed (e.g., for providing a custom ctypes
+# library when needed)
+echo "Location of host_injections: $(ls -l ${EESSI_CVMFS_REPO}/host_injections)"
+${EESSI_PREFIX}/scripts/extra/install_custom_ctypes.sh --temp-dir /tmp/temp
+
 # use PR patch file to determine in which easystack files stuff was added
 changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') 
 if [ -z "${changed_easystacks}" ]; then
diff --git a/eb_hooks.py b/eb_hooks.py
index e4a957acdc..14c7c30d91 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -5,6 +5,7 @@
 
 import easybuild.tools.environment as env
 from easybuild.easyblocks.generic.configuremake import obtain_config_guess
+from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES
 from easybuild.framework.easyconfig.constants import EASYCONFIG_CONSTANTS
 from easybuild.tools.build_log import EasyBuildError, print_msg
 from easybuild.tools.config import build_option, update_build_option
@@ -349,6 +350,32 @@ def parse_hook_lammps_remove_deps_for_CI_aarch64(ec, *args, **kwargs):
         raise EasyBuildError("LAMMPS-specific hook triggered for non-LAMMPS easyconfig?!")
 
 
+def parse_hook_librosa_custom_ctypes(ec, *args, **kwargs):
+    """
+    Add exts_filter to soundfile extension in exts_list
+    """
+    if ec.name == 'librosa' and ec.version in ('0.10.1',):
+        ec_dict = ec.asdict()
+        eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH')
+        custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1)
+        custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1)
+        custom_ctypes_path = os.path.join(custom_ctypes_path, "custom_ctypes")
+        ebpythonprefixes = "EBPYTHONPREFIXES=%s" % custom_ctypes_path
+        exts_list_new = []
+        for item in ec_dict['exts_list']:
+            if item[0] == 'soundfile':
+                ext_dict = item[2]
+                ext_dict['exts_filter'] = (ebpythonprefixes + ' ' + EXTS_FILTER_PYTHON_PACKAGES[0],
+                                           EXTS_FILTER_PYTHON_PACKAGES[1])
+                exts_list_new.append((item[0], item[1], ext_dict))
+            else:
+                exts_list_new.append(item)
+        ec['exts_list'] = exts_list_new
+        print_msg("New exts_list: '%s'", ec['exts_list'])
+    else:
+        raise EasyBuildError("librosa/0.10.1-specific hook triggered for non-librosa/0.10.1 easyconfig?!")
+
+
 def pre_prepare_hook_highway_handle_test_compilation_issues(self, *args, **kwargs):
     """
     Solve issues with compiling or running the tests on both
@@ -852,6 +879,37 @@ def inject_gpu_property(ec):
     return ec
 
 
+def pre_module_hook(self, *args, **kwargs):
+    """Main pre-module-check hook: trigger custom functions based on software name."""
+    if self.name in PRE_MODULE_HOOKS:
+        PRE_MODULE_HOOKS[self.name](self, *args, **kwargs)
+
+
+def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs):
+    """
+    Add EBPYTHONPREFIXES to modluafooter
+    """
+    if self.name == 'librosa' and self.version == '0.10.1':
+        eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH')
+        custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1)
+        custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1)
+        custom_ctypes_path = os.path.join(custom_ctypes_path, 'custom_ctypes')
+        key = 'modluafooter'
+        values = ['prepend_path("EBPYTHONPREFIXES","%s")' % (custom_ctypes_path)]
+        print_msg("Adding '%s' to modluafooter", values[0])
+        if not key in self.cfg:
+            self.cfg[key] = '\n'.join(values)
+        else:
+            new_value = self.cfg[key]
+            for value in values:
+                if not value in new_value:
+                    new_value = '\n'.join([new_value, value])
+            self.cfg[key] = new_value
+        print_msg("Full modluafooter is '%s'", self.cfg[key])
+    else:
+        raise EasyBuildError("librosa/0.10.1-specific hook triggered for non-librosa/0.10.1 easyconfig?!")
+
+
 PARSE_HOOKS = {
     'casacore': parse_hook_casacore_disable_vectorize,
     'CGAL': parse_hook_cgal_toolchainopts_precise,
@@ -859,6 +917,7 @@ def inject_gpu_property(ec):
     'GPAW': parse_hook_gpaw_harcoded_path,
     'ImageMagick': parse_hook_imagemagick_add_dependency,
     'LAMMPS': parse_hook_lammps_remove_deps_for_CI_aarch64,
+    'librosa': parse_hook_librosa_custom_ctypes,
     'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors,
     'Pillow-SIMD' : parse_hook_Pillow_SIMD_harcoded_paths,
     'pybind11': parse_hook_pybind11_replace_catch2,
@@ -909,3 +968,7 @@ def inject_gpu_property(ec):
     'cuDNN': post_sanitycheck_cudnn,
     'cuTENSOR': post_sanitycheck_cutensor,
 }
+
+PRE_MODULE_HOOKS = {
+    'librosa': pre_module_hook_librosa_augment_modluafooter,
+}
diff --git a/install_scripts.sh b/install_scripts.sh
index 07643a39e6..000ad99444 100755
--- a/install_scripts.sh
+++ b/install_scripts.sh
@@ -116,6 +116,12 @@ nvidia_files=(
 )
 copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}"
 
+# Copy files for the scripts/extra directory
+extra_files=(
+    install_custom_ctypes.sh
+)
+copy_files_by_list ${TOPDIR}/scripts/extra ${INSTALL_PREFIX}/scripts/extra "${extra_files[@]}"
+
 # Copy over EasyBuild hooks file used for installations
 hook_files=(
     eb_hooks.py
diff --git a/scripts/extra/install_custom_ctypes.sh b/scripts/extra/install_custom_ctypes.sh
new file mode 100755
index 0000000000..6392da00e5
--- /dev/null
+++ b/scripts/extra/install_custom_ctypes.sh
@@ -0,0 +1,96 @@
+#!/usr/bin/env bash
+
+# This script can be used to install custom ctypes under
+# CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/software/custom_ctypes
+# and then set EESSI_USE_CUSTOM_CTYPES_DIR=CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/custom_ctypes
+# before loading a module that needs the custom ctypes implementation
+# The custom ctypes is downloaded from https://github.com/NorESSI/custom_ctypes which is a fork of
+# https://github.com/ComputeCanada/custom_ctypes
+
+# The `host_injections` directory is a variant symlink that by default points to
+# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see
+# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the
+# installation to be successful, this directory needs to be writeable by the user
+# executing this script.
+
+# some logging
+echo ">>> Running ${BASH_SOURCE}"
+
+# Initialise our bash functions
+TOPDIR=$(dirname $(realpath ${BASH_SOURCE}))
+source "${TOPDIR}"/../utils.sh
+
+# Function to display help message
+show_help() {
+    echo "Usage: $0 [OPTIONS]"
+    echo "Options:"
+    echo "  --help                           Display this help message"
+    echo "  -t, --temp-dir /path/to/tmpdir   Specify a location to use for temporary"
+    echo "                                   storage during the installation"
+}
+
+# Initialize variables
+TEMP_DIR=
+
+# Parse command-line options
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --help)
+            show_help
+            exit 0
+            ;;
+        -t|--temp-dir)
+            if [ -n "$2" ]; then
+                TEMP_DIR="$2"
+                shift 2
+            else
+                echo "Error: Argument required for $1"
+                show_help
+                exit 1
+            fi
+            ;;
+        *)
+            show_help
+            fatal_error "Error: Unknown option: $1"
+            ;;
+    esac
+done
+
+# Make sure NESSI is initialised
+check_eessi_initialised
+
+# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` and
+# `software` with `extra`
+# also append `/custom_ctypes`
+NESSI_SITE_INSTALL=${EESSI_SOFTWARE_PATH/versions/host_injections}
+NESSI_SITE_INSTALL=${NESSI_SITE_INSTALL/software/extra}/custom_ctypes
+
+# we need a directory we can use for temporary storage
+if [[ -z "${TEMP_DIR}" ]]; then
+    tmpdir=$(mktemp -d)
+else
+    mkdir -p ${TEMP_DIR}
+    tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} custom_ctypes.XXX)
+    if [[ ! -d "$tmpdir" ]] ; then
+        fatal_error "Could not create directory ${tmpdir}"
+    fi
+fi
+echo "Created temporary directory '${tmpdir}'"
+
+# check if custom_ctypes has already been installed
+if [[ -d ${NESSI_SITE_INSTALL}/lib ]]; then
+    echo "INFO: Installation of custom_ctypes already found at '${NESSI_SITE_INSTALL}'"
+    exit 0
+fi
+
+# download custom_ctypes to temp directory
+wget https://github.com/NorESSI/custom_ctypes/archive/refs/heads/main.tar.gz -P ${tmpdir}
+
+# make sure target directory exists
+mkdir -p ${NESSI_SITE_INSTALL}
+
+# unpack custom_ctypes to target directory
+tar xvfz ${tmpdir}/main.tar.gz --strip-components=1 -C ${NESSI_SITE_INSTALL}
+
+# clean up tmpdir
+rm -rf "${tmpdir}"

From ebf5508d404c323d9016bce6003eed956b3efb39 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 5 Jun 2024 06:14:56 +0200
Subject: [PATCH 10/17] add cuda compute capabilities

---
 .../pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml      | 1 +
 1 file changed, 1 insertion(+)

diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml
index 2ac97f3a36..762f1a236f 100644
--- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml
+++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml
@@ -59,3 +59,4 @@ easyconfigs:
       # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20484
       options:
         from-pr: 20484
+        cuda-compute-capabilities: 6.0,6.1,7.0,7.5,8.0,8.6,8.9,9.0

From 8c1daf1e128afc6272c855ffa25eabb6b377adbb Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 5 Jun 2024 19:44:30 +0200
Subject: [PATCH 11/17] minor updates

---
 EESSI-install-software.sh | 10 +++++-----
 create_lmodsitepackage.py |  1 +
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index 7d6cad7add..d030a1f798 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -17,11 +17,6 @@ display_help() {
   echo "  --skip-cuda-install    -  disable installing a full CUDA SDK in the host_injections prefix (e.g. in CI)"
 }
 
-# Function to check if a command exists
-function command_exists() {
-    command -v "$1" >/dev/null 2>&1
-}
-
 function copy_build_log() {
     # copy specified build log to specified directory, with some context added
     build_log=${1}
@@ -159,8 +154,13 @@ fi
 #   are:
 #     - .lmod/lmodrc.lua
 #     - .lmod/SitePackage.lua
+#
 # We run scripts to create them if they don't exist or if the scripts have been
 # changed in the PR.
+#
+# (TODO do we need to change the path if we have sub-directories for
+# accelerators? And would we need different scripts for creating lua files under
+# different directories?)
 
 # Set base directory for software and for Lmod config files
 _eessi_software_path=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py
index 20c4098b8f..585c86ba8f 100755
--- a/create_lmodsitepackage.py
+++ b/create_lmodsitepackage.py
@@ -120,6 +120,7 @@
         -- simpleName is a module in packagesList
         -- get the full host_injections path
         local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections')
+
         -- build final path where the software should be installed
         local packageEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild"
         local packageDirExists = isDir(packageEasyBuildDir)

From a92de9751103acf18dbae56811cb24cb2eb65172 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Wed, 5 Jun 2024 23:26:46 +0200
Subject: [PATCH 12/17] update method and location to install custom_ctypes

---
 EESSI-install-software.sh                     |  5 +-
 eb_hooks.py                                   |  8 +-
 install_scripts.sh                            |  6 --
 scripts/extra/custom_ctypes-1.2.eb            | 29 ++++++
 .../extra/eessi-2023.06-extra-packages.yml    |  2 +
 scripts/extra/install_custom_ctypes.sh        | 96 -------------------
 scripts/extra/install_extra_packages.sh       | 95 ++++++++++++++++++
 7 files changed, 131 insertions(+), 110 deletions(-)
 create mode 100644 scripts/extra/custom_ctypes-1.2.eb
 create mode 100644 scripts/extra/eessi-2023.06-extra-packages.yml
 delete mode 100755 scripts/extra/install_custom_ctypes.sh
 create mode 100755 scripts/extra/install_extra_packages.sh

diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh
index d030a1f798..201ae6f8cb 100755
--- a/EESSI-install-software.sh
+++ b/EESSI-install-software.sh
@@ -258,8 +258,9 @@ fi
 
 # Install extra software that is needed (e.g., for providing a custom ctypes
 # library when needed)
-echo "Location of host_injections: $(ls -l ${EESSI_CVMFS_REPO}/host_injections)"
-${EESSI_PREFIX}/scripts/extra/install_custom_ctypes.sh --temp-dir /tmp/temp
+cd ${TOPDIR}/scripts/extra
+./install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml
+cd ${TOPDIR}
 
 # use PR patch file to determine in which easystack files stuff was added
 changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') 
diff --git a/eb_hooks.py b/eb_hooks.py
index 14c7c30d91..58523f160b 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -357,9 +357,7 @@ def parse_hook_librosa_custom_ctypes(ec, *args, **kwargs):
     if ec.name == 'librosa' and ec.version in ('0.10.1',):
         ec_dict = ec.asdict()
         eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH')
-        custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1)
-        custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1)
-        custom_ctypes_path = os.path.join(custom_ctypes_path, "custom_ctypes")
+        custom_ctypes_path = os.path.join(eessi_software_path, "software", "custom_ctypes", "1.2")
         ebpythonprefixes = "EBPYTHONPREFIXES=%s" % custom_ctypes_path
         exts_list_new = []
         for item in ec_dict['exts_list']:
@@ -891,9 +889,7 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs):
     """
     if self.name == 'librosa' and self.version == '0.10.1':
         eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH')
-        custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1)
-        custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1)
-        custom_ctypes_path = os.path.join(custom_ctypes_path, 'custom_ctypes')
+        custom_ctypes_path = os.path.join(eessi_software_path, "software", "custom_ctypes", "1.2")
         key = 'modluafooter'
         values = ['prepend_path("EBPYTHONPREFIXES","%s")' % (custom_ctypes_path)]
         print_msg("Adding '%s' to modluafooter", values[0])
diff --git a/install_scripts.sh b/install_scripts.sh
index 000ad99444..07643a39e6 100755
--- a/install_scripts.sh
+++ b/install_scripts.sh
@@ -116,12 +116,6 @@ nvidia_files=(
 )
 copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}"
 
-# Copy files for the scripts/extra directory
-extra_files=(
-    install_custom_ctypes.sh
-)
-copy_files_by_list ${TOPDIR}/scripts/extra ${INSTALL_PREFIX}/scripts/extra "${extra_files[@]}"
-
 # Copy over EasyBuild hooks file used for installations
 hook_files=(
     eb_hooks.py
diff --git a/scripts/extra/custom_ctypes-1.2.eb b/scripts/extra/custom_ctypes-1.2.eb
new file mode 100644
index 0000000000..35be6dcc41
--- /dev/null
+++ b/scripts/extra/custom_ctypes-1.2.eb
@@ -0,0 +1,29 @@
+##
+# This is a contribution from the NESSI project
+# Homepage: 	https://documentation.sigma2.no
+#
+# Authors::	Thomas Roeblitz <thomas.roblitz@uib.no>
+# License::	GPL-2.0-only
+#
+##
+
+easyblock = 'Tarball'
+
+name = 'custom_ctypes'
+version = '1.2'
+
+homepage = 'https://github.com/ComputeCanada/custom_ctypes'
+description = """custum_ctypes is a small Python package to fix the discovery of libraries with Python's ctypes module. It changes the behavior of find_library to return absolute paths to shared objects rather than just the names."""
+
+toolchain = SYSTEM
+
+source_urls = ['https://github.com/ComputeCanada/custom_ctypes/archive/refs/tags']
+sources = ['%(version)s.tar.gz']
+checksums = ['3b30ce633c6a329169f2b10ff24b8eaaeef3fa208a66cdacdb53c22f02a88d9b']
+
+sanity_check_paths = {
+    'files': ['README.md'],
+    'dirs': ['lib'],
+}
+
+moduleclass = 'lib'
diff --git a/scripts/extra/eessi-2023.06-extra-packages.yml b/scripts/extra/eessi-2023.06-extra-packages.yml
new file mode 100644
index 0000000000..22670ec7a3
--- /dev/null
+++ b/scripts/extra/eessi-2023.06-extra-packages.yml
@@ -0,0 +1,2 @@
+easyconfigs:
+  - custom_ctypes-1.2.eb
diff --git a/scripts/extra/install_custom_ctypes.sh b/scripts/extra/install_custom_ctypes.sh
deleted file mode 100755
index 6392da00e5..0000000000
--- a/scripts/extra/install_custom_ctypes.sh
+++ /dev/null
@@ -1,96 +0,0 @@
-#!/usr/bin/env bash
-
-# This script can be used to install custom ctypes under
-# CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/software/custom_ctypes
-# and then set EESSI_USE_CUSTOM_CTYPES_DIR=CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/custom_ctypes
-# before loading a module that needs the custom ctypes implementation
-# The custom ctypes is downloaded from https://github.com/NorESSI/custom_ctypes which is a fork of
-# https://github.com/ComputeCanada/custom_ctypes
-
-# The `host_injections` directory is a variant symlink that by default points to
-# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see
-# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the
-# installation to be successful, this directory needs to be writeable by the user
-# executing this script.
-
-# some logging
-echo ">>> Running ${BASH_SOURCE}"
-
-# Initialise our bash functions
-TOPDIR=$(dirname $(realpath ${BASH_SOURCE}))
-source "${TOPDIR}"/../utils.sh
-
-# Function to display help message
-show_help() {
-    echo "Usage: $0 [OPTIONS]"
-    echo "Options:"
-    echo "  --help                           Display this help message"
-    echo "  -t, --temp-dir /path/to/tmpdir   Specify a location to use for temporary"
-    echo "                                   storage during the installation"
-}
-
-# Initialize variables
-TEMP_DIR=
-
-# Parse command-line options
-while [[ $# -gt 0 ]]; do
-    case "$1" in
-        --help)
-            show_help
-            exit 0
-            ;;
-        -t|--temp-dir)
-            if [ -n "$2" ]; then
-                TEMP_DIR="$2"
-                shift 2
-            else
-                echo "Error: Argument required for $1"
-                show_help
-                exit 1
-            fi
-            ;;
-        *)
-            show_help
-            fatal_error "Error: Unknown option: $1"
-            ;;
-    esac
-done
-
-# Make sure NESSI is initialised
-check_eessi_initialised
-
-# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` and
-# `software` with `extra`
-# also append `/custom_ctypes`
-NESSI_SITE_INSTALL=${EESSI_SOFTWARE_PATH/versions/host_injections}
-NESSI_SITE_INSTALL=${NESSI_SITE_INSTALL/software/extra}/custom_ctypes
-
-# we need a directory we can use for temporary storage
-if [[ -z "${TEMP_DIR}" ]]; then
-    tmpdir=$(mktemp -d)
-else
-    mkdir -p ${TEMP_DIR}
-    tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} custom_ctypes.XXX)
-    if [[ ! -d "$tmpdir" ]] ; then
-        fatal_error "Could not create directory ${tmpdir}"
-    fi
-fi
-echo "Created temporary directory '${tmpdir}'"
-
-# check if custom_ctypes has already been installed
-if [[ -d ${NESSI_SITE_INSTALL}/lib ]]; then
-    echo "INFO: Installation of custom_ctypes already found at '${NESSI_SITE_INSTALL}'"
-    exit 0
-fi
-
-# download custom_ctypes to temp directory
-wget https://github.com/NorESSI/custom_ctypes/archive/refs/heads/main.tar.gz -P ${tmpdir}
-
-# make sure target directory exists
-mkdir -p ${NESSI_SITE_INSTALL}
-
-# unpack custom_ctypes to target directory
-tar xvfz ${tmpdir}/main.tar.gz --strip-components=1 -C ${NESSI_SITE_INSTALL}
-
-# clean up tmpdir
-rm -rf "${tmpdir}"
diff --git a/scripts/extra/install_extra_packages.sh b/scripts/extra/install_extra_packages.sh
new file mode 100755
index 0000000000..ccd2890864
--- /dev/null
+++ b/scripts/extra/install_extra_packages.sh
@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+
+# This script can be used to install extra packages under ${EESSI_SOFTWARE_PATH}
+
+# some logging
+echo ">>> Running ${BASH_SOURCE}"
+
+# Initialise our bash functions
+TOPDIR=$(dirname $(realpath ${BASH_SOURCE}))
+source "${TOPDIR}"/../utils.sh
+
+# Function to display help message
+show_help() {
+    echo "Usage: $0 [OPTIONS]"
+    echo "Options:"
+    echo "  --help                           Display this help message"
+    echo "  -e, --easystack EASYSTACKFILE    Easystack file which specifies easyconfigs to be installed."
+    echo "  -t, --temp-dir /path/to/tmpdir   Specify a location to use for temporary"
+    echo "                                   storage during the installation"
+}
+
+# Initialize variables
+TEMP_DIR=
+EASYSTACK_FILE=
+
+# Parse command-line options
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --help)
+            show_help
+            exit 0
+            ;;
+        -e|--easystack)
+            if [ -n "$2" ]; then
+                EASYSTACK_FILE="$2"
+                shift 2
+            else
+                echo "Error: Argument required for $1"
+                show_help
+                exit 1
+            fi
+            ;;
+        -t|--temp-dir)
+            if [ -n "$2" ]; then
+                TEMP_DIR="$2"
+                shift 2
+            else
+                echo "Error: Argument required for $1"
+                show_help
+                exit 1
+            fi
+            ;;
+        *)
+            show_help
+            fatal_error "Error: Unknown option: $1"
+            ;;
+    esac
+done
+
+if [[ -z ${EASYSTACK_FILE} ]]; then
+    show_help
+    fatal_error "Error: need to specify easystack file"
+fi
+
+# Make sure NESSI is initialised
+check_eessi_initialised
+
+# As an installation location just use $EESSI_SOFTWARE_PATH
+export NESSI_CVMFS_INSTALL=${EESSI_SOFTWARE_PATH}
+
+# we need a directory we can use for temporary storage
+if [[ -z "${TEMP_DIR}" ]]; then
+    tmpdir=$(mktemp -d)
+else
+    mkdir -p ${TEMP_DIR}
+    tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} extra.XXX)
+    if [[ ! -d "$tmpdir" ]] ; then
+        fatal_error "Could not create directory ${tmpdir}"
+    fi
+fi
+echo "Created temporary directory '${tmpdir}'"
+export WORKING_DIR=${tmpdir}
+
+# load EasyBuild
+ml EasyBuild
+
+# load NESSI-extend/2023.06-easybuild
+ml NESSI-extend/2023.06-easybuild
+
+eb --show-config
+
+eb --easystack ${EASYSTACK_FILE} --robot
+
+# clean up tmpdir
+rm -rf "${tmpdir}"

From 11d69921fe7335f84876781a3a446bedf5dc02a1 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 7 Jun 2024 13:50:30 +0200
Subject: [PATCH 13/17] add parse_hook for SentencePiece on aarch64/generic

---
 eb_hooks.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/eb_hooks.py b/eb_hooks.py
index 58523f160b..eaa3e5327e 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -312,6 +312,27 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix):
         raise EasyBuildError("Qt5-specific hook triggered for non-Qt5 easyconfig?!")
 
 
+def parse_hook_sentencepiece_disable_tcmalloc_aarch64(ec, eprefix):
+    """
+    Disable using TCMalloc
+    """
+    cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
+    if ec.name == 'SentencePiece' and ec.version in ['0.2.0'] and cpu_target == CPU_TARGET_AARCH64_GENERIC:
+        print_msg("parse_hook for SentencePiece: OLD '%s'", ec['components'])
+        new_components = []
+        for item in ec['components']:
+            if item[2]['easyblock'] == 'CMakeMake':
+                new_item = item[2]
+                new_item['configopts'] = '-DSPM_ENABLE_TCMALLOC=OFF'
+                new_components.append((item[0], item[1], new_item))
+            else:
+                new_components.append(item)
+        ec['components'] = new_components
+        print_msg("parse_hook for SentencePiece: NEW '%s'", ec['components'])
+    else:
+        raise EasyBuildError("SentencePiece-specific hook triggered for non-SentencePiece easyconfig?!")
+
+
 def parse_hook_ucx_eprefix(ec, eprefix):
     """Make UCX aware of compatibility layer via additional configuration options."""
     if ec.name == 'UCX':
@@ -918,6 +939,7 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs):
     'Pillow-SIMD' : parse_hook_Pillow_SIMD_harcoded_paths,
     'pybind11': parse_hook_pybind11_replace_catch2,
     'Qt5': parse_hook_qt5_check_qtwebengine_disable,
+    'SentencePiece': parse_hook_sentencepiece_disable_tcmalloc_aarch64,
     'UCX': parse_hook_ucx_eprefix,
 }
 

From 18f3672ac2f428786262ca1e952db3e9071c2887 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Fri, 7 Jun 2024 22:16:57 +0200
Subject: [PATCH 14/17] don't raise Error on non-aarch64/generic

---
 eb_hooks.py | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/eb_hooks.py b/eb_hooks.py
index eaa3e5327e..6e2af1e3fb 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -317,18 +317,21 @@ def parse_hook_sentencepiece_disable_tcmalloc_aarch64(ec, eprefix):
     Disable using TCMalloc
     """
     cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
-    if ec.name == 'SentencePiece' and ec.version in ['0.2.0'] and cpu_target == CPU_TARGET_AARCH64_GENERIC:
-        print_msg("parse_hook for SentencePiece: OLD '%s'", ec['components'])
-        new_components = []
-        for item in ec['components']:
-            if item[2]['easyblock'] == 'CMakeMake':
-                new_item = item[2]
-                new_item['configopts'] = '-DSPM_ENABLE_TCMALLOC=OFF'
-                new_components.append((item[0], item[1], new_item))
-            else:
-                new_components.append(item)
-        ec['components'] = new_components
-        print_msg("parse_hook for SentencePiece: NEW '%s'", ec['components'])
+    if ec.name == 'SentencePiece' and ec.version in ['0.2.0']:
+        if cpu_target == CPU_TARGET_AARCH64_GENERIC:
+            print_msg("parse_hook for SentencePiece: OLD '%s'", ec['components'])
+            new_components = []
+            for item in ec['components']:
+                if item[2]['easyblock'] == 'CMakeMake':
+                    new_item = item[2]
+                    new_item['configopts'] = '-DSPM_ENABLE_TCMALLOC=OFF'
+                    new_components.append((item[0], item[1], new_item))
+                else:
+                    new_components.append(item)
+            ec['components'] = new_components
+            print_msg("parse_hook for SentencePiece: NEW '%s'", ec['components'])
+        else:
+            print_msg("parse_hook for SentencePiece on %s -> leaving configopts unchanged", cpu_target)
     else:
         raise EasyBuildError("SentencePiece-specific hook triggered for non-SentencePiece easyconfig?!")
 

From 30d22b4770a003ab20c37bb0a0ff90013a003f3a Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 9 Jun 2024 20:02:22 +0200
Subject: [PATCH 15/17] update with nessi/nessi.no-2023.06

---
 _replace_files.txt                            |   1 -
 bot/build.sh                                  |  52 +--
 bot/inspect.sh                                |  20 +-
 create_lmodsitepackage.py                     |  18 +-
 .../2023.06/eessi-2023.06-eb-4.9.1-2022b.yml  |   2 +
 .../2023.06/eessi-2023.06-eb-4.9.1-2023b.yml  |   2 +
 eb_hooks.py                                   |   4 +-
 install_scripts.sh                            |   1 +
 replacement_files/ctypes/util.py              | 379 ------------------
 scripts/utils.sh                              |   2 +-
 10 files changed, 38 insertions(+), 443 deletions(-)
 delete mode 100644 _replace_files.txt
 delete mode 100644 replacement_files/ctypes/util.py

diff --git a/_replace_files.txt b/_replace_files.txt
deleted file mode 100644
index bb29d2ea26..0000000000
--- a/_replace_files.txt
+++ /dev/null
@@ -1 +0,0 @@
-__EESSI_SOFTWARE_PATH__/Python/3.11.3-GCCcore-12.3.0/lib/python3.11/ctypes/util.py replacement_files/ctypes/util.py
diff --git a/bot/build.sh b/bot/build.sh
index 9a48b5df16..de73faef0b 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -275,61 +275,11 @@ else
 fi
 # Retain location for host injections so we don't reinstall CUDA
 # (Always need to run the driver installation as available driver may change)
+
 if [[ ! -z ${SHARED_FS_PATH} ]]; then
     BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections")
 fi
 
-# replace some files using lower_dirs mechanism
-# - read replacements from replace_files.txt
-#   each line has the format __EESSI_SOFTWARE_PATH__/some_path relative_path
-#   /cvmfs/repo_name/versions/repo_version/software/os_type/software_dir/some_path
-# - for each replacement do
-#   - check if the target exists in the repository
-#   - create directory for replacement
-#   - copy target into directory
-rm -f ADD_LOWER_DIRS
-if [[ -f "replace_files.txt" ]]; then
-    LOWER_DIRS="${STORAGE}/lower_dirs"
-    mkdir -p "${LOWER_DIRS}"
-    echo "LOWER_DIRS: '${LOWER_DIRS}'"
-
-    repo_name=${EESSI_CVMFS_REPO_OVERRIDE}
-    repo_version=${EESSI_VERSION_OVERRIDE}
-    os_type=${EESSI_OS_TYPE}
-    software_subdir_override=${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
-    software_path="/cvmfs/${repo_name}/versions/${repo_version}/software/${os_type}/${software_subdir_override}/software"
-
-    cat replace_files.txt | while read replace_spec; do
-        echo "replace_spec: '${replace_spec}'"
-        target=$(echo "${replace_spec}" | cut -f1 -d' ')
-        target_full_path=$(echo "${target}" | sed -e "s+__EESSI_SOFTWARE_PATH__+${software_path}+")
-        replace=$(echo "${replace_spec}" | cut -f2 -d' ')
-        echo "target: '${target}'"
-        echo "target_full_path: '${target_full_path}'"
-        echo "replace: '${replace}'"
-        if [[ -f ${replace} ]]; then
-            echo "replacement file exists"
-            target_lower_path=$(echo "${target_full_path}" | cut -f4- -d/)
-            echo "target_lower_path: '${target_lower_path}'"
-            target_lower_dir=$(dirname ${target_lower_path})
-            echo "target_lower_dir: '${target_lower_dir}'"
-            mkdir -p ${LOWER_DIRS}/${target_lower_dir}
-            cp -a ${replace} ${LOWER_DIRS}/${target_lower_dir}/.
-            ls -lisa ${LOWER_DIRS}/${target_lower_dir}
-            touch ADD_LOWER_DIRS
-        else
-            echo "replacement file does NOT exist; ignoring replacement"
-        fi
-    done
-fi
-echo "LOWER_DIRS: '${LOWER_DIRS}'"
-if [[ -f ADD_LOWER_DIRS ]]; then
-    BUILD_STEP_ARGS+=("--lower-dirs" "${LOWER_DIRS}")
-    echo "Added '--lower-dirs ${LOWER_DIRS}' to build step arguments"
-else
-    echo "Nothing to be added for LOWER_DIRS"
-fi
-
 # create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)
 
diff --git a/bot/inspect.sh b/bot/inspect.sh
index 9d1fa87e1f..533968bffc 100755
--- a/bot/inspect.sh
+++ b/bot/inspect.sh
@@ -27,6 +27,10 @@
 # stop as soon as something fails
 set -e
 
+# script_dir is the directory that contains THIS (inspect.sh) script, usually
+# stored in the directory '.../bot'
+script_dir=$(dirname $(realpath $BASH_SOURCE))
+
 display_help() {
   echo "usage: $0 [OPTIONS]"
   echo "  -h | --help            -  display this usage information"
@@ -81,8 +85,8 @@ done
 set -- "${POSITIONAL_ARGS[@]}"
 
 # source utils.sh and cfg_files.sh
-source scripts/utils.sh
-source scripts/cfg_files.sh
+source ${script_dir}/../scripts/utils.sh
+source ${script_dir}/../scripts/cfg_files.sh
 
 if [[ -z ${resume_tgz} ]]; then
     echo_red "path to tarball for resuming build job is missing"
@@ -255,10 +259,8 @@ CMDLINE_ARGS+=("--storage" "${JOB_STORAGE}")
 
 # make sure some environment settings are available inside the shell started via
 # startprefix
-base_dir=$(dirname $(realpath $0))
-# base_dir of inspect.sh script is '.../bot', 'init' dir is at the same level
 # TODO better use script from tarball???
-source ${base_dir}/../init/eessi_defaults
+source ${script_dir}/../init/eessi_defaults
 
 if [ -z $EESSI_VERSION ]; then
     echo "ERROR: \$EESSI_VERSION must be set!" >&2
@@ -432,14 +434,14 @@ echo "Executing command to start interactive session to inspect build job:"
 # These initializations are combined into a single script that is executed when
 # the shell in startprefix is started. We set the env variable BASH_ENV here.
 if [[ -z ${run_in_prefix} ]]; then
-    echo "./eessi_container.sh ${CMDLINE_ARGS[@]}"
+    echo "${script_dir}/../eessi_container.sh ${CMDLINE_ARGS[@]}"
     echo "                     -- ${EESSI_COMPAT_LAYER_DIR}/startprefix"
-    ./eessi_container.sh "${CMDLINE_ARGS[@]}" \
+    ${script_dir}/../eessi_container.sh "${CMDLINE_ARGS[@]}" \
                      -- ${EESSI_COMPAT_LAYER_DIR}/startprefix
 else
-    echo "./eessi_container.sh ${CMDLINE_ARGS[@]}"
+    echo "${script_dir}/../eessi_container.sh ${CMDLINE_ARGS[@]}"
     echo "                     -- ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< ${run_in_prefix}"
-    ./eessi_container.sh "${CMDLINE_ARGS[@]}" \
+    ${script_dir}/../eessi_container.sh "${CMDLINE_ARGS[@]}" \
                      -- ${EESSI_COMPAT_LAYER_DIR}/startprefix <<< ${run_in_prefix}
 fi
 
diff --git a/create_lmodsitepackage.py b/create_lmodsitepackage.py
index 585c86ba8f..6ba0b6ebfb 100755
--- a/create_lmodsitepackage.py
+++ b/create_lmodsitepackage.py
@@ -120,7 +120,6 @@
         -- simpleName is a module in packagesList
         -- get the full host_injections path
         local hostInjections = string.gsub(os.getenv('EESSI_SOFTWARE_PATH') or "", 'versions', 'host_injections')
-
         -- build final path where the software should be installed
         local packageEasyBuildDir = hostInjections .. "/software/" .. t.modFullName .. "/easybuild"
         local packageDirExists = isDir(packageEasyBuildDir)
@@ -179,9 +178,26 @@
     end
 end
 
+local function eessi_espresso_deprecated_message(t)
+    local frameStk  = require("FrameStk"):singleton()
+    local mt        = frameStk:mt()
+    local simpleName = string.match(t.modFullName, "(.-)/")
+    local version = string.match(t.modFullName, "%d.%d.%d")
+    if simpleName == 'ESPResSo' and version == '4.2.1' then
+    -- Print a message on loading ESPreSso v <= 4.2.1 recommending using v 4.2.2 and above.
+    -- A message and not a warning as the exit code would break CI runs otherwise.
+        local advice = 'Prefer versions  >= 4.2.2 which include important bugfixes.\\n'
+        advice = advice .. 'For details see https://github.com/espressomd/espresso/releases/tag/4.2.2\\n'
+        advice = advice .. 'Use version 4.2.1 at your own risk!\\n'
+        LmodWarning("\\nESPResSo v4.2.1 has known issues and has been deprecated. ", advice)
+        LmodMessage("\\nESPResSo v4.2.1 has known issues and has been deprecated. ", advice)
+    end
+end
+
 -- Combine both functions into a single one, as we can only register one function as load hook in lmod
 -- Also: make it non-local, so it can be imported and extended by other lmodrc files if needed
 function eessi_load_hook(t)
+    eessi_espresso_deprecated_message(t)
     -- Only apply CUDA and libraries hook if the loaded module is in the NESSI prefix
     -- This avoids getting an Lmod Error when trying to load a CUDA or library module from a local software stack
     if from_eessi_prefix(t) then
diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml
index fd5c379ed2..1bca410982 100644
--- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml
+++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2022b.yml
@@ -11,3 +11,5 @@ easyconfigs:
   - R-bundle-Bioconductor-3.16-foss-2022b-R-4.2.2.eb:
       options:
         from-pr: 20379
+  - ParaView-5.11.1-foss-2022b.eb
+  - SEPP-4.5.1-foss-2022b.eb
diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml
index 5a7453a9c0..03da218a78 100644
--- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml
+++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023b.yml
@@ -24,3 +24,5 @@ easyconfigs:
         from-pr: 20439
   - GDB-13.2-GCCcore-13.2.0.eb
   - IPython-8.17.2-GCCcore-13.2.0.eb
+  - Qt5-5.15.13-GCCcore-13.2.0.eb
+  - NLTK-3.8.1-foss-2023b.eb
diff --git a/eb_hooks.py b/eb_hooks.py
index 6e2af1e3fb..49d2c4ce29 100644
--- a/eb_hooks.py
+++ b/eb_hooks.py
@@ -312,9 +312,10 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix):
         raise EasyBuildError("Qt5-specific hook triggered for non-Qt5 easyconfig?!")
 
 
+
 def parse_hook_sentencepiece_disable_tcmalloc_aarch64(ec, eprefix):
     """
-    Disable using TCMalloc
+    Disable using TC_Malloc on 'aarch64/generic'
     """
     cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
     if ec.name == 'SentencePiece' and ec.version in ['0.2.0']:
@@ -901,6 +902,7 @@ def inject_gpu_property(ec):
     return ec
 
 
+
 def pre_module_hook(self, *args, **kwargs):
     """Main pre-module-check hook: trigger custom functions based on software name."""
     if self.name in PRE_MODULE_HOOKS:
diff --git a/install_scripts.sh b/install_scripts.sh
index 07643a39e6..d2e45466e3 100755
--- a/install_scripts.sh
+++ b/install_scripts.sh
@@ -113,6 +113,7 @@ nvidia_files=(
     eessi-2023.06-cuda-and-libraries.yml
     install_cuda_and_libraries.sh
     link_nvidia_host_libraries.sh
+    copy_nvidia_host_libraries.sh
 )
 copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}"
 
diff --git a/replacement_files/ctypes/util.py b/replacement_files/ctypes/util.py
deleted file mode 100644
index b4cb4becb0..0000000000
--- a/replacement_files/ctypes/util.py
+++ /dev/null
@@ -1,379 +0,0 @@
-import os
-import shutil
-import subprocess
-import sys
-
-# find_library(name) returns the pathname of a library, or None.
-if os.name == "nt":
-
-    def _get_build_version():
-        """Return the version of MSVC that was used to build Python.
-
-        For Python 2.3 and up, the version number is included in
-        sys.version.  For earlier versions, assume the compiler is MSVC 6.
-        """
-        # This function was copied from Lib/distutils/msvccompiler.py
-        prefix = "MSC v."
-        i = sys.version.find(prefix)
-        if i == -1:
-            return 6
-        i = i + len(prefix)
-        s, rest = sys.version[i:].split(" ", 1)
-        majorVersion = int(s[:-2]) - 6
-        if majorVersion >= 13:
-            majorVersion += 1
-        minorVersion = int(s[2:3]) / 10.0
-        # I don't think paths are affected by minor version in version 6
-        if majorVersion == 6:
-            minorVersion = 0
-        if majorVersion >= 6:
-            return majorVersion + minorVersion
-        # else we don't know what version of the compiler this is
-        return None
-
-    def find_msvcrt():
-        """Return the name of the VC runtime dll"""
-        version = _get_build_version()
-        if version is None:
-            # better be safe than sorry
-            return None
-        if version <= 6:
-            clibname = 'msvcrt'
-        elif version <= 13:
-            clibname = 'msvcr%d' % (version * 10)
-        else:
-            # CRT is no longer directly loadable. See issue23606 for the
-            # discussion about alternative approaches.
-            return None
-
-        # If python was built with in debug mode
-        import importlib.machinery
-        if '_d.pyd' in importlib.machinery.EXTENSION_SUFFIXES:
-            clibname += 'd'
-        return clibname+'.dll'
-
-    def find_library(name):
-        if name in ('c', 'm'):
-            return find_msvcrt()
-        # See MSDN for the REAL search order.
-        for directory in os.environ['PATH'].split(os.pathsep):
-            fname = os.path.join(directory, name)
-            if os.path.isfile(fname):
-                return fname
-            if fname.lower().endswith(".dll"):
-                continue
-            fname = fname + ".dll"
-            if os.path.isfile(fname):
-                return fname
-        return None
-
-elif os.name == "posix" and sys.platform == "darwin":
-    from ctypes.macholib.dyld import dyld_find as _dyld_find
-    def find_library(name):
-        possible = ['lib%s.dylib' % name,
-                    '%s.dylib' % name,
-                    '%s.framework/%s' % (name, name)]
-        for name in possible:
-            try:
-                return _dyld_find(name)
-            except ValueError:
-                continue
-        return None
-
-elif sys.platform.startswith("aix"):
-    # AIX has two styles of storing shared libraries
-    # GNU auto_tools refer to these as svr4 and aix
-    # svr4 (System V Release 4) is a regular file, often with .so as suffix
-    # AIX style uses an archive (suffix .a) with members (e.g., shr.o, libssl.so)
-    # see issue#26439 and _aix.py for more details
-
-    from ctypes._aix import find_library
-
-elif os.name == "posix":
-    # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump
-    import re, tempfile
-
-    def _is_elf(filename):
-        "Return True if the given file is an ELF file"
-        elf_header = b'\x7fELF'
-        with open(filename, 'br') as thefile:
-            return thefile.read(4) == elf_header
-
-    def _findLib_gcc(name):
-        # Run GCC's linker with the -t (aka --trace) option and examine the
-        # library name it prints out. The GCC command will fail because we
-        # haven't supplied a proper program with main(), but that does not
-        # matter.
-        expr = os.fsencode(r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name))
-
-        c_compiler = shutil.which('gcc')
-        if not c_compiler:
-            c_compiler = shutil.which('cc')
-        if not c_compiler:
-            # No C compiler available, give up
-            return None
-
-        temp = tempfile.NamedTemporaryFile()
-        try:
-            args = [c_compiler, '-Wl,-t', '-o', temp.name, '-l' + name]
-
-            env = dict(os.environ)
-            env['LC_ALL'] = 'C'
-            env['LANG'] = 'C'
-            try:
-                proc = subprocess.Popen(args,
-                                        stdout=subprocess.PIPE,
-                                        stderr=subprocess.STDOUT,
-                                        env=env)
-            except OSError:  # E.g. bad executable
-                return None
-            with proc:
-                trace = proc.stdout.read()
-        finally:
-            try:
-                temp.close()
-            except FileNotFoundError:
-                # Raised if the file was already removed, which is the normal
-                # behaviour of GCC if linking fails
-                pass
-        res = re.findall(expr, trace)
-        if not res:
-            return None
-
-        for file in res:
-            # Check if the given file is an elf file: gcc can report
-            # some files that are linker scripts and not actual
-            # shared objects. See bpo-41976 for more details
-            if not _is_elf(file):
-                continue
-            return os.fsdecode(file)
-
-
-    if sys.platform == "sunos5":
-        # use /usr/ccs/bin/dump on solaris
-        def _get_soname(f):
-            if not f:
-                return None
-
-            try:
-                proc = subprocess.Popen(("/usr/ccs/bin/dump", "-Lpv", f),
-                                        stdout=subprocess.PIPE,
-                                        stderr=subprocess.DEVNULL)
-            except OSError:  # E.g. command not found
-                return None
-            with proc:
-                data = proc.stdout.read()
-            res = re.search(br'\[.*\]\sSONAME\s+([^\s]+)', data)
-            if not res:
-                return None
-            return os.fsdecode(res.group(1))
-    else:
-        def _get_soname(f):
-            # assuming GNU binutils / ELF
-            if not f:
-                return None
-            objdump = shutil.which('objdump')
-            if not objdump:
-                # objdump is not available, give up
-                return None
-
-            try:
-                proc = subprocess.Popen((objdump, '-p', '-j', '.dynamic', f),
-                                        stdout=subprocess.PIPE,
-                                        stderr=subprocess.DEVNULL)
-            except OSError:  # E.g. bad executable
-                return None
-            with proc:
-                dump = proc.stdout.read()
-            res = re.search(br'\sSONAME\s+([^\s]+)', dump)
-            if not res:
-                return None
-            return os.fsdecode(res.group(1))
-
-    if sys.platform.startswith(("freebsd", "openbsd", "dragonfly")):
-
-        def _num_version(libname):
-            # "libxyz.so.MAJOR.MINOR" => [ MAJOR, MINOR ]
-            parts = libname.split(b".")
-            nums = []
-            try:
-                while parts:
-                    nums.insert(0, int(parts.pop()))
-            except ValueError:
-                pass
-            return nums or [sys.maxsize]
-
-        def find_library(name):
-            ename = re.escape(name)
-            expr = r':-l%s\.\S+ => \S*/(lib%s\.\S+)' % (ename, ename)
-            expr = os.fsencode(expr)
-
-            try:
-                proc = subprocess.Popen(('/sbin/ldconfig', '-r'),
-                                        stdout=subprocess.PIPE,
-                                        stderr=subprocess.DEVNULL)
-            except OSError:  # E.g. command not found
-                data = b''
-            else:
-                with proc:
-                    data = proc.stdout.read()
-
-            res = re.findall(expr, data)
-            if not res:
-                return _get_soname(_findLib_gcc(name))
-            res.sort(key=_num_version)
-            return os.fsdecode(res[-1])
-
-    elif sys.platform == "sunos5":
-
-        def _findLib_crle(name, is64):
-            if not os.path.exists('/usr/bin/crle'):
-                return None
-
-            env = dict(os.environ)
-            env['LC_ALL'] = 'C'
-
-            if is64:
-                args = ('/usr/bin/crle', '-64')
-            else:
-                args = ('/usr/bin/crle',)
-
-            paths = None
-            try:
-                proc = subprocess.Popen(args,
-                                        stdout=subprocess.PIPE,
-                                        stderr=subprocess.DEVNULL,
-                                        env=env)
-            except OSError:  # E.g. bad executable
-                return None
-            with proc:
-                for line in proc.stdout:
-                    line = line.strip()
-                    if line.startswith(b'Default Library Path (ELF):'):
-                        paths = os.fsdecode(line).split()[4]
-
-            if not paths:
-                return None
-
-            for dir in paths.split(":"):
-                libfile = os.path.join(dir, "lib%s.so" % name)
-                if os.path.exists(libfile):
-                    return libfile
-
-            return None
-
-        def find_library(name, is64 = False):
-            return _get_soname(_findLib_crle(name, is64) or _findLib_gcc(name))
-
-    else:
-
-        def _findSoname_ldconfig(name):
-            import struct
-            if struct.calcsize('l') == 4:
-                machine = os.uname().machine + '-32'
-            else:
-                machine = os.uname().machine + '-64'
-            mach_map = {
-                'x86_64-64': 'libc6,x86-64',
-                'ppc64-64': 'libc6,64bit',
-                'sparc64-64': 'libc6,64bit',
-                's390x-64': 'libc6,64bit',
-                'ia64-64': 'libc6,IA-64',
-                }
-            abi_type = mach_map.get(machine, 'libc6')
-
-            # XXX assuming GLIBC's ldconfig (with option -p)
-            regex = r'\s+(lib%s\.[^\s]+)\s+\(%s'
-            regex = os.fsencode(regex % (re.escape(name), abi_type))
-            try:
-                with subprocess.Popen(['/sbin/ldconfig', '-p'],
-                                      stdin=subprocess.DEVNULL,
-                                      stderr=subprocess.DEVNULL,
-                                      stdout=subprocess.PIPE,
-                                      env={'LC_ALL': 'C', 'LANG': 'C'}) as p:
-                    res = re.search(regex, p.stdout.read())
-                    if res:
-                        return os.fsdecode(res.group(1))
-            except OSError:
-                pass
-
-        def _findLib_ld(name):
-            # See issue #9998 for why this is needed
-            expr = r'[^\(\)\s]*lib%s\.[^\(\)\s]*' % re.escape(name)
-            cmd = ['ld', '-t']
-            libpath = os.environ.get('LD_LIBRARY_PATH')
-            if libpath:
-                for d in libpath.split(':'):
-                    cmd.extend(['-L', d])
-            cmd.extend(['-o', os.devnull, '-l%s' % name])
-            result = None
-            try:
-                p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
-                                     stderr=subprocess.PIPE,
-                                     universal_newlines=True)
-                out, _ = p.communicate()
-                res = re.findall(expr, os.fsdecode(out))
-                for file in res:
-                    # Check if the given file is an elf file: gcc can report
-                    # some files that are linker scripts and not actual
-                    # shared objects. See bpo-41976 for more details
-                    if not _is_elf(file):
-                        continue
-                    return os.fsdecode(file)
-            except Exception:
-                pass  # result will be None
-            return result
-
-        def find_library(name):
-            # See issue #9998
-            lib = _findLib_gcc(name)
-            # return absolute path
-            return _findSoname_ldconfig(name) or \
-                    os.path.join(os.path.dirname(lib), _get_soname(lib)) or \
-                   _get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name))
-
-################################################################
-# test code
-
-def test():
-    from ctypes import cdll
-    if os.name == "nt":
-        print(cdll.msvcrt)
-        print(cdll.load("msvcrt"))
-        print(find_library("msvcrt"))
-
-    if os.name == "posix":
-        # find and load_version
-        print(find_library("m"))
-        print(find_library("c"))
-        print(find_library("bz2"))
-
-        # load
-        if sys.platform == "darwin":
-            print(cdll.LoadLibrary("libm.dylib"))
-            print(cdll.LoadLibrary("libcrypto.dylib"))
-            print(cdll.LoadLibrary("libSystem.dylib"))
-            print(cdll.LoadLibrary("System.framework/System"))
-        # issue-26439 - fix broken test call for AIX
-        elif sys.platform.startswith("aix"):
-            from ctypes import CDLL
-            if sys.maxsize < 2**32:
-                print(f"Using CDLL(name, os.RTLD_MEMBER): {CDLL('libc.a(shr.o)', os.RTLD_MEMBER)}")
-                print(f"Using cdll.LoadLibrary(): {cdll.LoadLibrary('libc.a(shr.o)')}")
-                # librpm.so is only available as 32-bit shared library
-                print(find_library("rpm"))
-                print(cdll.LoadLibrary("librpm.so"))
-            else:
-                print(f"Using CDLL(name, os.RTLD_MEMBER): {CDLL('libc.a(shr_64.o)', os.RTLD_MEMBER)}")
-                print(f"Using cdll.LoadLibrary(): {cdll.LoadLibrary('libc.a(shr_64.o)')}")
-            print(f"crypt\t:: {find_library('crypt')}")
-            print(f"crypt\t:: {cdll.LoadLibrary(find_library('crypt'))}")
-            print(f"crypto\t:: {find_library('crypto')}")
-            print(f"crypto\t:: {cdll.LoadLibrary(find_library('crypto'))}")
-        else:
-            print(cdll.LoadLibrary("libm.so"))
-            print(cdll.LoadLibrary("libcrypt.so"))
-            print(find_library("crypt"))
-
-if __name__ == "__main__":
-    test()
diff --git a/scripts/utils.sh b/scripts/utils.sh
index 962decd20e..fec6368eb0 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -78,7 +78,7 @@ function create_directory_structure() {
   return $return_code
 }
 
-# Function to check if a command exists
+# function to check if a command exists
 function command_exists() {
     command -v "$1" >/dev/null 2>&1
 }

From 77fd3749b2b51a0ae974a1eb18cd37eea8480c94 Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 9 Jun 2024 20:06:34 +0200
Subject: [PATCH 16/17] add PyTorch-bundle with CUDA

---
 .../2023.06/eessi-2023.06-eb-4.9.1-2023a.yml  | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml
index 762f1a236f..903979da71 100644
--- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml
+++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml
@@ -55,6 +55,26 @@ easyconfigs:
   - PyTorch-2.1.2-foss-2023a-CUDA-12.1.1.eb:
       options:
         cuda-compute-capabilities: 6.0,6.1,7.0,7.5,8.0,8.6,8.9,9.0
+  # PyTorch-bundle-CUDA's dependencies without CUDA
+  - librosa-0.10.1-foss-2023a.eb
+  - NLTK-3.8.1-foss-2023a.eb
+  - parameterized-0.9.0-GCCcore-12.3.0.eb
+  - Scalene-1.5.26-GCCcore-12.3.0.eb
+  - scikit-image-0.22.0-foss-2023a.eb
+  - SentencePiece-0.2.0-GCC-12.3.0.eb:
+      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19987
+      options:
+        from-pr: 19987
+  - libmad-0.15.1b-GCCcore-12.3.0.eb:
+      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19987
+      options:
+        from-pr: 19987
+  - SoX-14.4.2-GCCcore-12.3.0.eb:
+      # see https://github.com/easybuilders/easybuild-easyconfigs/pull/19987
+      options:
+        from-pr: 19987
+  - tensorboard-2.15.1-gfbf-2023a.eb
+  - tqdm-4.66.1-GCCcore-12.3.0.eb
   - PyTorch-bundle-2.1.2-foss-2023a-CUDA-12.1.1.eb:
       # see https://github.com/easybuilders/easybuild-easyconfigs/pull/20484
       options:

From 6752be305290151901e8bc05b8e65c3d27098f7d Mon Sep 17 00:00:00 2001
From: Thomas Roeblitz <thomas.roblitz@uib.no>
Date: Sun, 9 Jun 2024 21:20:06 +0200
Subject: [PATCH 17/17] use tmp for host-injections

---
 bot/build.sh | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/bot/build.sh b/bot/build.sh
index de73faef0b..70d30a8c10 100755
--- a/bot/build.sh
+++ b/bot/build.sh
@@ -276,9 +276,12 @@ fi
 # Retain location for host injections so we don't reinstall CUDA
 # (Always need to run the driver installation as available driver may change)
 
-if [[ ! -z ${SHARED_FS_PATH} ]]; then
-    BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections")
-fi
+# use $STORAGE for host-injections (maybe shared fs is not behaving well)
+mkdir -p ${STORAGE}/host_injections
+BUILD_STEP_ARGS+=("--host-injections" "${STORAGE}/host-injections")
+#if [[ ! -z ${SHARED_FS_PATH} ]]; then
+#    BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections")
+#fi
 
 # create tmp file for output of build step
 build_outerr=$(mktemp build.outerr.XXXX)