From a92de9751103acf18dbae56811cb24cb2eb65172 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 5 Jun 2024 23:26:46 +0200 Subject: [PATCH] update method and location to install custom_ctypes --- EESSI-install-software.sh | 5 +- eb_hooks.py | 8 +- install_scripts.sh | 6 -- scripts/extra/custom_ctypes-1.2.eb | 29 ++++++ .../extra/eessi-2023.06-extra-packages.yml | 2 + scripts/extra/install_custom_ctypes.sh | 96 ------------------- scripts/extra/install_extra_packages.sh | 95 ++++++++++++++++++ 7 files changed, 131 insertions(+), 110 deletions(-) create mode 100644 scripts/extra/custom_ctypes-1.2.eb create mode 100644 scripts/extra/eessi-2023.06-extra-packages.yml delete mode 100755 scripts/extra/install_custom_ctypes.sh create mode 100755 scripts/extra/install_extra_packages.sh diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index d030a1f798..201ae6f8cb 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -258,8 +258,9 @@ fi # Install extra software that is needed (e.g., for providing a custom ctypes # library when needed) -echo "Location of host_injections: $(ls -l ${EESSI_CVMFS_REPO}/host_injections)" -${EESSI_PREFIX}/scripts/extra/install_custom_ctypes.sh --temp-dir /tmp/temp +cd ${TOPDIR}/scripts/extra +./install_extra_packages.sh --temp-dir /tmp/temp --easystack eessi-2023.06-extra-packages.yml +cd ${TOPDIR} # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') diff --git a/eb_hooks.py b/eb_hooks.py index 14c7c30d91..58523f160b 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -357,9 +357,7 @@ def parse_hook_librosa_custom_ctypes(ec, *args, **kwargs): if ec.name == 'librosa' and ec.version in ('0.10.1',): ec_dict = ec.asdict() eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') - custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1) - custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1) - custom_ctypes_path = os.path.join(custom_ctypes_path, "custom_ctypes") + custom_ctypes_path = os.path.join(eessi_software_path, "software", "custom_ctypes", "1.2") ebpythonprefixes = "EBPYTHONPREFIXES=%s" % custom_ctypes_path exts_list_new = [] for item in ec_dict['exts_list']: @@ -891,9 +889,7 @@ def pre_module_hook_librosa_augment_modluafooter(self, *args, **kwargs): """ if self.name == 'librosa' and self.version == '0.10.1': eessi_software_path = get_eessi_envvar('EESSI_SOFTWARE_PATH') - custom_ctypes_path = eessi_software_path.replace('versions', 'host_injections', 1) - custom_ctypes_path = custom_ctypes_path.replace('software', 'extra', 1) - custom_ctypes_path = os.path.join(custom_ctypes_path, 'custom_ctypes') + custom_ctypes_path = os.path.join(eessi_software_path, "software", "custom_ctypes", "1.2") key = 'modluafooter' values = ['prepend_path("EBPYTHONPREFIXES","%s")' % (custom_ctypes_path)] print_msg("Adding '%s' to modluafooter", values[0]) diff --git a/install_scripts.sh b/install_scripts.sh index 000ad99444..07643a39e6 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -116,12 +116,6 @@ nvidia_files=( ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" -# Copy files for the scripts/extra directory -extra_files=( - install_custom_ctypes.sh -) -copy_files_by_list ${TOPDIR}/scripts/extra ${INSTALL_PREFIX}/scripts/extra "${extra_files[@]}" - # Copy over EasyBuild hooks file used for installations hook_files=( eb_hooks.py diff --git a/scripts/extra/custom_ctypes-1.2.eb b/scripts/extra/custom_ctypes-1.2.eb new file mode 100644 index 0000000000..35be6dcc41 --- /dev/null +++ b/scripts/extra/custom_ctypes-1.2.eb @@ -0,0 +1,29 @@ +## +# This is a contribution from the NESSI project +# Homepage: https://documentation.sigma2.no +# +# Authors:: Thomas Roeblitz +# License:: GPL-2.0-only +# +## + +easyblock = 'Tarball' + +name = 'custom_ctypes' +version = '1.2' + +homepage = 'https://github.com/ComputeCanada/custom_ctypes' +description = """custum_ctypes is a small Python package to fix the discovery of libraries with Python's ctypes module. It changes the behavior of find_library to return absolute paths to shared objects rather than just the names.""" + +toolchain = SYSTEM + +source_urls = ['https://github.com/ComputeCanada/custom_ctypes/archive/refs/tags'] +sources = ['%(version)s.tar.gz'] +checksums = ['3b30ce633c6a329169f2b10ff24b8eaaeef3fa208a66cdacdb53c22f02a88d9b'] + +sanity_check_paths = { + 'files': ['README.md'], + 'dirs': ['lib'], +} + +moduleclass = 'lib' diff --git a/scripts/extra/eessi-2023.06-extra-packages.yml b/scripts/extra/eessi-2023.06-extra-packages.yml new file mode 100644 index 0000000000..22670ec7a3 --- /dev/null +++ b/scripts/extra/eessi-2023.06-extra-packages.yml @@ -0,0 +1,2 @@ +easyconfigs: + - custom_ctypes-1.2.eb diff --git a/scripts/extra/install_custom_ctypes.sh b/scripts/extra/install_custom_ctypes.sh deleted file mode 100755 index 6392da00e5..0000000000 --- a/scripts/extra/install_custom_ctypes.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash - -# This script can be used to install custom ctypes under -# CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/software/custom_ctypes -# and then set EESSI_USE_CUSTOM_CTYPES_DIR=CVMFS_REPO/host_injections/VERSION/extra/OS_TYPE/SOFTWARE_SUBDIR/custom_ctypes -# before loading a module that needs the custom ctypes implementation -# The custom ctypes is downloaded from https://github.com/NorESSI/custom_ctypes which is a fork of -# https://github.com/ComputeCanada/custom_ctypes - -# The `host_injections` directory is a variant symlink that by default points to -# `/opt/eessi`, unless otherwise defined in the local CVMFS configuration (see -# https://cvmfs.readthedocs.io/en/stable/cpt-repo.html#variant-symlinks). For the -# installation to be successful, this directory needs to be writeable by the user -# executing this script. - -# some logging -echo ">>> Running ${BASH_SOURCE}" - -# Initialise our bash functions -TOPDIR=$(dirname $(realpath ${BASH_SOURCE})) -source "${TOPDIR}"/../utils.sh - -# Function to display help message -show_help() { - echo "Usage: $0 [OPTIONS]" - echo "Options:" - echo " --help Display this help message" - echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" - echo " storage during the installation" -} - -# Initialize variables -TEMP_DIR= - -# Parse command-line options -while [[ $# -gt 0 ]]; do - case "$1" in - --help) - show_help - exit 0 - ;; - -t|--temp-dir) - if [ -n "$2" ]; then - TEMP_DIR="$2" - shift 2 - else - echo "Error: Argument required for $1" - show_help - exit 1 - fi - ;; - *) - show_help - fatal_error "Error: Unknown option: $1" - ;; - esac -done - -# Make sure NESSI is initialised -check_eessi_initialised - -# As an installation location just use $EESSI_SOFTWARE_PATH but replacing `versions` with `host_injections` and -# `software` with `extra` -# also append `/custom_ctypes` -NESSI_SITE_INSTALL=${EESSI_SOFTWARE_PATH/versions/host_injections} -NESSI_SITE_INSTALL=${NESSI_SITE_INSTALL/software/extra}/custom_ctypes - -# we need a directory we can use for temporary storage -if [[ -z "${TEMP_DIR}" ]]; then - tmpdir=$(mktemp -d) -else - mkdir -p ${TEMP_DIR} - tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} custom_ctypes.XXX) - if [[ ! -d "$tmpdir" ]] ; then - fatal_error "Could not create directory ${tmpdir}" - fi -fi -echo "Created temporary directory '${tmpdir}'" - -# check if custom_ctypes has already been installed -if [[ -d ${NESSI_SITE_INSTALL}/lib ]]; then - echo "INFO: Installation of custom_ctypes already found at '${NESSI_SITE_INSTALL}'" - exit 0 -fi - -# download custom_ctypes to temp directory -wget https://github.com/NorESSI/custom_ctypes/archive/refs/heads/main.tar.gz -P ${tmpdir} - -# make sure target directory exists -mkdir -p ${NESSI_SITE_INSTALL} - -# unpack custom_ctypes to target directory -tar xvfz ${tmpdir}/main.tar.gz --strip-components=1 -C ${NESSI_SITE_INSTALL} - -# clean up tmpdir -rm -rf "${tmpdir}" diff --git a/scripts/extra/install_extra_packages.sh b/scripts/extra/install_extra_packages.sh new file mode 100755 index 0000000000..ccd2890864 --- /dev/null +++ b/scripts/extra/install_extra_packages.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +# This script can be used to install extra packages under ${EESSI_SOFTWARE_PATH} + +# some logging +echo ">>> Running ${BASH_SOURCE}" + +# Initialise our bash functions +TOPDIR=$(dirname $(realpath ${BASH_SOURCE})) +source "${TOPDIR}"/../utils.sh + +# Function to display help message +show_help() { + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --help Display this help message" + echo " -e, --easystack EASYSTACKFILE Easystack file which specifies easyconfigs to be installed." + echo " -t, --temp-dir /path/to/tmpdir Specify a location to use for temporary" + echo " storage during the installation" +} + +# Initialize variables +TEMP_DIR= +EASYSTACK_FILE= + +# Parse command-line options +while [[ $# -gt 0 ]]; do + case "$1" in + --help) + show_help + exit 0 + ;; + -e|--easystack) + if [ -n "$2" ]; then + EASYSTACK_FILE="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + -t|--temp-dir) + if [ -n "$2" ]; then + TEMP_DIR="$2" + shift 2 + else + echo "Error: Argument required for $1" + show_help + exit 1 + fi + ;; + *) + show_help + fatal_error "Error: Unknown option: $1" + ;; + esac +done + +if [[ -z ${EASYSTACK_FILE} ]]; then + show_help + fatal_error "Error: need to specify easystack file" +fi + +# Make sure NESSI is initialised +check_eessi_initialised + +# As an installation location just use $EESSI_SOFTWARE_PATH +export NESSI_CVMFS_INSTALL=${EESSI_SOFTWARE_PATH} + +# we need a directory we can use for temporary storage +if [[ -z "${TEMP_DIR}" ]]; then + tmpdir=$(mktemp -d) +else + mkdir -p ${TEMP_DIR} + tmpdir=$(mktemp -d --tmpdir=${TEMP_DIR} extra.XXX) + if [[ ! -d "$tmpdir" ]] ; then + fatal_error "Could not create directory ${tmpdir}" + fi +fi +echo "Created temporary directory '${tmpdir}'" +export WORKING_DIR=${tmpdir} + +# load EasyBuild +ml EasyBuild + +# load NESSI-extend/2023.06-easybuild +ml NESSI-extend/2023.06-easybuild + +eb --show-config + +eb --easystack ${EASYSTACK_FILE} --robot + +# clean up tmpdir +rm -rf "${tmpdir}"