From f60ecd8529985dcc2c73044e1ee671aa67d51e0e Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 23 May 2024 09:16:08 +0200 Subject: [PATCH 01/10] {2023.06}[foss/2023a] PyTorch v2.1.2 with CUDA/12.1.1 --- .../pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index aeaf710837..c70385abaf 100644 --- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -36,3 +36,4 @@ easyconfigs: from-pr: 19451 - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb - ABySS-2.3.7-foss-2023a.eb + - PyTorch-2.1.2-foss-2023a-CUDA-12.1.1.eb From 31eef035269187f96ad97abafda7ea180f2c1f1d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 23 May 2024 15:01:21 +0200 Subject: [PATCH 02/10] add compute capability setting (just 7.0 for now) --- .../pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index c70385abaf..d49004d02c 100644 --- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -36,4 +36,6 @@ easyconfigs: from-pr: 19451 - OSU-Micro-Benchmarks-7.2-gompi-2023a-CUDA-12.1.1.eb - ABySS-2.3.7-foss-2023a.eb - - PyTorch-2.1.2-foss-2023a-CUDA-12.1.1.eb + - PyTorch-2.1.2-foss-2023a-CUDA-12.1.1.eb: + options: + cuda-compute-capabilities: 7.0 From fcc0eaf8e4fb00b4b47f110e9d3e002964f66193 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 23 May 2024 18:43:33 +0200 Subject: [PATCH 03/10] add CUPTI lib path to LIBRARY_PATH --- eb_hooks.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index c7b2a62f64..c4c2a0261e 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -425,6 +425,28 @@ def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs): raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") +def pre_configure_hook_pytorch_add_cupti_libdir(self, *args, **kwargs): + """ + Pre-configure hook for PyTorch: add directory $EESSI_SOFTWARE_PATH/CUDA/12.1.1/extras/CUPTI/lib64 to LIBRARY_PATH + """ + #if self.name == 'PyTorch' and 'CUDA' in self.versionsuffix: + if self.name == 'PyTorch': + library_path = os.getenv('LIBRARY_PATH') + print_msg("library_path: '%s'", library_path) + #lib_dirs = library_path.split(':') + eessi_software_path = os.getenv('EESSI_SOFTWARE_PATH') + print_msg("eessi_software_path: '%s'", eessi_software_path) + #for lib_dir in lib_dirs: + # if lib_dir.startswith(eessi_software_path + '/CUDA'): + # + cupti_lib_dir = os.path.join(eessi_software_path, 'CUDA', '12.1.1', 'extras', 'CUPTI', 'lib64') + print_msg("cupti_lib_dir: '%s'", cupti_lib_dir) + env.setvar('LIBRARY_PATH', ':'.join([library_path, cupti_lib_dir])) + print_msg("LIBRARY_PATH: '%s'", os.getenv('LIBRARY_PATH')) + else: + raise EasyBuildError("PyTorch-specific hook triggered for non-PyTorch easyconfig?!") + + def pre_configure_hook_libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs): """Add --disable-psm3 to libfabric configure options when building with --optarch=GENERIC on x86_64.""" if self.name == 'libfabric': @@ -851,6 +873,7 @@ def inject_gpu_property(ec): 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, + 'PyTorch': pre_configure_hook_pytorch_add_cupti_libdir, 'WRF': pre_configure_hook_wrf_aarch64, 'at-spi2-core': pre_configure_hook_atspi2core_filter_ld_library_path, } From de17ad31d0c21d70ab2973e94463db6b2382681d Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 23 May 2024 19:51:31 +0200 Subject: [PATCH 04/10] add missing 'software' to library path --- eb_hooks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index c4c2a0261e..7b5c082ddc 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -427,7 +427,7 @@ def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs): def pre_configure_hook_pytorch_add_cupti_libdir(self, *args, **kwargs): """ - Pre-configure hook for PyTorch: add directory $EESSI_SOFTWARE_PATH/CUDA/12.1.1/extras/CUPTI/lib64 to LIBRARY_PATH + Pre-configure hook for PyTorch: add directory $EESSI_SOFTWARE_PATH/software/CUDA/12.1.1/extras/CUPTI/lib64 to LIBRARY_PATH """ #if self.name == 'PyTorch' and 'CUDA' in self.versionsuffix: if self.name == 'PyTorch': @@ -439,7 +439,7 @@ def pre_configure_hook_pytorch_add_cupti_libdir(self, *args, **kwargs): #for lib_dir in lib_dirs: # if lib_dir.startswith(eessi_software_path + '/CUDA'): # - cupti_lib_dir = os.path.join(eessi_software_path, 'CUDA', '12.1.1', 'extras', 'CUPTI', 'lib64') + cupti_lib_dir = os.path.join(eessi_software_path, 'software', 'CUDA', '12.1.1', 'extras', 'CUPTI', 'lib64') print_msg("cupti_lib_dir: '%s'", cupti_lib_dir) env.setvar('LIBRARY_PATH', ':'.join([library_path, cupti_lib_dir])) print_msg("LIBRARY_PATH: '%s'", os.getenv('LIBRARY_PATH')) From b053cbd960f6f0308f8364c7969a0420d4068b06 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Thu, 23 May 2024 23:03:45 +0200 Subject: [PATCH 05/10] nvcc sanity check --- EESSI-install-software.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 6c59f46570..11b2b3069e 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -208,6 +208,9 @@ if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then -e ${EESSI_PREFIX}/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml \ -t /tmp/temp \ --accept-cuda-eula + ml CUDA/12.1.1 + command -V nvcc + ml rm CUDA/12.1.1 else echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed" fi From 8951b4f2f721fc50e5a4eba084f8f83935902a44 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 25 May 2024 11:13:53 +0200 Subject: [PATCH 06/10] improve hook to add CUPTI/lib64 dir to LIBRARY_PATH --- eb_hooks.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/eb_hooks.py b/eb_hooks.py index 7b5c082ddc..f01e1ca55c 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -429,20 +429,21 @@ def pre_configure_hook_pytorch_add_cupti_libdir(self, *args, **kwargs): """ Pre-configure hook for PyTorch: add directory $EESSI_SOFTWARE_PATH/software/CUDA/12.1.1/extras/CUPTI/lib64 to LIBRARY_PATH """ - #if self.name == 'PyTorch' and 'CUDA' in self.versionsuffix: if self.name == 'PyTorch': - library_path = os.getenv('LIBRARY_PATH') - print_msg("library_path: '%s'", library_path) - #lib_dirs = library_path.split(':') - eessi_software_path = os.getenv('EESSI_SOFTWARE_PATH') - print_msg("eessi_software_path: '%s'", eessi_software_path) - #for lib_dir in lib_dirs: - # if lib_dir.startswith(eessi_software_path + '/CUDA'): - # - cupti_lib_dir = os.path.join(eessi_software_path, 'software', 'CUDA', '12.1.1', 'extras', 'CUPTI', 'lib64') - print_msg("cupti_lib_dir: '%s'", cupti_lib_dir) - env.setvar('LIBRARY_PATH', ':'.join([library_path, cupti_lib_dir])) - print_msg("LIBRARY_PATH: '%s'", os.getenv('LIBRARY_PATH')) + if 'cudaver' in self.cfg.template_values and self.cfg.template_values['cudaver'] == '12.1.1': + _cudaver = self.cfg.template_values['cudaver'] + print_msg("pre_configure_hook_pytorch_add_cupti_libdir: CUDA version: '%s'" % _cudaver) + _library_path = os.getenv('LIBRARY_PATH') + print_msg("pre_configure_hook_pytorch_add_cupti_libdir: library_path: '%s'", _library_path) + _eessi_software_path = os.getenv('EESSI_SOFTWARE_PATH') + print_msg("pre_configure_hook_pytorch_add_cupti_libdir: eessi_software_path: '%s'", _eessi_software_path) + _cupti_lib_dir = os.path.join(_eessi_software_path, 'software', 'CUDA', _cudaver, 'extras', 'CUPTI', 'lib64') + print_msg("pre_configure_hook_pytorch_add_cupti_libdir: cupti_lib_dir: '%s'", _cupti_lib_dir) + if _library_path: + env.setvar('LIBRARY_PATH', ':'.join([_library_path, _cupti_lib_dir])) + else: + env.setvar('LIBRARY_PATH', _cupti_lib_dir) + print_msg("pre_configure_hook_pytorch_add_cupti_libdir: LIBRARY_PATH: '%s'", os.getenv('LIBRARY_PATH')) else: raise EasyBuildError("PyTorch-specific hook triggered for non-PyTorch easyconfig?!") From 0e4e8243e54ba5ded373b9a5697de278f9cffbbd Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sat, 25 May 2024 11:17:46 +0200 Subject: [PATCH 07/10] don't load/unload CUDA --- EESSI-install-software.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 11b2b3069e..6c59f46570 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -208,9 +208,6 @@ if [ -z "${skip_cuda_install}" ] || [ ! "${skip_cuda_install}" ]; then -e ${EESSI_PREFIX}/scripts/gpu_support/nvidia/eessi-2023.06-cuda-and-libraries.yml \ -t /tmp/temp \ --accept-cuda-eula - ml CUDA/12.1.1 - command -V nvcc - ml rm CUDA/12.1.1 else echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed" fi From 47eeac1b0739d3155a374a3c42f64159da0fc1e4 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 26 May 2024 08:10:15 +0200 Subject: [PATCH 08/10] sync with state of relevant EESSI PRs (579, 581, 586) --- EESSI-install-software.sh | 70 +++++++++++++++++++++++--------------- bot/build.sh | 5 +++ eessi_container.sh | 6 ---- install_scripts.sh | 2 -- run_in_compat_layer_env.sh | 3 ++ 5 files changed, 51 insertions(+), 35 deletions(-) diff --git a/EESSI-install-software.sh b/EESSI-install-software.sh index 6c59f46570..ca6fed71d5 100755 --- a/EESSI-install-software.sh +++ b/EESSI-install-software.sh @@ -17,6 +17,11 @@ display_help() { echo " --skip-cuda-install - disable installing a full CUDA SDK in the host_injections prefix (e.g. in CI)" } +# Function to check if a command exists +function command_exists() { + command -v "$1" >/dev/null 2>&1 +} + function copy_build_log() { # copy specified build log to specified directory, with some context added build_log=${1} @@ -147,6 +152,39 @@ else mkdir -p ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} fi +# We need to ensure that certain files are present or updated before we source +# $TOPDIR/init/eessi_environment_variables +# Particularly the files we need to have present/updated in +# ${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +# are: +# - .lmod/lmodrc.lua +# - .lmod/SitePackage.lua +# We run scripts to create them if they don't exist or if the scripts have been +# changed in the PR. + +# Set base directory for software and for Lmod config files +_eessi_software_path=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE} +_lmod_cfg_dir=${_eessi_software_path}/.lmod + +# We assume there's only one diff file that corresponds to the PR patch file +pr_diff=$(ls [0-9]*.diff | head -1) + +# Create or update ${_eessi_software_path}/.lmod/lmodrc.lua +_lmodrc_file=${_lmod_cfg_dir}/lmodrc.lua +_lmodrc_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodrc.py$' > /dev/null; echo $?) +if [ ! -f "${_lmodrc_file}" ] || [ "${_lmodrc_changed}" == '0' ]; then + python3 ${TOPDIR}/create_lmodrc.py ${_eessi_software_path} + check_exit_code $? "${_lmodrc_file} created/updated" "Failed to create/update ${_lmodrc_file}" +fi + +# Create or update ${_eessi_software_path}/.lmod/SitePackage.lua +_lmod_sitepackage_file=${_lmod_cfg_dir}/SitePackage.lua +_sitepackage_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodsitepackage.py$' > /dev/null; echo $?) +if [ ! -f "${_lmod_sitepackage_file}" ] || [ "${_sitepackage_changed}" == '0' ]; then + python3 ${TOPDIR}/create_lmodsitepackage.py ${_eessi_software_path} + check_exit_code $? "${_lmod_sitepackage_file} created/updated" "Failed to create/update ${_lmod_sitepackage_file}" +fi + # Set all the EESSI environment variables (respecting $EESSI_SOFTWARE_SUBDIR_OVERRIDE) # $EESSI_SILENT - don't print any messages # $EESSI_BASIC_ENV - give a basic set of environment variables @@ -212,13 +250,11 @@ else echo "Skipping installation of CUDA SDK and cu* libraries in host_injections, since the --skip-cuda-install flag was passed" fi -# Install drivers in host_injections -# TODO: this is commented out for now, because the script assumes that nvidia-smi is available and works; -# if not, an error is produced, and the bot flags the whole build as failed (even when not installing GPU software) -# ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh - -# Don't run the Lmod GPU driver check when doing builds (may not have a GPU, and it's not relevant for vanilla builds anyway) -export EESSI_OVERRIDE_GPU_CHECK=1 +# Install NVIDIA drivers in host_injections (if they exist) +if command_exists "nvidia-smi"; then + echo "Command 'nvidia-smi' found. Installing NVIDIA drivers for use in prefix shell..." + ${EESSI_PREFIX}/scripts/gpu_support/nvidia/link_nvidia_host_libraries.sh +fi # use PR patch file to determine in which easystack files stuff was added changed_easystacks=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^easystacks/.*yml$' | egrep -v 'known-issues|missing') @@ -268,25 +304,5 @@ else done fi -### add packages here - -echo ">> Creating/updating Lmod RC file..." -export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod" -lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua" -lmodrc_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodrc.py$' > /dev/null; echo $?) -if [ ! -f $lmod_rc_file ] || [ ${lmodrc_changed} == '0' ]; then - python3 $TOPDIR/create_lmodrc.py ${EASYBUILD_INSTALLPATH} - check_exit_code $? "$lmod_rc_file created" "Failed to create $lmod_rc_file" -fi - -echo ">> Creating/updating Lmod SitePackage.lua ..." -export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod" -lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua" -sitepackage_changed=$(cat ${pr_diff} | grep '^+++' | cut -f2 -d' ' | sed 's@^[a-z]/@@g' | grep '^create_lmodsitepackage.py$' > /dev/null; echo $?) -if [ ! -f "$lmod_sitepackage_file" ] || [ "${sitepackage_changed}" == '0' ]; then - python3 $TOPDIR/create_lmodsitepackage.py ${EASYBUILD_INSTALLPATH} - check_exit_code $? "$lmod_sitepackage_file created" "Failed to create $lmod_sitepackage_file" -fi - echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} diff --git a/bot/build.sh b/bot/build.sh index 23f5fd952b..6e835cb6aa 100755 --- a/bot/build.sh +++ b/bot/build.sh @@ -271,6 +271,11 @@ if [[ ! -z ${SHARED_FS_PATH} ]]; then BUILD_STEP_ARGS+=("--host-injections" "${SHARED_FS_PATH}/host-injections") fi +# Don't run the Lmod GPU driver check when doing builds (may not have a GPU, and it's not relevant for vanilla builds anyway) +echo "EESSI_OVERRIDE_GPU_CHECK='${EESSI_OVERRIDE_GPU_CHECK}'" +export EESSI_OVERRIDE_GPU_CHECK=1 +echo "EESSI_OVERRIDE_GPU_CHECK='${EESSI_OVERRIDE_GPU_CHECK}'" + # create tmp file for output of build step build_outerr=$(mktemp build.outerr.XXXX) diff --git a/eessi_container.sh b/eessi_container.sh index c9ed97e5c6..a9405b6d8e 100755 --- a/eessi_container.sh +++ b/eessi_container.sh @@ -477,12 +477,6 @@ if [[ ${SETUP_NVIDIA} -eq 1 ]]; then mkdir -p ${EESSI_USR_LOCAL_CUDA} BIND_PATHS="${BIND_PATHS},${EESSI_VAR_LOG}:/var/log,${EESSI_USR_LOCAL_CUDA}:/usr/local/cuda" [[ ${VERBOSE} -eq 1 ]] && echo "BIND_PATHS=${BIND_PATHS}" - if [[ "${NVIDIA_MODE}" == "install" ]] ; then - # We need to "trick" our LMOD_RC file to allow us to load CUDA modules even without a CUDA driver - # (this works because we build within a container and the LMOD_RC recognises that) - touch ${EESSI_TMPDIR}/libcuda.so - export SINGULARITY_CONTAINLIBS="${EESSI_TMPDIR}/libcuda.so" - fi fi fi diff --git a/install_scripts.sh b/install_scripts.sh index 17712a0ae7..07643a39e6 100755 --- a/install_scripts.sh +++ b/install_scripts.sh @@ -112,8 +112,6 @@ copy_files_by_list ${TOPDIR}/scripts ${INSTALL_PREFIX}/scripts "${script_files[@ nvidia_files=( eessi-2023.06-cuda-and-libraries.yml install_cuda_and_libraries.sh - install_cuda_host_injections.sh - install_cuDNN_host_injections.sh link_nvidia_host_libraries.sh ) copy_files_by_list ${TOPDIR}/scripts/gpu_support/nvidia ${INSTALL_PREFIX}/scripts/gpu_support/nvidia "${nvidia_files[@]}" diff --git a/run_in_compat_layer_env.sh b/run_in_compat_layer_env.sh index efd30e7cc0..393956a0c1 100755 --- a/run_in_compat_layer_env.sh +++ b/run_in_compat_layer_env.sh @@ -29,6 +29,9 @@ fi if [ ! -z ${http_proxy} ]; then INPUT="export http_proxy=${http_proxy}; ${INPUT}" fi +if [ ! -z ${EESSI_OVERRIDE_GPU_CHECK} ]; then + INPUT="export EESSI_OVERRIDE_GPU_CHECK=${EESSI_OVERRIDE_GPU_CHECK}; ${INPUT}" +fi if [ ! -z ${https_proxy} ]; then INPUT="export https_proxy=${https_proxy}; ${INPUT}" fi From be1de45639066705785e1000bce9c128dd8e46f3 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Sun, 26 May 2024 08:11:50 +0200 Subject: [PATCH 09/10] building for all relevant compute capabilities --- .../pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml index 4b2965554e..bd0de1a6c8 100644 --- a/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml +++ b/easystacks/pilot.nessi.no/2023.06/eessi-2023.06-eb-4.9.1-2023a.yml @@ -41,4 +41,4 @@ easyconfigs: from-pr: 20540 - PyTorch-2.1.2-foss-2023a-CUDA-12.1.1.eb: options: - cuda-compute-capabilities: 7.0 + cuda-compute-capabilities: 6.0,6.1,7.0,7.5,8.0,8.6,8.9,9.0 From f780ca552270086f34798c4500978be453fe4ef8 Mon Sep 17 00:00:00 2001 From: Thomas Roeblitz Date: Wed, 29 May 2024 23:11:47 +0200 Subject: [PATCH 10/10] run PyTorch tests sequentially --- eb_hooks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eb_hooks.py b/eb_hooks.py index f01e1ca55c..e4a957acdc 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -583,6 +583,12 @@ def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs): """ if self.name == 'PyTorch' and self.version == '2.1.2' and get_cpu_architecture() == AARCH64: self.cfg['max_failed_tests'] = 10 + if 'cudaver' in self.cfg.template_values and self.cfg.template_values['cudaver'] == '12.1.1': + _cudaver = self.cfg.template_values['cudaver'] + _runtest = self.cfg['runtest'] + self.cfg['runtest'] = _runtest.replace( + 'PYTHONUNBUFFERED', + 'PYTORCH_TEST_RUN_EVERYTHING_IN_SERIAL=1 PYTHONUNBUFFERED') def pre_single_extension_hook(ext, *args, **kwargs):