From 64d4c819ab1a563b67882a573fc0c7de16a22b65 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 2 Oct 2024 20:19:38 +0000 Subject: [PATCH 01/14] using Rocky 8 on CSPs --- .gitmodules | 10 ++++++---- modulefiles/module_gwsetup.noaacloud.lua | 12 ++++++------ versions/build.noaacloud.ver | 2 +- versions/run.noaacloud.ver | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5c9e569243..b9790e9f81 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,16 +1,18 @@ [submodule "sorc/ufs_model.fd"] path = sorc/ufs_model.fd - url = https://github.com/ufs-community/ufs-weather-model - ignore = dirty + url = git@github.com:NOAA-EPIC/ufs-weather-model.git + branch = csps-rocky8 [submodule "sorc/wxflow"] path = sorc/wxflow url = https://github.com/NOAA-EMC/wxflow [submodule "sorc/gfs_utils.fd"] path = sorc/gfs_utils.fd - url = https://github.com/NOAA-EMC/gfs-utils + url = ssh://git@github.com/NOAA-EPIC/gfs-utils.git + branch = csps-rocky8 [submodule "sorc/ufs_utils.fd"] path = sorc/ufs_utils.fd - url = https://github.com/ufs-community/UFS_UTILS.git + url = git@github.com:NOAA-EPIC/UFS_UTILS-cloud.git + branch = csps-rocky8 [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd url = https://github.com/NOAA-EMC/EMC_verif-global.git diff --git a/modulefiles/module_gwsetup.noaacloud.lua b/modulefiles/module_gwsetup.noaacloud.lua index f3845e8d72..5be99341e4 100644 --- a/modulefiles/module_gwsetup.noaacloud.lua +++ b/modulefiles/module_gwsetup.noaacloud.lua @@ -4,13 +4,13 @@ Load environment to run GFS workflow setup scripts on noaacloud load(pathJoin("rocoto")) -prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/ue-intel/install/modulefiles/Core") +prepend_path("MODULEPATH", "/apps/modules/modulefiles") +load("gnu") +load("stack-intel") +load("stack-intel-oneapi-mpi") +unload("gnu") -local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0" -local python_ver=os.getenv("python_ver") or "3.10.3" - -load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("python", python_ver)) load("py-jinja2") load("py-pyyaml") load("py-numpy") diff --git a/versions/build.noaacloud.ver b/versions/build.noaacloud.ver index fc288b76b5..34caa15317 100644 --- a/versions/build.noaacloud.ver +++ b/versions/build.noaacloud.ver @@ -2,4 +2,4 @@ export stack_intel_ver=2021.3.0 export stack_impi_ver=2021.3.0 export spack_env=gsi-addon-env source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" +export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core" diff --git a/versions/run.noaacloud.ver b/versions/run.noaacloud.ver index 1fc3779b2e..97173aa445 100644 --- a/versions/run.noaacloud.ver +++ b/versions/run.noaacloud.ver @@ -3,6 +3,6 @@ export stack_impi_ver=2021.3.0 export spack_env=gsi-addon-env source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/contrib/spack-stack/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" +export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core" export cdo_ver=2.2.0 From ac435a2bc27b4838622b9a652cc7b84dc3fc10d4 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Tue, 8 Oct 2024 21:25:39 +0000 Subject: [PATCH 02/14] compiled and run G-W with Rocky 8 --- env/AWSPW.env | 4 ++++ env/AZUREPW.env | 16 ++++++++++------ env/GOOGLEPW.env | 4 ++-- modulefiles/module_base.noaacloud.lua | 7 +++++-- modulefiles/module_gwsetup.noaacloud.lua | 4 ++-- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/env/AWSPW.env b/env/AWSPW.env index e366128a1d..0527f5b3e7 100755 --- a/env/AWSPW.env +++ b/env/AWSPW.env @@ -43,6 +43,10 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then export APRUN_UFS="${launcher} -n ${ufs_ntasks}" unset nnodes ufs_ntasks +elif [[ "${step}" = "prep_emissions" ]]; then + + export APRUN="${APRUN_default}" + elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then export CFP_MP="YES" diff --git a/env/AZUREPW.env b/env/AZUREPW.env index c2faeb2bf6..a4baff94fa 100755 --- a/env/AZUREPW.env +++ b/env/AZUREPW.env @@ -50,6 +50,10 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} export wavempexec=${launcher} export wave_mpmd=${mpmd_opt} +elif [[ "${step}" = "prep_emissions" ]]; then + + export APRUN = ${APRUN_default} + elif [[ "${step}" = "post" ]]; then export NTHREADS_NP=${NTHREADS1} @@ -71,7 +75,7 @@ elif [[ "${step}" = "oceanice_products" ]]; then elif [[ "${step}" = "ecen" ]]; then export NTHREADS_ECEN=${NTHREADSmax} - export APRUN_ECEN="${APRUN}" + export APRUN_ECEN="${APRUN_default}" export NTHREADS_CHGRES=${threads_per_task_chgres:-12} [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node} @@ -79,25 +83,25 @@ elif [[ "${step}" = "ecen" ]]; then export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} - export APRUN_CALCINC="${APRUN}" + export APRUN_CALCINC="${APRUN_default}" elif [[ "${step}" = "esfc" ]]; then export NTHREADS_ESFC=${NTHREADSmax} - export APRUN_ESFC="${APRUN}" + export APRUN_ESFC="${APRUN_default}" export NTHREADS_CYCLE=${threads_per_task_cycle:-14} [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} - export APRUN_CYCLE="${APRUN}" + export APRUN_CYCLE="${APRUN_default}" elif [[ "${step}" = "epos" ]]; then export NTHREADS_EPOS=${NTHREADSmax} - export APRUN_EPOS="${APRUN}" + export APRUN_EPOS="${APRUN_default}" elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN}" + export MPIRUN="${APRUN_default}" fi diff --git a/env/GOOGLEPW.env b/env/GOOGLEPW.env index c3b5ec806a..ba3887499f 100755 --- a/env/GOOGLEPW.env +++ b/env/GOOGLEPW.env @@ -45,7 +45,7 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then elif [[ "${step}" = "prep_emissions" ]]; then - export APRUN + export APRUN = ${APRUN_default} elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then @@ -102,6 +102,6 @@ elif [[ "${step}" = "epos" ]]; then elif [[ "${step}" = "fit2obs" ]]; then export NTHREADS_FIT2OBS=${NTHREADS1} - export MPIRUN="${APRUN}" + export MPIRUN="${APRUN_default}" fi diff --git a/modulefiles/module_base.noaacloud.lua b/modulefiles/module_base.noaacloud.lua index 7997b618e4..fc8e6d0585 100644 --- a/modulefiles/module_base.noaacloud.lua +++ b/modulefiles/module_base.noaacloud.lua @@ -5,8 +5,11 @@ Load environment to run GFS on noaacloud local spack_mod_path=(os.getenv("spack_mod_path") or "None") prepend_path("MODULEPATH", spack_mod_path) -load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) -load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) +load("gnu") +load("stack-intel") +load("stack-intel-oneapi-mpi") +unload("gnu") + load(pathJoin("python", (os.getenv("python_ver") or "None"))) load(pathJoin("jasper", (os.getenv("jasper_ver") or "None"))) diff --git a/modulefiles/module_gwsetup.noaacloud.lua b/modulefiles/module_gwsetup.noaacloud.lua index 5be99341e4..a22aad5ede 100644 --- a/modulefiles/module_gwsetup.noaacloud.lua +++ b/modulefiles/module_gwsetup.noaacloud.lua @@ -14,7 +14,7 @@ unload("gnu") load("py-jinja2") load("py-pyyaml") load("py-numpy") -local git_ver=os.getenv("git_ver") or "1.8.3.1" -load(pathJoin("git", git_ver)) +--local git_ver=os.getenv("git_ver") or "1.8.3.1" +--load(pathJoin("git", git_ver)) whatis("Description: GFS run setup environment") From feb2c2a7477e21fb9dd207caa80a9e7ec3ef9916 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 14 Nov 2024 20:54:23 +0000 Subject: [PATCH 03/14] update submodules --- .gitmodules | 10 +- env/AWSPW.env | 172 ++++++++++++++++++++--- modulefiles/module_gwsetup.noaacloud.lua | 8 +- sorc/build_ufs.sh | 2 +- 4 files changed, 167 insertions(+), 25 deletions(-) diff --git a/.gitmodules b/.gitmodules index b9790e9f81..5c9e569243 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,18 +1,16 @@ [submodule "sorc/ufs_model.fd"] path = sorc/ufs_model.fd - url = git@github.com:NOAA-EPIC/ufs-weather-model.git - branch = csps-rocky8 + url = https://github.com/ufs-community/ufs-weather-model + ignore = dirty [submodule "sorc/wxflow"] path = sorc/wxflow url = https://github.com/NOAA-EMC/wxflow [submodule "sorc/gfs_utils.fd"] path = sorc/gfs_utils.fd - url = ssh://git@github.com/NOAA-EPIC/gfs-utils.git - branch = csps-rocky8 + url = https://github.com/NOAA-EMC/gfs-utils [submodule "sorc/ufs_utils.fd"] path = sorc/ufs_utils.fd - url = git@github.com:NOAA-EPIC/UFS_UTILS-cloud.git - branch = csps-rocky8 + url = https://github.com/ufs-community/UFS_UTILS.git [submodule "sorc/verif-global.fd"] path = sorc/verif-global.fd url = https://github.com/NOAA-EMC/EMC_verif-global.git diff --git a/env/AWSPW.env b/env/AWSPW.env index 0527f5b3e7..5e49e2a132 100755 --- a/env/AWSPW.env +++ b/env/AWSPW.env @@ -13,9 +13,13 @@ export launcher="srun -l --export=ALL" export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment -export OMP_STACKSIZE=2048000 +#export OMP_STACKSIZE=2048000 export NTHSTACK=1024000000 +export OMP_STACKSIZE=512M +export KMP_AFFINITY=scatter +export OMP_NUM_THREADS=1 + ulimit -s unlimited ulimit -a @@ -33,15 +37,16 @@ else exit 2 fi -if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then +if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then - export launcher="srun --mpi=pmi2 -l" + export POE="NO" + export BACK="NO" + export sys_tp="HERA" + export launcher_PREP="srun" - (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) - (( ufs_ntasks = nnodes*tasks_per_node )) - # With ESMF threading, the model wants to use the full node - export APRUN_UFS="${launcher} -n ${ufs_ntasks}" - unset nnodes ufs_ntasks +elif [[ "${step}" = "prepsnowobs" ]]; then + + export APRUN_CALCFIMS="${APRUN_default}" elif [[ "${step}" = "prep_emissions" ]]; then @@ -54,24 +59,159 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} export wavempexec=${launcher} export wave_mpmd=${mpmd_opt} -elif [[ "${step}" = "post" ]]; then +elif [[ "${step}" = "atmanlvar" ]]; then - export NTHREADS_NP=${NTHREADS1} - export APRUN_NP="${APRUN_default}" + export NTHREADS_ATMANLVAR=${NTHREADSmax} + export APRUN_ATMANLVAR="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLVAR}" - export NTHREADS_DWN=${threads_per_task_dwn:-1} - [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task} - export APRUN_DWN="${launcher} -n ${ntasks_dwn}" +elif [[ "${step}" = "atmensanlletkf" ]]; then -elif [[ "${step}" = "atmos_products" ]]; then + export NTHREADS_ATMENSANLLETKF=${NTHREADSmax} + export APRUN_ATMENSANLLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLLETKF}" - export USE_CFP="YES" # Use MPMD for downstream product generation on Hera +elif [[ "${step}" = "atmensanlfv3inc" ]]; then + + export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} + export APRUN_ATMENSANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" + +elif [[ "${step}" = "aeroanlrun" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_AEROANL=${NTHREADSmax} + export APRUN_AEROANL="${APRUN_default} --cpus-per-task=${NTHREADS_AEROANL}" + +elif [[ "${step}" = "atmanlfv3inc" ]]; then + + export NTHREADS_ATMANLFV3INC=${NTHREADSmax} + export APRUN_ATMANLFV3INC="${APRUN_default} --cpus-per-task=${NTHREADS_ATMANLFV3INC}" + +elif [[ "${step}" = "prepobsaero" ]]; then + + export NTHREADS_PREPOBSAERO=${NTHREADS1} + export APRUN_PREPOBSAERO="${APRUN_default} --cpus-per-task=${NTHREADS_PREPOBSAERO}" + +elif [[ "${step}" = "snowanl" ]]; then + + export NTHREADS_SNOWANL=${NTHREADSmax} + export APRUN_SNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_SNOWANL}" + + export APRUN_APPLY_INCR="${launcher} -n 6" + +elif [[ "${step}" = "marinebmat" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUN_MARINEBMAT="${APRUN_default}" + +elif [[ "${step}" = "marinebmat" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + export APRUN_MARINEBMAT="${APRUN_default}" + +elif [[ "${step}" = "ocnanalrun" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + + export APRUN_OCNANAL="${APRUN_default}" + +elif [[ "${step}" = "ocnanalchkpt" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd --multi-prog" + + export APRUN_OCNANAL="${APRUN_default}" + +elif [[ "${step}" = "ocnanalecen" ]]; then + + export NTHREADS_OCNANALECEN=${NTHREADSmax} + export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" + +elif [[ "${step}" = "marineanalletkf" ]]; then + + export NTHREADS_MARINEANALLETKF=${NTHREADSmax} + export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + +elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then + + export MKL_NUM_THREADS=4 + export MKL_CBWR=AUTO + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_GSI=${NTHREADSmax} + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" + + export NTHREADS_CALCINC=${threads_per_task_calcinc:-1} + [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task} + export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}" + + export NTHREADS_CYCLE=${threads_per_task_cycle:-12} + [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node} + ntasks_cycle=${ntiles:-6} + export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}" + + export NTHREADS_GAUSFCANL=1 + ntasks_gausfcanl=${ntasks_gausfcanl:-1} + export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}" + +elif [[ "${step}" = "sfcanl" ]]; then + + export NTHREADS_CYCLE=${threads_per_task:-14} + export APRUN_CYCLE="${APRUN_default} --cpus-per-task=${NTHREADS_CYCLE}" + +elif [[ "${step}" = "eobs" ]]; then + + export MKL_NUM_THREADS=4 + export MKL_CBWR=AUTO + + export NTHREADS_GSI=${NTHREADSmax} + export APRUN_GSI="${APRUN_default} --cpus-per-task=${NTHREADS_GSI}" + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + +elif [[ "${step}" = "eupd" ]]; then + + export NTHREADS_ENKF=${NTHREADSmax} + export APRUN_ENKF="${launcher} -n ${ntasks_enkf:-${ntasks}} --cpus-per-task=${NTHREADS_ENKF}" + + export CFP_MP=${CFP_MP:-"YES"} + export USE_CFP=${USE_CFP:-"YES"} + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + +elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then + + export launcher="srun --mpi=pmi2 -l" + + (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node )) + (( ufs_ntasks = nnodes*tasks_per_node )) + # With ESMF threading, the model wants to use the full node + export APRUN_UFS="${launcher} -n ${ufs_ntasks}" + unset nnodes ufs_ntasks + +elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then + + export CFP_MP="YES" + if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi + export wavempexec=${launcher} + export wave_mpmd=${mpmd_opt} + +elif [[ "${step}" = "post" ]]; then + + export NTHREADS_UPP=${NTHREADS1} + export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}" elif [[ "${step}" = "oceanice_products" ]]; then export NTHREADS_OCNICEPOST=${NTHREADS1} export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}" +elif [[ "${step}" = "atmos_products" ]]; then + + export USE_CFP="YES" # Use MPMD for downstream product generation on Hera + elif [[ "${step}" = "ecen" ]]; then export NTHREADS_ECEN=${NTHREADSmax} diff --git a/modulefiles/module_gwsetup.noaacloud.lua b/modulefiles/module_gwsetup.noaacloud.lua index a22aad5ede..94976524f1 100644 --- a/modulefiles/module_gwsetup.noaacloud.lua +++ b/modulefiles/module_gwsetup.noaacloud.lua @@ -6,9 +6,13 @@ load(pathJoin("rocoto")) prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/ue-intel/install/modulefiles/Core") prepend_path("MODULEPATH", "/apps/modules/modulefiles") + load("gnu") -load("stack-intel") -load("stack-intel-oneapi-mpi") +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0" +local stack_mpi_ver=os.getenv("stack_mpi_ver") or "2021.10.0" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-intel-oneapi-mpi", stack_mpi_ver)) unload("gnu") load("py-jinja2") diff --git a/sorc/build_ufs.sh b/sorc/build_ufs.sh index 44c8c7a2ad..ee784cb097 100755 --- a/sorc/build_ufs.sh +++ b/sorc/build_ufs.sh @@ -10,7 +10,7 @@ PDLIB="ON" while getopts ":da:fj:vw" option; do case "${option}" in - d) BUILD_TYPE="Debug";; + d) BUILD_TYPE="DEBUG";; a) APP="${OPTARG}";; f) FASTER="ON";; j) BUILD_JOBS="${OPTARG}";; From b0784842d6c0483f214ebc5ffc7bbb4208d5eadf Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 21 Nov 2024 04:31:50 +0000 Subject: [PATCH 04/14] make AWS use different partitions for fcst and others --- parm/config/gefs/config.resources | 2 +- parm/config/gefs/config.resources.AWSPW | 60 +++++++++++++++++++++++++ parm/config/gfs/config.base | 16 +++++++ parm/config/gfs/config.resources | 4 +- parm/config/gfs/config.resources.AWSPW | 24 ++++++++++ sorc/gfs_utils.fd | 2 +- sorc/ufs_model.fd | 2 +- sorc/ufs_utils.fd | 2 +- 8 files changed, 106 insertions(+), 6 deletions(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index 690fdf919a..d9e817ecca 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -41,7 +41,7 @@ case ${machine} in ;; "AWSPW") export PARTITION_BATCH="compute" - max_tasks_per_node=36 + max_tasks_per_node=48 ;; "AZUREPW") export PARTITION_BATCH="compute" diff --git a/parm/config/gefs/config.resources.AWSPW b/parm/config/gefs/config.resources.AWSPW index a735c7622d..310f030a3e 100644 --- a/parm/config/gefs/config.resources.AWSPW +++ b/parm/config/gefs/config.resources.AWSPW @@ -9,3 +9,63 @@ unset memory for mem_var in $(env | grep '^memory_' | cut -d= -f1); do unset "${mem_var}" done + +step=$1 + +echo "step: $step" + +case ${step} in + "fcst" | "efcs") + export PARTITION_BATCH="compute" + max_tasks_per_node=48 + ;; + + "arch") + export PARTITION_BATCH="process" + max_tasks_per_node=24 + ;; + + "prep_emissions") + export PARTITION_BATCH="process" + max_tasks_per_node=24 + export ntasks=1 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + ;; + + "waveinit") + export PARTITION_BATCH="process" + max_tasks_per_node=24 + export ntasks=12 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export NTASKS=${ntasks} + ;; + + "wavepostpnt") + export PARTITION_BATCH="compute" + max_tasks_per_node=48 + export ntasks=240 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export NTASKS=${ntasks} + ;; + + "wavepostsbs" | "wavepostbndpnt" | "wavepostbndpntbll") + export PARTITION_BATCH="process" + max_tasks_per_node=24 + export ntasks=24 + export threads_per_task=1 + export tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export NTASKS=${ntasks} + ;; + + *) + export PARTITION_BATCH="process" + max_tasks_per_node=24 + ;; + +esac + +export max_tasks_per_node + diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 4f702f9668..2500cd378a 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -483,4 +483,20 @@ export OFFSET_START_HOUR=0 # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} +# The tracker, genesis, and METplus jobs are not supported on CSPs yet +# TODO: we should place these in workflow/hosts/awspw.yaml as part of AWS/AZURE setup, not for general. +if [[ "${machine}" =~ "PW" ]]; then + export DO_TRACKER="NO" + export DO_GENESIS="NO" + export DO_METP="NO" + export DO_WAVE="NO" +fi + +# The tracker and genesis are not installed on Orion/Hercules yet; this requires spack-stack builds of the package. +# TODO: we should place these in workflow/hosts/[orion|hercules].yaml. +if [[ "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then + export DO_TRACKER="NO" + export DO_GENESIS="NO" +fi + echo "END: config.base" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index cddd1643fd..174c6c949b 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -107,8 +107,8 @@ case ${machine} in ;; "AWSPW") export PARTITION_BATCH="compute" - npe_node_max=36 - max_tasks_per_node=36 + npe_node_max=48 + max_tasks_per_node=48 # TODO Supply a max mem/node value for AWS # shellcheck disable=SC2034 mem_node_max="" diff --git a/parm/config/gfs/config.resources.AWSPW b/parm/config/gfs/config.resources.AWSPW index a735c7622d..22fe110670 100644 --- a/parm/config/gfs/config.resources.AWSPW +++ b/parm/config/gfs/config.resources.AWSPW @@ -9,3 +9,27 @@ unset memory for mem_var in $(env | grep '^memory_' | cut -d= -f1); do unset "${mem_var}" done + +step=$1 + +case ${step} in + "fcst" | "efcs") + export PARTITION_BATCH="compute" + max_tasks_per_node=48 + ;; + + "arch") + export PARTITION_BATCH="process" + max_tasks_per_node=24 + ;; + + + *) + export PARTITION_BATCH="process" + max_tasks_per_node=24 + ;; + +esac + +export max_tasks_per_node + diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 856a42076a..4848ecbb5e 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 856a42076a65256aaae9b29f4891532cb4a3fbca +Subproject commit 4848ecbb5e713b16127433e11f7d3edc6ac784c4 diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 6a4e09e947..29c2703c71 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 6a4e09e94773ffa39ce7ab6a54a885efada91f21 +Subproject commit 29c2703c715ebdb47bbd4bcc811db340eae530e5 diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index 06eec5b6f6..2323761084 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit 06eec5b6f636123835e2dfd9fc5229980c006735 +Subproject commit 23237610845c3a4438b21b25e9b3dc25c4c15b73 From cf62b9bf7ee67c6b544d449cba523f41830fe92b Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 21 Nov 2024 15:20:31 +0000 Subject: [PATCH 05/14] sync with develop --- env/AWSPW.env | 6 +----- modulefiles/module_base.noaacloud.lua | 4 ++-- modulefiles/module_gwsetup.noaacloud.lua | 1 - parm/config/gefs/config.resources.AWSPW | 2 -- versions/build.noaacloud.ver | 2 +- workflow/hosts/awspw.yaml | 2 +- 6 files changed, 5 insertions(+), 12 deletions(-) diff --git a/env/AWSPW.env b/env/AWSPW.env index 5e49e2a132..641ba4e09e 100755 --- a/env/AWSPW.env +++ b/env/AWSPW.env @@ -13,13 +13,9 @@ export launcher="srun -l --export=ALL" export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment -#export OMP_STACKSIZE=2048000 +export OMP_STACKSIZE=2048000 export NTHSTACK=1024000000 -export OMP_STACKSIZE=512M -export KMP_AFFINITY=scatter -export OMP_NUM_THREADS=1 - ulimit -s unlimited ulimit -a diff --git a/modulefiles/module_base.noaacloud.lua b/modulefiles/module_base.noaacloud.lua index fc8e6d0585..3a7cc75d7a 100644 --- a/modulefiles/module_base.noaacloud.lua +++ b/modulefiles/module_base.noaacloud.lua @@ -6,8 +6,8 @@ local spack_mod_path=(os.getenv("spack_mod_path") or "None") prepend_path("MODULEPATH", spack_mod_path) load("gnu") -load("stack-intel") -load("stack-intel-oneapi-mpi") +load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) +load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) unload("gnu") load(pathJoin("python", (os.getenv("python_ver") or "None"))) diff --git a/modulefiles/module_gwsetup.noaacloud.lua b/modulefiles/module_gwsetup.noaacloud.lua index 94976524f1..c9b3231c37 100644 --- a/modulefiles/module_gwsetup.noaacloud.lua +++ b/modulefiles/module_gwsetup.noaacloud.lua @@ -5,7 +5,6 @@ Load environment to run GFS workflow setup scripts on noaacloud load(pathJoin("rocoto")) prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/ue-intel/install/modulefiles/Core") -prepend_path("MODULEPATH", "/apps/modules/modulefiles") load("gnu") local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.10.0" diff --git a/parm/config/gefs/config.resources.AWSPW b/parm/config/gefs/config.resources.AWSPW index 310f030a3e..f91460b6aa 100644 --- a/parm/config/gefs/config.resources.AWSPW +++ b/parm/config/gefs/config.resources.AWSPW @@ -12,8 +12,6 @@ done step=$1 -echo "step: $step" - case ${step} in "fcst" | "efcs") export PARTITION_BATCH="compute" diff --git a/versions/build.noaacloud.ver b/versions/build.noaacloud.ver index 34caa15317..adda3ca19b 100644 --- a/versions/build.noaacloud.ver +++ b/versions/build.noaacloud.ver @@ -2,4 +2,4 @@ export stack_intel_ver=2021.3.0 export stack_impi_ver=2021.3.0 export spack_env=gsi-addon-env source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core" +export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index b98c838faa..ce7c187374 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -28,4 +28,4 @@ DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' SUPPORT_WAVES: 'NO' -SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. +SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. From 86b973ec29616ad234a1111dd7d2a4a9fa4a226d Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 21 Nov 2024 15:35:25 +0000 Subject: [PATCH 06/14] updates at Azure --- env/AZUREPW.env | 3 ++- workflow/hosts/azurepw.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/env/AZUREPW.env b/env/AZUREPW.env index a4baff94fa..b2b4063ff3 100755 --- a/env/AZUREPW.env +++ b/env/AZUREPW.env @@ -15,6 +15,7 @@ export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" # Configure MPI environment export OMP_STACKSIZE=2048000 export NTHSTACK=1024000000 +export UCX_TLS=ud,sm,self ulimit -s unlimited ulimit -a @@ -52,7 +53,7 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step} elif [[ "${step}" = "prep_emissions" ]]; then - export APRUN = ${APRUN_default} + export APRUN="${APRUN_default}" elif [[ "${step}" = "post" ]]; then diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index 4725e28962..e7747b3f25 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -25,4 +25,4 @@ ATARDIR: '' # TODO: This will not yet work from AZURE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' SUPPORT_WAVES: 'NO' -SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. +SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. From 944cfb344d232897c784f8e1ccd2adde09b75e9e Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 21 Nov 2024 16:39:03 +0000 Subject: [PATCH 07/14] GPC update to Rocky8 --- env/GOOGLEPW.env | 2 +- workflow/hosts/googlepw.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/env/GOOGLEPW.env b/env/GOOGLEPW.env index ba3887499f..d84008d648 100755 --- a/env/GOOGLEPW.env +++ b/env/GOOGLEPW.env @@ -45,7 +45,7 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then elif [[ "${step}" = "prep_emissions" ]]; then - export APRUN = ${APRUN_default} + export APRUN="${APRUN_default}" elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index 1b979b6bc9..63d21250e3 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -25,4 +25,4 @@ ATARDIR: '' # TODO: This will not yet work from GOOGLE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' SUPPORT_WAVES: 'NO' -SUPPORTED_RESOLUTIONS: ['C48', 'C96'] # TODO: Test and support all cubed-sphere resolutions. +SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384'] # TODO: Test and support all cubed-sphere resolutions. From bc7c970b605586fea1c5fca4c95fd76e83854933 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 21 Nov 2024 18:52:30 +0000 Subject: [PATCH 08/14] sync module versions --- modulefiles/module_gwci.noaacloud.lua | 6 +++--- modulefiles/module_gwsetup.noaacloud.lua | 2 -- versions/build.noaacloud.ver | 4 ++-- versions/run.noaacloud.ver | 6 +++--- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/modulefiles/module_gwci.noaacloud.lua b/modulefiles/module_gwci.noaacloud.lua index c3142cd60d..2ac284ef85 100644 --- a/modulefiles/module_gwci.noaacloud.lua +++ b/modulefiles/module_gwci.noaacloud.lua @@ -2,10 +2,10 @@ help([[ Load environment to run GFS workflow setup scripts on noaacloud ]]) -prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/ue-env/install/modulefiles/Core") -load(pathJoin("stack-intel", os.getenv("2021.3.0"))) -load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.3.0"))) +load(pathJoin("stack-intel", os.getenv("2021.10.0"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.10.0"))) load(pathJoin("netcdf-c", os.getenv("4.9.2"))) load(pathJoin("netcdf-fortran", os.getenv("4.6.1"))) diff --git a/modulefiles/module_gwsetup.noaacloud.lua b/modulefiles/module_gwsetup.noaacloud.lua index c9b3231c37..e2aa4050a3 100644 --- a/modulefiles/module_gwsetup.noaacloud.lua +++ b/modulefiles/module_gwsetup.noaacloud.lua @@ -17,7 +17,5 @@ unload("gnu") load("py-jinja2") load("py-pyyaml") load("py-numpy") ---local git_ver=os.getenv("git_ver") or "1.8.3.1" ---load(pathJoin("git", git_ver)) whatis("Description: GFS run setup environment") diff --git a/versions/build.noaacloud.ver b/versions/build.noaacloud.ver index adda3ca19b..b5fd272b4b 100644 --- a/versions/build.noaacloud.ver +++ b/versions/build.noaacloud.ver @@ -1,5 +1,5 @@ -export stack_intel_ver=2021.3.0 -export stack_impi_ver=2021.3.0 +export stack_intel_ver=2021.10.0 +export stack_impi_ver=2021.10.0 export spack_env=gsi-addon-env source "${HOMEgfs:-}/versions/spack.ver" export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" diff --git a/versions/run.noaacloud.ver b/versions/run.noaacloud.ver index 97173aa445..98ec2b36f9 100644 --- a/versions/run.noaacloud.ver +++ b/versions/run.noaacloud.ver @@ -1,8 +1,8 @@ -export stack_intel_ver=2021.3.0 -export stack_impi_ver=2021.3.0 +export stack_intel_ver=2021.10.0 +export stack_impi_ver=2021.10.0 export spack_env=gsi-addon-env source "${HOMEgfs:-}/versions/spack.ver" -export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core" +export spack_mod_path="/contrib/spack-stack-rocky8/spack-stack-${spack_stack_ver}/envs/gsi-addon-env/install/modulefiles/Core" export cdo_ver=2.2.0 From e97088dd099bf8b321956331fde8770ed7380569 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 21 Nov 2024 20:05:03 +0000 Subject: [PATCH 09/14] update azure resource --- parm/config/gfs/config.resources | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 174c6c949b..74d661b5be 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -115,8 +115,8 @@ case ${machine} in ;; "AZUREPW") export PARTITION_BATCH="compute" - npe_node_max=24 - max_tasks_per_node=24 + npe_node_max=36 + max_tasks_per_node=36 # TODO Supply a max mem/node value for AZURE # shellcheck disable=SC2034 mem_node_max="" From 3c85ff0180d0c181a4ba1260608b489111c1e726 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 21 Nov 2024 20:07:09 +0000 Subject: [PATCH 10/14] update azure resource --- parm/config/gefs/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index d9e817ecca..f396a6ccc4 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -45,7 +45,7 @@ case ${machine} in ;; "AZUREPW") export PARTITION_BATCH="compute" - max_tasks_per_node=24 + max_tasks_per_node=36 ;; "GOOGLEPW") export PARTITION_BATCH="compute" From 19e75a3739635e69adf34a2559e9ccffbfa55741 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Thu, 21 Nov 2024 20:15:43 +0000 Subject: [PATCH 11/14] update gcp resource --- parm/config/gefs/config.resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index d9e817ecca..dff56fd9ca 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -49,7 +49,7 @@ case ${machine} in ;; "GOOGLEPW") export PARTITION_BATCH="compute" - max_tasks_per_node=32 + max_tasks_per_node=30 ;; *) echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}" From fe0f3eef8b11e0a1ced8e76166554931e0f60b04 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Fri, 22 Nov 2024 00:12:58 +0000 Subject: [PATCH 12/14] wave runs fine on AWS for C48_S2SWA_gefs --- workflow/hosts/awspw.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index ce7c187374..83380c98b5 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -27,5 +27,5 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' -SUPPORT_WAVES: 'NO' +SUPPORT_WAVES: 'YES' SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. From ea07117c901c02b92e59d7c2eb3d3283fa953db0 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Tue, 26 Nov 2024 16:01:44 +0000 Subject: [PATCH 13/14] move CSPs specific packages to workflow/hosts/ --- parm/config/gfs/config.base | 16 ---------------- workflow/hosts/azurepw.yaml | 5 ++++- workflow/hosts/googlepw.yaml | 5 ++++- 3 files changed, 8 insertions(+), 18 deletions(-) diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base index 3160a96a57..8150d2e39c 100644 --- a/parm/config/gfs/config.base +++ b/parm/config/gfs/config.base @@ -479,20 +479,4 @@ export OFFSET_START_HOUR=0 # Number of regional collectives to create soundings for export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9} -# The tracker, genesis, and METplus jobs are not supported on CSPs yet -# TODO: we should place these in workflow/hosts/awspw.yaml as part of AWS/AZURE setup, not for general. -if [[ "${machine}" =~ "PW" ]]; then - export DO_TRACKER="NO" - export DO_GENESIS="NO" - export DO_METP="NO" - export DO_WAVE="NO" -fi - -# The tracker and genesis are not installed on Orion/Hercules yet; this requires spack-stack builds of the package. -# TODO: we should place these in workflow/hosts/[orion|hercules].yaml. -if [[ "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then - export DO_TRACKER="NO" - export DO_GENESIS="NO" -fi - echo "END: config.base" diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index e7747b3f25..61aa777ff3 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -24,5 +24,8 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from AZURE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORT_WAVES: 'NO' +DO_TRACKER: 'NO' +DO_GENESIS: 'NO' +DO_METP: 'NO' +SUPPORT_WAVES: 'YES' SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index 63d21250e3..ec71fd4f3a 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -24,5 +24,8 @@ LOCALARCH: 'NO' ATARDIR: '' # TODO: This will not yet work from GOOGLE. MAKE_NSSTBUFR: 'NO' MAKE_ACFTBUFR: 'NO' -SUPPORT_WAVES: 'NO' +DO_TRACKER: 'NO' +DO_GENESIS: 'NO' +DO_METP: 'NO' +SUPPORT_WAVES: 'YES' SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384'] # TODO: Test and support all cubed-sphere resolutions. From c81c8540a36a194cb7fe6350b0d5cbdea11809c0 Mon Sep 17 00:00:00 2001 From: Wei Huang Date: Wed, 27 Nov 2024 21:26:55 +0000 Subject: [PATCH 14/14] remove SUPPORT_WAVE from *pw.yaml, and setup_expt.py as WAVE works on all these platforms --- workflow/hosts/awspw.yaml | 1 - workflow/hosts/azurepw.yaml | 1 - workflow/hosts/googlepw.yaml | 1 - workflow/setup_expt.py | 4 ---- 4 files changed, 7 deletions(-) diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index 83380c98b5..ebc551ded2 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -27,5 +27,4 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' -SUPPORT_WAVES: 'YES' SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/azurepw.yaml b/workflow/hosts/azurepw.yaml index 61aa777ff3..d7c064dc60 100644 --- a/workflow/hosts/azurepw.yaml +++ b/workflow/hosts/azurepw.yaml @@ -27,5 +27,4 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' -SUPPORT_WAVES: 'YES' SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384', 'C768'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/hosts/googlepw.yaml b/workflow/hosts/googlepw.yaml index ec71fd4f3a..8ba8e18e74 100644 --- a/workflow/hosts/googlepw.yaml +++ b/workflow/hosts/googlepw.yaml @@ -27,5 +27,4 @@ MAKE_ACFTBUFR: 'NO' DO_TRACKER: 'NO' DO_GENESIS: 'NO' DO_METP: 'NO' -SUPPORT_WAVES: 'YES' SUPPORTED_RESOLUTIONS: ['C48', 'C96', 'C384'] # TODO: Test and support all cubed-sphere resolutions. diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 27da4943d3..c9b22ee8b3 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -371,7 +371,6 @@ def query_and_clean(dirname, force_clean=False): def validate_user_request(host, inputs): supp_res = host.info['SUPPORTED_RESOLUTIONS'] - supp_waves = host.info.get('SUPPORT_WAVES', 'YES') machine = host.machine for attr in ['resdetatmos', 'resensatmos']: try: @@ -381,9 +380,6 @@ def validate_user_request(host, inputs): if expt_res not in supp_res: raise NotImplementedError(f"Supported resolutions on {machine} are:\n{', '.join(supp_res)}") - if "W" in inputs.app and supp_waves == "NO": - raise NotImplementedError(f"Waves are not supported on {machine}") - def get_ocean_resolution(resdetatmos): """