From d4c55d1011f8b0385d25b62ac04710837ed8413e Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Fri, 19 Jan 2024 15:43:37 -0500 Subject: [PATCH 1/9] Update typing hint for WCOSS version of python (#2238) The typing hint `typing.List` was deprecated with python 3.9 in favor of using the primitive `list[str]`, but the functional version of python on WCOSS2 is <3.9, causing `setup_xml.py` to fail there. This replaces `list[str]` as a typing hint with the deprecated form until the supported version on WCOSS2 is >=3.9. --- workflow/rocoto/tasks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 8a40e645d0..d16a009eff 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -4,6 +4,7 @@ from applications.applications import AppConfig import rocoto.rocoto as rocoto from wxflow import Template, TemplateConstants, to_timedelta +from typing import List __all__ = ['Tasks'] @@ -124,7 +125,7 @@ def _template_to_rocoto_cycstring(self, template: str, subs_dict: dict = {}) -> rocoto_conversion_dict.get) @staticmethod - def _get_forecast_hours(cdump, config) -> list[str]: + def _get_forecast_hours(cdump, config) -> List[str]: fhmin = config['FHMIN'] fhmax = config['FHMAX'] fhout = config['FHOUT'] From f4d187f4e45fe89583d18987d68a883490827104 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Mon, 22 Jan 2024 12:28:28 -0500 Subject: [PATCH 2/9] Converts obsproc to obsprep in prepoceanobs config file (#2236) Converts obsproc to obsprep in prepoceanobs config file as a compliment to mutually depended GDASApp PR NOAA-EMC/GDASApp#858 The motivations are explained in refs NOAA-EMC/GDASApp#857 --- parm/config/gfs/config.prepoceanobs | 2 +- sorc/gdas.cd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/parm/config/gfs/config.prepoceanobs b/parm/config/gfs/config.prepoceanobs index d7c4e37bb9..56fc349ce2 100644 --- a/parm/config/gfs/config.prepoceanobs +++ b/parm/config/gfs/config.prepoceanobs @@ -7,7 +7,7 @@ echo "BEGIN: config.prepoceanobs" export OCNOBS2IODAEXEC=${HOMEgfs}/sorc/gdas.cd/build/bin/gdas_obsprovider2ioda.x export OBS_YAML_DIR=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/config -export OBSPROC_YAML=@OBSPROC_YAML@ +export OBSPREP_YAML=@OBSPREP_YAML@ export OBS_LIST=@SOCA_OBS_LIST@ [[ -n "${OBS_LIST}" ]] || export OBS_LIST=${HOMEgfs}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml export OBS_YAML=${OBS_LIST} diff --git a/sorc/gdas.cd b/sorc/gdas.cd index b79c0a2c36..6044d09962 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit b79c0a2c36cb632212e6991f784190310a2e0e41 +Subproject commit 6044d099626f568a1a671ea86398e2aedfaf72f4 From 2445d44d0d66f35512080b0bd5867501660793bb Mon Sep 17 00:00:00 2001 From: RussTreadon-NOAA <26926959+RussTreadon-NOAA@users.noreply.github.com> Date: Thu, 25 Jan 2024 06:17:11 -0500 Subject: [PATCH 3/9] Simplify and extend load_ufsda_modules to Hercules (#2245) GDASApp jobs do not run on Hercules because `ush/load_ufsda_modules.sh` does not include logic to load the appropriate GDASApp modules on Hercules. This PR extends `load_ufsda_modules.sh` functionality to Hercules, thereby enabling GDASApp jobs to run on Hercules. Resolves #2244 --- ush/load_ufsda_modules.sh | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/ush/load_ufsda_modules.sh b/ush/load_ufsda_modules.sh index da8e2d8096..1ab730c30e 100755 --- a/ush/load_ufsda_modules.sh +++ b/ush/load_ufsda_modules.sh @@ -44,20 +44,15 @@ elif [[ -d /scratch1 ]] ; then # set NETCDF variable based on ncdump location NETCDF=$( which ncdump ) export NETCDF - # prod_util stuff, find a better solution later... - module use /scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/compiler/intel/2022.1.2/ - module load prod_util elif [[ -d /work ]] ; then - # We are on MSU Orion - # prod_util stuff, find a better solution later... - #module use /apps/contrib/NCEP/hpc-stack/libs/hpc-stack/modulefiles/compiler/intel/2022.1.2/ - #module load prod_util - export UTILROOT=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2 - export MDATE=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2/bin/mdate - export NDATE=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2/bin/ndate - export NHOUR=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2/bin/nhour - export FSYNC=/work2/noaa/da/python/opt/intel-2022.1.2/prod_util/1.2.2/bin/fsync_file - module load "${MODS}/orion" + # We are on MSU Orion or Hercules + if [[ -d /apps/other ]] ; then + # Hercules + module load "${MODS}/hercules" + else + # Orion + module load "${MODS}/orion" + fi # set NETCDF variable based on ncdump location ncdump=$( which ncdump ) NETCDF=$( echo "${ncdump}" | cut -d " " -f 3 ) From ee6f536ea0228c60f5a8bec4037cd6f7ea63b816 Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Thu, 25 Jan 2024 07:43:13 -0500 Subject: [PATCH 4/9] Update GFS version to v16.3.13 in index.rst (#2256) GFSv16.3.13 WAFS update was implemented Refs #2013 --- docs/source/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/index.rst b/docs/source/index.rst index 2eb786199a..4381488078 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -10,7 +10,7 @@ Status ====== * State of develop (HEAD) branch: GFSv17+ development -* State of operations (dev/gfs.v16 branch): GFS v16.3.12 `tag: [gfs.v16.3.12] `_ +* State of operations (dev/gfs.v16 branch): GFS v16.3.13 `tag: [gfs.v16.3.13] `_ ============= Code managers From 553b4f2e74ef610115436b75f7f6df100babd8dd Mon Sep 17 00:00:00 2001 From: WenMeng-NOAA <48260754+WenMeng-NOAA@users.noreply.github.com> Date: Thu, 25 Jan 2024 13:00:45 -0500 Subject: [PATCH 5/9] Fix post parm links (#2243) Change symbolic links under parm/post with the latest version of develop branch from UPP repository and enable MERRA2 aerosol fields. Resolves #2259 --- parm/post/upp.yaml | 4 ++-- sorc/link_workflow.sh | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/parm/post/upp.yaml b/parm/post/upp.yaml index 651f3c12a8..ea7237dc83 100644 --- a/parm/post/upp.yaml +++ b/parm/post/upp.yaml @@ -28,7 +28,7 @@ analysis: forecast: config: - rdaod: False + rdaod: True data_in: copy: {% if forecast_hour == 0 %} @@ -47,7 +47,7 @@ forecast: goes: config: - rdaod: True + rdaod: False data_in: copy: {% set crtm_coefficients = [ diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 2e190e3285..6d5d40a354 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -130,15 +130,21 @@ cd "${HOMEgfs}/parm/ufs" || exit 1 ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_model.fd/tests/parm/noahmptable.tbl" . cd "${HOMEgfs}/parm/post" || exit 1 -for file in postxconfig-NT-GEFS-ANL.txt postxconfig-NT-GEFS-F00.txt postxconfig-NT-GEFS.txt postxconfig-NT-GFS-ANL.txt \ - postxconfig-NT-GFS-F00-TWO.txt postxconfig-NT-GFS-F00.txt postxconfig-NT-GFS-FLUX-F00.txt postxconfig-NT-GFS-FLUX.txt \ - postxconfig-NT-GFS-GOES.txt postxconfig-NT-GFS-TWO.txt \ - postxconfig-NT-GFS.txt postxconfig-NT-gefs-aerosol.txt postxconfig-NT-gefs-chem.txt params_grib2_tbl_new \ - post_tag_gfs128 post_tag_gfs65 nam_micro_lookup.dat \ - AEROSOL_LUTS.dat optics_luts_DUST.dat optics_luts_SALT.dat optics_luts_SOOT.dat optics_luts_SUSO.dat optics_luts_WASO.dat +for file in postxconfig-NT-GEFS-F00.txt postxconfig-NT-GEFS.txt postxconfig-NT-GEFS-WAFS.txt \ + postxconfig-NT-GEFS-F00-aerosol.txt postxconfig-NT-GEFS-aerosol.txt \ + postxconfig-NT-GFS-ANL.txt postxconfig-NT-GFS-F00.txt postxconfig-NT-GFS-FLUX-F00.txt \ + postxconfig-NT-GFS.txt postxconfig-NT-GFS-FLUX.txt postxconfig-NT-GFS-GOES.txt \ + postxconfig-NT-GFS-F00-TWO.txt postxconfig-NT-GFS-TWO.txt \ + params_grib2_tbl_new post_tag_gfs128 post_tag_gfs65 nam_micro_lookup.dat do ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/parm/${file}" . done +for file in optics_luts_DUST.dat optics_luts_DUST_nasa.dat optics_luts_NITR_nasa.dat \ + optics_luts_SALT.dat optics_luts_SALT_nasa.dat optics_luts_SOOT.dat optics_luts_SOOT_nasa.dat \ + optics_luts_SUSO.dat optics_luts_SUSO_nasa.dat optics_luts_WASO.dat optics_luts_WASO_nasa.dat +do + ${LINK_OR_COPY} "${HOMEgfs}/sorc/upp.fd/fix/chem/${file}" . +done cd "${HOMEgfs}/scripts" || exit 8 ${LINK_OR_COPY} "${HOMEgfs}/sorc/ufs_utils.fd/scripts/exemcsfc_global_sfc_prep.sh" . From 66f58b8ab1a9524d6be95271f27a06c2f32e5f78 Mon Sep 17 00:00:00 2001 From: Guillaume Vernieres Date: Thu, 25 Jan 2024 13:16:41 -0500 Subject: [PATCH 6/9] Added missing container case in gfs/config.resources (#2258) fixes #2257 --- parm/config/gfs/config.resources | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 26d5aa9126..d4530cfcc1 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -60,6 +60,9 @@ case ${machine} in export PARTITION_BATCH="compute" npe_node_max=40 ;; + "CONTAINER") + npe_node_max=1 + ;; *) echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}" exit 2 @@ -1159,4 +1162,4 @@ case ${step} in esac -echo "END: config.resources" \ No newline at end of file +echo "END: config.resources" From 3d44ff38c5c3324c22fc104fe3259b4ac864c6d6 Mon Sep 17 00:00:00 2001 From: Barry Baker Date: Thu, 25 Jan 2024 14:33:27 -0500 Subject: [PATCH 7/9] GOCART ExtData biogenic climatology fix (#2253) Updates the ExtData for biogenic emissions to be climatology rather than for current time. Fixes an issue which will crash by default for other years. --- parm/ufs/gocart/ExtData.other | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index c58b3e9ae8..7a0d63d6ca 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -54,10 +54,10 @@ pSO2_OCS NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # --------------- # # VOCs - OFFLINE MEGAN BIOG -OC_ISOPRENE NA N Y %y4-%m2-%d2t12:00:00 none none isoprene ExtData/nexus/MEGAN_OFFLINE_BVOC/v2021-12/MEGAN_OFFLINE_CLIMO_2000_2022_%m2.nc -OC_LIMO NA N Y %y4-%m2-%d2t12:00:00 none none limo ExtData/nexus/MEGAN_OFFLINE_BVOC/v2021-12/MEGAN_OFFLINE_CLIMO_2000_2022_%m2.nc -OC_MTPA NA N Y %y4-%m2-%d2t12:00:00 none none mtpa ExtData/nexus/MEGAN_OFFLINE_BVOC/v2021-12/MEGAN_OFFLINE_CLIMO_2000_2022_%m2.nc -OC_MTPO NA N Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_OFFLINE_BVOC/v2021-12/MEGAN_OFFLINE_CLIMO_2000_2022_%m2.nc +OC_ISOPRENE NA Y Y %y4-%m2-%d2t12:00:00 none none isoprene ExtData/nexus/MEGAN_OFFLINE_BVOC/v2021-12/MEGAN_OFFLINE_CLIMO_2000_2022_%m2.nc +OC_LIMO NA Y Y %y4-%m2-%d2t12:00:00 none none limo ExtData/nexus/MEGAN_OFFLINE_BVOC/v2021-12/MEGAN_OFFLINE_CLIMO_2000_2022_%m2.nc +OC_MTPA NA Y Y %y4-%m2-%d2t12:00:00 none none mtpa ExtData/nexus/MEGAN_OFFLINE_BVOC/v2021-12/MEGAN_OFFLINE_CLIMO_2000_2022_%m2.nc +OC_MTPO NA Y Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_OFFLINE_BVOC/v2021-12/MEGAN_OFFLINE_CLIMO_2000_2022_%m2.nc # Biofuel Source -- Included in AeroCom anthropogenic emissions OC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null From 8ff344844e28c3b2d03a0356f88b14635f318c12 Mon Sep 17 00:00:00 2001 From: Rahul Mahajan Date: Fri, 26 Jan 2024 10:12:18 -0500 Subject: [PATCH 8/9] Add a yaml for snow DA testing. (#2199) - adds a new test yaml C96_atmsnowDA.yaml for 3DVar atmosphere with GSI and Land (Snow) DA with JEDI - moves a few yamls from platforms/ to yamls/ - adds ability to overwrite a previously created experiment as an addition to user input. --------- Co-authored-by: Cory Martin --- ci/cases/pr/C48_ATM.yaml | 2 +- ci/cases/pr/C48_S2SW.yaml | 2 +- ci/cases/pr/C48_S2SWA_gefs.yaml | 2 +- ci/cases/pr/C96C48_hybatmDA.yaml | 2 +- ci/cases/pr/C96_atm3DVar.yaml | 2 +- ci/cases/pr/C96_atmsnowDA.yaml | 21 +++++++++++++++++ ci/cases/weekly/C384C192_hybatmda.yaml | 2 +- ci/cases/weekly/C384_S2SWA.yaml | 2 +- ci/cases/weekly/C384_atm3DVar.yaml | 2 +- ci/cases/yamls/atmsnowDA_defaults_ci.yaml | 6 +++++ .../yamls}/gefs_ci_defaults.yaml | 0 .../yamls}/gfs_defaults_ci.yaml | 0 jobs/JGLOBAL_LAND_ANALYSIS | 1 + jobs/JGLOBAL_PREP_LAND_OBS | 1 + sorc/gdas.cd | 2 +- ush/python/pygfs/task/land_analysis.py | 11 +++++---- workflow/create_experiment.py | 7 +++++- workflow/setup_expt.py | 23 +++++++++++++------ 18 files changed, 66 insertions(+), 22 deletions(-) create mode 100644 ci/cases/pr/C96_atmsnowDA.yaml create mode 100644 ci/cases/yamls/atmsnowDA_defaults_ci.yaml rename ci/{platforms => cases/yamls}/gefs_ci_defaults.yaml (100%) rename ci/{platforms => cases/yamls}/gfs_defaults_ci.yaml (100%) diff --git a/ci/cases/pr/C48_ATM.yaml b/ci/cases/pr/C48_ATM.yaml index 39412e8aeb..79706556e6 100644 --- a/ci/cases/pr/C48_ATM.yaml +++ b/ci/cases/pr/C48_ATM.yaml @@ -10,4 +10,4 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2021032312 edate: 2021032312 - yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C48_S2SW.yaml b/ci/cases/pr/C48_S2SW.yaml index 2aba42f562..6367564514 100644 --- a/ci/cases/pr/C48_S2SW.yaml +++ b/ci/cases/pr/C48_S2SW.yaml @@ -11,4 +11,4 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2021032312 edate: 2021032312 - yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml index d68360bf44..d42f4cd15b 100644 --- a/ci/cases/pr/C48_S2SWA_gefs.yaml +++ b/ci/cases/pr/C48_S2SWA_gefs.yaml @@ -15,4 +15,4 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2021032312 edate: 2021032312 - yaml: {{ HOMEgfs }}/ci/platforms/gefs_ci_defaults.yaml + yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml diff --git a/ci/cases/pr/C96C48_hybatmDA.yaml b/ci/cases/pr/C96C48_hybatmDA.yaml index be35283cff..d08374d4e0 100644 --- a/ci/cases/pr/C96C48_hybatmDA.yaml +++ b/ci/cases/pr/C96C48_hybatmDA.yaml @@ -16,4 +16,4 @@ arguments: nens: 2 gfs_cyc: 1 start: cold - yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/ci/cases/pr/C96_atm3DVar.yaml index dee1525d80..d992938f7f 100644 --- a/ci/cases/pr/C96_atm3DVar.yaml +++ b/ci/cases/pr/C96_atm3DVar.yaml @@ -14,4 +14,4 @@ arguments: nens: 0 gfs_cyc: 1 start: cold - yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/pr/C96_atmsnowDA.yaml b/ci/cases/pr/C96_atmsnowDA.yaml new file mode 100644 index 0000000000..35fcc10fb2 --- /dev/null +++ b/ci/cases/pr/C96_atmsnowDA.yaml @@ -0,0 +1,21 @@ +experiment: + system: gfs + mode: cycled + +arguments: + pslot: {{ 'pslot' | getenv }} + app: ATM + resdetatmos: 96 + comroot: {{ 'RUNTESTS' | getenv }}/COMROOT + expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48 + idate: 2021122012 + edate: 2021122100 + nens: 0 + gfs_cyc: 1 + start: cold + yaml: {{ HOMEgfs }}/ci/cases/yamls/atmsnowDA_defaults_ci.yaml + +skip_ci_on_hosts: + - orion + - hercules diff --git a/ci/cases/weekly/C384C192_hybatmda.yaml b/ci/cases/weekly/C384C192_hybatmda.yaml index a4eae7d9a1..131ada95d5 100644 --- a/ci/cases/weekly/C384C192_hybatmda.yaml +++ b/ci/cases/weekly/C384C192_hybatmda.yaml @@ -16,4 +16,4 @@ arguments: nens: 2 gfs_cyc: 1 start: cold - yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/weekly/C384_S2SWA.yaml b/ci/cases/weekly/C384_S2SWA.yaml index 813188015f..7bbdc44671 100644 --- a/ci/cases/weekly/C384_S2SWA.yaml +++ b/ci/cases/weekly/C384_S2SWA.yaml @@ -11,4 +11,4 @@ arguments: expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR idate: 2016070100 edate: 2016070100 - yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/weekly/C384_atm3DVar.yaml b/ci/cases/weekly/C384_atm3DVar.yaml index 479d731b25..40487f3b47 100644 --- a/ci/cases/weekly/C384_atm3DVar.yaml +++ b/ci/cases/weekly/C384_atm3DVar.yaml @@ -16,4 +16,4 @@ arguments: nens: 0 gfs_cyc: 1 start: cold - yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml + yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/ci/cases/yamls/atmsnowDA_defaults_ci.yaml b/ci/cases/yamls/atmsnowDA_defaults_ci.yaml new file mode 100644 index 0000000000..f805902931 --- /dev/null +++ b/ci/cases/yamls/atmsnowDA_defaults_ci.yaml @@ -0,0 +1,6 @@ +defaults: + !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml +base: + DOIAU: "NO" + DO_JEDILANDDA: "YES" + ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }} diff --git a/ci/platforms/gefs_ci_defaults.yaml b/ci/cases/yamls/gefs_ci_defaults.yaml similarity index 100% rename from ci/platforms/gefs_ci_defaults.yaml rename to ci/cases/yamls/gefs_ci_defaults.yaml diff --git a/ci/platforms/gfs_defaults_ci.yaml b/ci/cases/yamls/gfs_defaults_ci.yaml similarity index 100% rename from ci/platforms/gfs_defaults_ci.yaml rename to ci/cases/yamls/gfs_defaults_ci.yaml diff --git a/jobs/JGLOBAL_LAND_ANALYSIS b/jobs/JGLOBAL_LAND_ANALYSIS index 3ff7e72a35..0cef66de09 100755 --- a/jobs/JGLOBAL_LAND_ANALYSIS +++ b/jobs/JGLOBAL_LAND_ANALYSIS @@ -1,6 +1,7 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" +export DATA=${DATA:-${DATAROOT}/${RUN}landanl_${cyc}} source "${HOMEgfs}/ush/jjob_header.sh" -e "landanl" -c "base landanl" ############################################## diff --git a/jobs/JGLOBAL_PREP_LAND_OBS b/jobs/JGLOBAL_PREP_LAND_OBS index 025adae529..9b14451568 100755 --- a/jobs/JGLOBAL_PREP_LAND_OBS +++ b/jobs/JGLOBAL_PREP_LAND_OBS @@ -1,6 +1,7 @@ #! /usr/bin/env bash source "${HOMEgfs}/ush/preamble.sh" +export DATA=${DATA:-${DATAROOT}/${RUN}landanl_${cyc}} source "${HOMEgfs}/ush/jjob_header.sh" -e "preplandobs" -c "base preplandobs" ############################################## diff --git a/sorc/gdas.cd b/sorc/gdas.cd index 6044d09962..32ce3f53d2 160000 --- a/sorc/gdas.cd +++ b/sorc/gdas.cd @@ -1 +1 @@ -Subproject commit 6044d099626f568a1a671ea86398e2aedfaf72f4 +Subproject commit 32ce3f53d203e243399c66aa1bb34020d303ee8d diff --git a/ush/python/pygfs/task/land_analysis.py b/ush/python/pygfs/task/land_analysis.py index 307e875183..821caf2305 100644 --- a/ush/python/pygfs/task/land_analysis.py +++ b/ush/python/pygfs/task/land_analysis.py @@ -26,7 +26,7 @@ class LandAnalysis(Analysis): Class for global land analysis tasks """ - NMEM_LANDENS = 2 # The size of the land ensemble is fixed at 2. Does this need to be a variable? + NMEM_LANDENS = 2 @logit(logger, name="LandAnalysis") def __init__(self, config): @@ -74,7 +74,7 @@ def prepare_GTS(self) -> None: # create a temporary dict of all keys needed in this method localconf = AttrDict() keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', - 'OPREFIX', 'CASE', 'ntiles'] + 'OPREFIX', 'CASE', 'OCNRES', 'ntiles'] for key in keys: localconf[key] = self.task_config[key] @@ -198,7 +198,7 @@ def prepare_IMS(self) -> None: raise WorkflowException(f"An error occured during execution of {exe}") # Ensure the snow depth IMS file is produced by the above executable - input_file = f"IMSscf.{to_YMD(localconf.current_cycle)}.{localconf.CASE}.mx{localconf.OCNRES}_oro_data.nc" + input_file = f"IMSscf.{to_YMD(localconf.current_cycle)}.{localconf.CASE}_oro_data.nc" if not os.path.isfile(f"{os.path.join(localconf.DATA, input_file)}"): logger.exception(f"{self.task_config.CALCFIMSEXE} failed to produce {input_file}") raise FileNotFoundError(f"{os.path.join(localconf.DATA, input_file)}") @@ -249,7 +249,7 @@ def initialize(self) -> None: # create a temporary dict of all keys needed in this method localconf = AttrDict() keys = ['DATA', 'current_cycle', 'COM_OBS', 'COM_ATMOS_RESTART_PREV', - 'OPREFIX', 'CASE', 'ntiles'] + 'OPREFIX', 'CASE', 'OCNRES', 'ntiles'] for key in keys: localconf[key] = self.task_config[key] @@ -301,7 +301,7 @@ def execute(self) -> None: localconf = AttrDict() keys = ['HOMEgfs', 'DATA', 'current_cycle', 'COM_ATMOS_RESTART_PREV', 'COM_LAND_ANALYSIS', 'APREFIX', - 'SNOWDEPTHVAR', 'BESTDDEV', 'CASE', 'ntiles', + 'SNOWDEPTHVAR', 'BESTDDEV', 'CASE', 'OCNRES', 'ntiles', 'APRUN_LANDANL', 'JEDIEXE', 'jedi_yaml', 'APPLY_INCR_NML_TMPL', 'APPLY_INCR_EXE', 'APRUN_APPLY_INCR'] for key in keys: @@ -530,6 +530,7 @@ def add_increments(config: Dict) -> None: DATA current_cycle CASE + OCNRES ntiles APPLY_INCR_NML_TMPL APPLY_INCR_EXE diff --git a/workflow/create_experiment.py b/workflow/create_experiment.py index 7e0f350c0f..708cf432bf 100755 --- a/workflow/create_experiment.py +++ b/workflow/create_experiment.py @@ -63,7 +63,9 @@ def input_args(): formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument( - '--yaml', help='full path to yaml file describing the experiment configuration', type=Path, required=True) + '-y', '--yaml', help='full path to yaml file describing the experiment configuration', type=Path, required=True) + parser.add_argument( + '-o', '--overwrite', help='overwrite previously created experiment', action="store_true", required=False) return parser.parse_args() @@ -89,6 +91,9 @@ def input_args(): setup_expt_args.append(f"--{kk}") setup_expt_args.append(str(vv)) + if user_inputs.overwrite: + setup_expt_args.append("--overwrite") + logger.info(f"Call: setup_expt.main()") logger.debug(f"setup_expt.py {' '.join(setup_expt_args)}") setup_expt.main(setup_expt_args) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 7d7ac84aad..df9687bb72 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -399,6 +399,8 @@ def _common_args(parser): parser.add_argument('--idate', help='starting date of experiment, initial conditions must exist!', required=True, type=lambda dd: to_datetime(dd)) parser.add_argument('--edate', help='end date experiment', required=True, type=lambda dd: to_datetime(dd)) + parser.add_argument('--overwrite', help='overwrite previously created experiment (if it exists)', + action='store_true', required=False) return parser def _gfs_args(parser): @@ -493,17 +495,19 @@ def _gefs_args(parser): return parser.parse_args(list(*argv) if len(argv) else None) -def query_and_clean(dirname): +def query_and_clean(dirname, force_clean=False): """ Method to query if a directory exists and gather user input for further action """ create_dir = True if os.path.exists(dirname): - print() - print(f'directory already exists in {dirname}') - print() - overwrite = input('Do you wish to over-write [y/N]: ') + print(f'\ndirectory already exists in {dirname}') + if force_clean: + overwrite = True + print(f'removing directory ........ {dirname}\n') + else: + overwrite = input('Do you wish to over-write [y/N]: ') create_dir = True if overwrite in [ 'y', 'yes', 'Y', 'YES'] else False if create_dir: @@ -553,8 +557,8 @@ def main(*argv): rotdir = os.path.join(user_inputs.comroot, user_inputs.pslot) expdir = os.path.join(user_inputs.expdir, user_inputs.pslot) - create_rotdir = query_and_clean(rotdir) - create_expdir = query_and_clean(expdir) + create_rotdir = query_and_clean(rotdir, force_clean=user_inputs.overwrite) + create_expdir = query_and_clean(expdir, force_clean=user_inputs.overwrite) if create_rotdir: makedirs_if_missing(rotdir) @@ -565,6 +569,11 @@ def main(*argv): fill_EXPDIR(user_inputs) update_configs(host, user_inputs) + print(f"*" * 100) + print(f'EXPDIR: {expdir}') + print(f'ROTDIR: {rotdir}') + print(f"*" * 100) + if __name__ == '__main__': From d5bee38979cde547861261d1cd150f3a61601d4b Mon Sep 17 00:00:00 2001 From: Kate Friedman Date: Mon, 29 Jan 2024 14:35:02 -0500 Subject: [PATCH 9/9] Correct typos in GFS config.resources (#2267) This PR corrects some typos in `parm/config/gfs/config.resources` that were introduced in PR #2216. The esfc job was failing in tests on WCOSS2 due to insufficient memory. This lead to discovering the other typos. The esfc job completes without error after its memory is set back to `80GB` from the incorrect `8GB`. Resolves #2266 --- parm/config/gfs/config.resources | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index d4530cfcc1..36d8e788c3 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -813,7 +813,7 @@ case ${step} in export npe_cleanup=1 export npe_node_cleanup=1 export nth_cleanup=1 - export memory_cleanu="4096M" + export memory_cleanup="4096M" ;; "stage_ic") @@ -821,7 +821,7 @@ case ${step} in export npe_stage_ic=1 export npe_node_stage_ic=1 export nth_stage_ic=1 - export is_exclusive=Tue + export is_exclusive=True ;; "atmensanlinit") @@ -956,7 +956,7 @@ case ${step} in export npe_node_esfc=$(( npe_node_max / nth_esfc )) export nth_cycle=${nth_esfc} export npe_node_cycle=$(( npe_node_max / nth_cycle )) - export memory_esfc="8GB" + export memory_esfc="80GB" ;; "epos") @@ -986,7 +986,7 @@ case ${step} in export npe_awips=1 export npe_node_awips=1 export nth_awips=1 - export memory_awip="3GB" + export memory_awips="3GB" ;; "npoess") @@ -994,7 +994,7 @@ case ${step} in export npe_npoess=1 export npe_node_npoess=1 export nth_npoess=1 - export memory_npoes="3GB" + export memory_npoess="3GB" ;; "gempak") @@ -1015,7 +1015,7 @@ case ${step} in export nth_mos_stn_prep=1 export memory_mos_stn_prep="5GB" export NTASK="${npe_mos_stn_prep}" - export PTILE="${npe_node_mos_stn_pep}" + export PTILE="${npe_node_mos_stn_prep}" ;; "mos_grd_prep") @@ -1025,7 +1025,7 @@ case ${step} in export nth_mos_grd_prep=1 export memory_mos_grd_prep="16GB" export NTASK="${npe_mos_grd_prep}" - export PTILE="${npe_node_mos_grd_pep}" + export PTILE="${npe_node_mos_grd_prep}" ;; "mos_ext_stn_prep") @@ -1035,7 +1035,7 @@ case ${step} in export nth_mos_ext_stn_prep=1 export memory_mos_ext_stn_prep="5GB" export NTASK="${npe_mos_ext_stn_prep}" - export PTILE="${npe_node_mos_ext_stn_pep}" + export PTILE="${npe_node_mos_ext_stn_prep}" ;; "mos_ext_grd_prep") @@ -1045,7 +1045,7 @@ case ${step} in export nth_mos_ext_grd_prep=1 export memory_mos_ext_grd_prep="3GB" export NTASK="${npe_mos_ext_grd_prep}" - export PTILE="${npe_node_mos_ext_grd_pep}" + export PTILE="${npe_node_mos_ext_grd_prep}" ;; "mos_stn_fcst") @@ -1055,7 +1055,7 @@ case ${step} in export nth_mos_stn_fcst=1 export memory_mos_stn_fcst="40GB" export NTASK="${npe_mos_stn_fcst}" - export PTILE="${npe_node_mos_stn_fst}" + export PTILE="${npe_node_mos_stn_fcst}" ;; "mos_grd_fcst") @@ -1065,7 +1065,7 @@ case ${step} in export nth_mos_grd_fcst=1 export memory_mos_grd_fcst="50GB" export NTASK="${npe_mos_grd_fcst}" - export PTILE="${npe_node_mos_grd_fst}" + export PTILE="${npe_node_mos_grd_fcst}" ;; "mos_ext_stn_fcst") @@ -1086,7 +1086,7 @@ case ${step} in export nth_mos_ext_grd_fcst=1 export memory_mos_ext_grd_fcst="50GB" export NTASK="${npe_mos_ext_grd_fcst}" - export PTILE="${npe_node_mos_ext_grd_fst}" + export PTILE="${npe_node_mos_ext_grd_fcst}" ;; "mos_stn_prdgen") @@ -1108,7 +1108,7 @@ case ${step} in export memory_mos_grd_prdgen="20GB" export NTASK="${npe_mos_grd_prdgen}" export PTILE="${npe_node_mos_grd_prdgen}" - export OMP_NUM_THREADS="${nth_mos_grd_prden}" + export OMP_NUM_THREADS="${nth_mos_grd_prdgen}" ;; "mos_ext_stn_prdgen") @@ -1130,7 +1130,7 @@ case ${step} in export memory_mos_ext_grd_prdgen="30GB" export NTASK="${npe_mos_ext_grd_prdgen}" export PTILE="${npe_node_mos_ext_grd_prdgen}" - export OMP_NUM_THREADS="${nth_mos_ext_grd_prden}" + export OMP_NUM_THREADS="${nth_mos_ext_grd_prdgen}" ;; "mos_wx_prdgen") @@ -1141,7 +1141,7 @@ case ${step} in export memory_mos_wx_prdgen="10GB" export NTASK="${npe_mos_wx_prdgen}" export PTILE="${npe_node_mos_wx_prdgen}" - export OMP_NUM_THREADS="${nth_mos_wx_prden}" + export OMP_NUM_THREADS="${nth_mos_wx_prdgen}" ;; "mos_wx_ext_prdgen")