Merge remote-tracking branch 'origin/develop' into feature/debug_switch

* origin/develop: Return ocnanalrun npes resource setting back to previous value (NOAA-EMC#2386) Add new BUFR table file parm/product/bufr_ij9km.txt for GFSv17 C1152 (NOAA-EMC#2383) Add global-workflow infrastructure for ocean analysis recentering task (NOAA-EMC#2299) Clean out non-gfs top level variables (NOAA-EMC#2366) Forgotten templated DO_VRFY_OCEANDA (NOAA-EMC#2379) Add Hercules support for the GSI monitor (NOAA-EMC#2373) Reformat snowDA templates to jinja2 (NOAA-EMC#2371) Reduce Jenkins messaging to GitHub (NOAA-EMC#2370) Adding debug option for all build scripts (NOAA-EMC#2326) Sort list of coupler restart files for restart time determination (NOAA-EMC#2360) Update stage IC to handle ocean perturbations (NOAA-EMC#2364) Turn on C48mx500_3DVarAOWCDA test on hera (NOAA-EMC#2363) Move Jenkinsfile into ci subdirectory (NOAA-EMC#2355)
KateFriedman-NOAA · Mar 11, 2024 · 1b92914 · 1b92914
2 parents c313ce5 + a337460
commit 1b92914
Show file tree

Hide file tree

Showing 195 changed files with 3,021 additions and 1,131 deletions.
diff --git a/.gitignore b/.gitignore
@@ -30,7 +30,6 @@ fix/chem
 fix/cice
 fix/cpl
 fix/datm
-fix/gldas
 fix/gdas
 fix/gsi
 fix/lut
@@ -43,8 +42,12 @@ fix/wave
 
 # Ignore parm file symlinks
 #--------------------------
-parm/config/config.base
-parm/gldas
+parm/gdas/aero
+parm/gdas/atm
+parm/gdas/io
+parm/gdas/ioda
+parm/gdas/snow
+parm/gdas/soca
 parm/monitor
 parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat

diff --git a/.gitmodules b/.gitmodules
@@ -1,28 +1,28 @@
 [submodule "sorc/ufs_model.fd"]
-	path = sorc/ufs_model.fd
-	url = https://github.com/ufs-community/ufs-weather-model
-	ignore = dirty
+  path = sorc/ufs_model.fd
+  url = https://github.com/ufs-community/ufs-weather-model
+  ignore = dirty
 [submodule "sorc/wxflow"]
-	path = sorc/wxflow
-	url = https://github.com/NOAA-EMC/wxflow
+  path = sorc/wxflow
+  url = https://github.com/NOAA-EMC/wxflow
 [submodule "sorc/gfs_utils.fd"]
-	path = sorc/gfs_utils.fd
-	url = https://github.com/NOAA-EMC/gfs-utils
+  path = sorc/gfs_utils.fd
+  url = https://github.com/NOAA-EMC/gfs-utils
 [submodule "sorc/ufs_utils.fd"]
-	path = sorc/ufs_utils.fd
-	url = https://github.com/ufs-community/UFS_UTILS.git
+  path = sorc/ufs_utils.fd
+  url = https://github.com/ufs-community/UFS_UTILS.git
 [submodule "sorc/verif-global.fd"]
-	path = sorc/verif-global.fd
-	url = https://github.com/NOAA-EMC/EMC_verif-global.git
+  path = sorc/verif-global.fd
+  url = https://github.com/NOAA-EMC/EMC_verif-global.git
 [submodule "sorc/gsi_enkf.fd"]
-	path = sorc/gsi_enkf.fd
-	url = https://github.com/NOAA-EMC/GSI.git
+  path = sorc/gsi_enkf.fd
+  url = https://github.com/NOAA-EMC/GSI.git
 [submodule "sorc/gdas.cd"]
-	path = sorc/gdas.cd
-	url = https://github.com/NOAA-EMC/GDASApp.git
+  path = sorc/gdas.cd
+  url = https://github.com/NOAA-EMC/GDASApp.git
 [submodule "sorc/gsi_utils.fd"]
-	path = sorc/gsi_utils.fd
-	url = https://github.com/NOAA-EMC/GSI-Utils.git
+  path = sorc/gsi_utils.fd
+  url = https://github.com/NOAA-EMC/GSI-Utils.git
 [submodule "sorc/gsi_monitor.fd"]
-	path = sorc/gsi_monitor.fd
-	url = https://github.com/NOAA-EMC/GSI-Monitor.git
+  path = sorc/gsi_monitor.fd
+  url = https://github.com/NOAA-EMC/GSI-Monitor.git
diff --git a/Jenkinsfile → ci/Jenkinsfile b/Jenkinsfile → ci/Jenkinsfile
@@ -50,7 +50,7 @@ pipeline {
                             pullRequest.removeLabel("CI-${Machine}-Ready")
                         }
                     }
-                    pullRequest.comment("Building and running on ${Machine} in directory ${HOME}")
+                    echo "Building and running on ${Machine} in directory ${HOME}"
                 }
             }
         }
@@ -76,7 +76,6 @@ pipeline {
                                 ws(HOMEgfs) {
                                     if (fileExists("${HOMEgfs}/sorc/BUILT_semaphor")) { // if the system is already built, skip the build in the case of re-runs
                                         sh(script: "cat ${HOMEgfs}/sorc/BUILT_semaphor", returnStdout: true).trim() // TODO: and user configurable control to manage build semphore
-                                        pullRequest.comment("Cloned PR already built (or build skipped) on ${machine} in directory ${HOMEgfs}<br>Still doing a checkout to get the latest changes")
                                         checkout scm
                                         dir('sorc') {
                                             sh(script: './link_workflow.sh')
@@ -93,10 +92,14 @@ pipeline {
                                         }
                                     }
                                     if (env.CHANGE_ID && system == 'gfs') {
-                                       if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) {
-                                           pullRequest.removeLabel("CI-${Machine}-Building")
-                                       }
-                                       pullRequest.addLabel("CI-${Machine}-Running")
+                                        try {
+                                            if (pullRequest.labels.any { value -> value.matches("CI-${Machine}-Building") }) {
+                                               pullRequest.removeLabel("CI-${Machine}-Building")
+                                            }
+                                            pullRequest.addLabel("CI-${Machine}-Running")
+                                        } catch (Exception e) {
+                                            echo "Failed to update label from Buildng to Running: ${e.getMessage()}"
+                                        }
                                     }
                                     if (system == 'gfs') {
                                         caseList = sh(script: "${HOMEgfs}/ci/scripts/utils/get_host_case_list.py ${machine}", returnStdout: true).trim().split()
@@ -146,11 +149,9 @@ pipeline {
                                 HOMEgfs = "${HOME}/gfs"  // common HOMEgfs is used to launch the scripts that run the experiments
                                 ws(HOMEgfs) {
                                     pslot = sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh get_pslot ${HOME}/RUNTESTS ${Case}", returnStdout: true).trim()
-                                    pullRequest.comment("**Running** experiment: ${Case} on ${Machine}<br>With the experiment in directory:<br>`${HOME}/RUNTESTS/${pslot}`")
                                     try {
                                        sh(script: "${HOMEgfs}/ci/scripts/run-check_ci.sh ${HOME} ${pslot}")
                                     } catch (Exception e) {
-                                        pullRequest.comment("**FAILURE** running experiment: ${Case} on ${Machine}")
                                         sh(script: "${HOMEgfs}/ci/scripts/utils/ci_utils_wrapper.sh cancel_all_batch_jobs ${HOME}/RUNTESTS")
                                         ws(HOME) {
                                             if (fileExists('RUNTESTS/error.logs')) {
@@ -165,7 +166,6 @@ pipeline {
                                         error("Failed to run experiments ${Case} on ${Machine}")
                                     }
                                 }
-                                pullRequest.comment("**SUCCESS** running experiment: ${Case} on ${Machine}")
                             }
                         }
                     }
@@ -179,29 +179,41 @@ pipeline {
         always {
             script {
                 if(env.CHANGE_ID) {
-                    for (label in pullRequest.labels) {
-                        if (label.contains("${Machine}")) {
-                            pullRequest.removeLabel(label)
+                    try {
+                        for (label in pullRequest.labels) {
+                            if (label.contains("${Machine}")) {
+                                pullRequest.removeLabel(label)
+                            }
                         }
+                    }  catch (Exception e) {
+                    echo "Failed to remove labels: ${e.getMessage()}"
                     }
                 }
             }
         }
         success {
             script {
                 if(env.CHANGE_ID) {
-                    pullRequest.addLabel("CI-${Machine}-Passed")
-                    def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
-                    pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
+                    try {
+                        pullRequest.addLabel("CI-${Machine}-Passed")
+                        def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
+                        pullRequest.comment("**CI SUCCESS** ${Machine} at ${timestamp}\n\nBuilt and ran in directory `${HOME}`")
+                    } catch (Exception e) {
+                        echo "Failed to add success label or comment: ${e.getMessage()}"
+                    }
                 }
             }
         }
         failure {
             script {
                 if(env.CHANGE_ID) {
-                    pullRequest.addLabel("CI-${Machine}-Failed")
-                    def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
-                    pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
+                    try {
+                        pullRequest.addLabel("CI-${Machine}-Failed")
+                        def timestamp = new Date().format('MM dd HH:mm:ss', TimeZone.getTimeZone('America/New_York'))
+                        pullRequest.comment("**CI FAILED** ${Machine} at ${timestamp}<br>Built and ran in directory `${HOME}`")
+                    } catch (Exception e) {
+                        echo "Failed to add failure label or comment: ${e.getMessage()}"
+                    }
                 }
             }
         }

diff --git a/env/HERA.env b/env/HERA.env
@@ -107,6 +107,14 @@ elif [[ "${step}" = "ocnanalchkpt" ]]; then
 
     export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt}"
 
+elif [[ "${step}" = "ocnanalecen" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_ocnanalecen))
+
+    export NTHREADS_OCNANALECEN=${nth_ocnanalecen:-${nth_max}}
+    [[ ${NTHREADS_OCNANALECEN} -gt ${nth_max} ]] && export NTHREADS_OCNANALECEN=${nth_max}
+    export APRUN_OCNANALECEN="${launcher} -n ${npe_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

diff --git a/env/ORION.env b/env/ORION.env
@@ -118,6 +118,14 @@ elif [[ "${step}" = "ocnanalchkpt" ]]; then
     [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
     export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt} --cpus-per-task=${NTHREADS_OCNANAL}"
 
+elif [[ "${step}" = "ocnanalecen" ]]; then
+
+    nth_max=$((npe_node_max / npe_node_ocnanalecen))
+
+    export NTHREADS_OCNANALECEN=${nth_ocnanalecen:-${nth_max}}
+    [[ ${NTHREADS_OCNANALECEN} -gt ${nth_max} ]] && export NTHREADS_OCNANALECEN=${nth_max}
+    export APRUN_OCNANALECEN="${launcher} -n ${npe_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
+
 elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export MKL_NUM_THREADS=4

diff --git a/gempak/fix/gfs_meta b/gempak/fix/gfs_meta
@@ -1,23 +1,23 @@
-$USHgempak/gfs_meta_us.sh 36 84 126 216
-$USHgempak/gfs_meta_bwx.sh 36 84 126 180
-$USHgempak/gfs_meta_comp.sh 36 84 126 
-$USHgempak/gfs_meta_ak.sh 36 84 132 216
-$USHgempak/gfs_meta_crb.sh 126 
-$USHgempak/gfs_meta_hur.sh 36 84 126 
-$USHgempak/gfs_meta_qpf.sh 36 84 132 216 
-$USHgempak/gfs_meta_precip.sh 36 84 132 216 384
-$USHgempak/gfs_meta_sa.sh 126  
-$USHgempak/gfs_meta_ver.sh 126  
-$USHgempak/gfs_meta_hi.sh 384
-$USHgempak/gfs_meta_nhsh.sh 384
-$USHgempak/gfs_meta_trop.sh 384
-$USHgempak/gfs_meta_usext.sh 384
-$USHgempak/gfs_meta_mar_ql.sh 24 48 96 180
-$USHgempak/gfs_meta_mar_comp.sh 126
-$USHgempak/gfs_meta_opc_na_ver 126
-$USHgempak/gfs_meta_opc_np_ver 126
-$USHgempak/gfs_meta_mar_atl.sh 180
-$USHgempak/gfs_meta_mar_pac.sh 180
-$USHgempak/gfs_meta_mar_ver.sh 48
-$USHgempak/gfs_meta_mar_skewt.sh 72
-$USHgempak/gfs_meta_sa2.sh 144
+${HOMEgfs}/gempak/ush/gfs_meta_us.sh 36 84 126 216
+${HOMEgfs}/gempak/ush/gfs_meta_bwx.sh 36 84 126 180
+${HOMEgfs}/gempak/ush/gfs_meta_comp.sh 36 84 126
+${HOMEgfs}/gempak/ush/gfs_meta_ak.sh 36 84 132 216
+${HOMEgfs}/gempak/ush/gfs_meta_crb.sh 126
+${HOMEgfs}/gempak/ush/gfs_meta_hur.sh 36 84 126
+${HOMEgfs}/gempak/ush/gfs_meta_qpf.sh 36 84 132 216
+${HOMEgfs}/gempak/ush/gfs_meta_precip.sh 36 84 132 216 384
+${HOMEgfs}/gempak/ush/gfs_meta_sa.sh 126
+${HOMEgfs}/gempak/ush/gfs_meta_ver.sh 126
+${HOMEgfs}/gempak/ush/gfs_meta_hi.sh 384
+${HOMEgfs}/gempak/ush/gfs_meta_nhsh.sh 384
+${HOMEgfs}/gempak/ush/gfs_meta_trop.sh 384
+${HOMEgfs}/gempak/ush/gfs_meta_usext.sh 384
+${HOMEgfs}/gempak/ush/gfs_meta_mar_ql.sh 24 48 96 180
+${HOMEgfs}/gempak/ush/gfs_meta_mar_comp.sh 126
+${HOMEgfs}/gempak/ush/gfs_meta_opc_na_ver 126
+${HOMEgfs}/gempak/ush/gfs_meta_opc_np_ver 126
+${HOMEgfs}/gempak/ush/gfs_meta_mar_atl.sh 180
+${HOMEgfs}/gempak/ush/gfs_meta_mar_pac.sh 180
+${HOMEgfs}/gempak/ush/gfs_meta_mar_ver.sh 48
+${HOMEgfs}/gempak/ush/gfs_meta_mar_skewt.sh 72
+${HOMEgfs}/gempak/ush/gfs_meta_sa2.sh 144
diff --git a/gempak/ush/gdas_meta_na.sh b/gempak/ush/gdas_meta_na.sh
@@ -51,7 +51,7 @@ PROJ    = str/90;-105;0
 LATLON  = 1
 
 
-restore $USHgempak/restore/pmsl_thkn.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/pmsl_thkn.2.nts
 CLRBAR  = 1
 HLSYM   = 2;1.5//21//hw
 TEXT    = 1/21//hw
@@ -60,7 +60,7 @@ l
 ru
 
 
-restore $USHgempak/restore/850mb_hght_tmpc.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/850mb_hght_tmpc.2.nts
 CLRBAR  = 1
 TEXT    = 1/21//hw
 SKIP    = 0         !0         !0         !0         !/3
@@ -70,23 +70,23 @@ l
 ru
 
 
-restore $USHgempak/restore/700mb_hght_relh_omeg.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/700mb_hght_relh_omeg.2.nts
 CLRBAR  = 1
 TEXT    = 1/21//hw
 TITLE	= 5/-2/~ ? $m_title @ HGT, REL HUMIDITY AND OMEGA|~@ HGT, RH AND OMEGA!0
 l
 ru
 
 
-restore $USHgempak/restore/500mb_hght_absv.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/500mb_hght_absv.2.nts
 CLRBAR  = 1
 TEXT    = 1/21//hw
 TITLE	= 5/-2/~ ? $m_title @ HGT AND VORTICITY|~@ HGT AND VORTICITY!0
 l
 ru
 
 
-restore $USHgempak/restore/250mb_hght_wnd.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/250mb_hght_wnd.2.nts
 CLRBAR  = 1
 TEXT    = 1/21//hw
 TITLE	= 5/-2/~ ? $m_title @ HGT, ISOTACHS AND WIND (KTS)|~@ HGT AND WIND!0

diff --git a/gempak/ush/gfs_meta_hi.sh b/gempak/ush/gfs_meta_hi.sh
@@ -45,9 +45,9 @@ MAP	= 1
 CLEAR	= yes
 CLRBAR  = 1
 
-restore ${USHgempak}/restore/garea_hi.nts
+restore ${HOMEgfs}/gempak/ush/restore/garea_hi.nts
 
-restore ${USHgempak}/restore/pmsl_thkn.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/pmsl_thkn.2.nts
 CLRBAR  = 1
 HLSYM   = 2;1.5//21//hw
 TEXT    = 1/21//hw
@@ -56,47 +56,47 @@ l
 ru
 
 
-restore ${USHgempak}/restore/850mb_hght_tmpc.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/850mb_hght_tmpc.2.nts
 CLRBAR  = 1
 TEXT    = 1/21//hw
 TITLE	= 5/-2/~ ? $m_title @ HGHTS, TEMPERATURE AND WIND (KTS)|~@ HGHT, TMP, WIND!0!0!0
 l
 ru
 
 
-restore ${USHgempak}/restore/700mb_hght_relh_omeg.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/700mb_hght_relh_omeg.2.nts
 CLRBAR  = 1
 TEXT    = 1/21//hw
 TITLE	= 5/-2/~ ? $m_title @ HGHTS, REL HUMIDITY AND OMEGA|~@ HGHT, RH AND OMEGA!0
 l
 ru
 
 
-restore ${USHgempak}/restore/500mb_hght_absv.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/500mb_hght_absv.2.nts
 CLRBAR  = 1
 TEXT    = 1/21//hw
 TITLE	= 5/-2/~ ? $m_title @ HEIGHTS AND VORTICITY|~@ HGHT AND VORTICITY!0
 l
 ru
 
 
-restore ${USHgempak}/restore/200mb_hght_wnd.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/200mb_hght_wnd.2.nts
 CLRBAR  = 1
 TEXT    = 1/21//hw
 TITLE	= 5/-2/~ ? $m_title @ HEIGHTS, ISOTACHS AND WIND (KTS)|~@ HGHT AND WIND!0
 l
 ru
 
 
-restore ${USHgempak}/restore/250mb_hght_wnd.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/250mb_hght_wnd.2.nts
 CLRBAR  = 1
 TEXT    = 1/21//hw
 TITLE	= 5/-2/~ ? $m_title @ HEIGHTS, ISOTACHS AND WIND (KTS)|~@ HGHT AND WIND!0
 l
 ru
 
 
-restore ${USHgempak}/restore/300mb_hght_wnd.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/300mb_hght_wnd.2.nts
 CLRBAR  = 1
 TEXT    = 1/21//hw
 TITLE	= 5/-2/~ ? $m_title @ HEIGHTS, ISOTACHS AND WIND (KTS)|~@ HGHT AND WIND!0
@@ -149,7 +149,7 @@ CLRBAR  = 1
 r
 
 
-restore ${USHgempak}/restore/precip.2.nts
+restore ${HOMEgfs}/gempak/ush/restore/precip.2.nts
 CLRBAR  = 1
 TEXT    = 1/21//hw
 GDATTIM = F12-F192-06; F214-F384-12