Cycling on native grid and create JEDI-based analysis calc job

docs/source/jobs.rst

@@ -33,7 +33,7 @@ Jobs in the GFS Configuration
 | anal              | Runs the analysis. 1) Runs the atmospheric analysis (global_gsi) to produce analysis increments; 2) Update surface    |
 |                   | guess file via global_cycle to create surface analysis on tiles.                                                      |
 +-------------------+-----------------------------------------------------------------------------------------------------------------------+
-| analcalc          | Adds the analysis increments to previous cycle’s forecasts to produce atmospheric analysis files. Produces surface    |
+| analcalc_gsi      | Adds the analysis increments to previous cycle’s forecasts to produce atmospheric analysis files. Produces surface    |
 |                   | analysis file on Gaussian grid.                                                                                       |
 +-------------------+-----------------------------------------------------------------------------------------------------------------------+
 | analdiag          | Creates netCDF diagnostic files containing observation values, innovation (O-F), error, quality control, as well as   |

env/GAEA.env

@@ -41,7 +41,7 @@ if [[ "${step}" = "prep" ]]; then
     export sys_tp="GAEA"
     export launcher_PREP="srun"
 
-elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc_gsi" ]]; then
 
     export MKL_NUM_THREADS=4
     export MKL_CBWR=AUTO

env/HERA.env

@@ -158,7 +158,12 @@ elif [[ "${step}" = "marineanlletkf" ]]; then
     export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
     export APRUN_MARINEANLLETKF=${APRUN_default}
 
-elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+elif [[ "${step}" = "analcalc_fv3jedi" ]]; then
+
+    export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax}
+    export APRUN_ANALCALC_FV3JEDI="${APRUN_default} --cpus-per-task=${NTHREADS_ANALCALC_FV3JEDI}"
+
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc_gsi" ]]; then
 
     export MKL_NUM_THREADS=4
     export MKL_CBWR=AUTO

env/HERCULES.env

@@ -154,8 +154,13 @@ case ${step} in
 
     export NTHREADS_OCNANAL=${NTHREADSmax}
     export APRUN_MARINEANLCHKPT="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANAL}"
- ;;
- "anal" | "analcalc")
+;;
+ "analcalc_fv3jedi")
+
+    export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax}
+    export APRUN_ANALCALC_FV3JEDI="${APRUN_default} --cpus-per-task=${NTHREADS_ANALCALC_FV3JEDI}"
+;; 
+ "anal" | "analcalc_gsi")
 
     export MKL_NUM_THREADS=4
     export MKL_CBWR=AUTO

env/JET.env

@@ -126,7 +126,12 @@ elif [[ "${step}" = "marineanlvar" ]]; then
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
     export APRUN_MARINEANLVAR="${APRUN_default}"
 
-elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+elif [[ "${step}" = "analcalc_fv3jedi" ]]; then
+
+    export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax}
+    export APRUN_ANALCALC_FV3JEDI="${APRUN_default}"
+
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc_gsi" ]]; then
 
     export MKL_NUM_THREADS=4
     export MKL_CBWR=AUTO

env/ORION.env

@@ -154,7 +154,12 @@ elif [[ "${step}" = "marineanlletkf" ]]; then
     export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
     export APRUN_MARINEANLLETKF="${APRUN_default}"
 
-elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+elif [[ "${step}" = "analcalc_fv3jedi" ]]; then
+
+    export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax}
+    export APRUN_ANALCALC_FV3JEDI="${APRUN_default} --cpus-per-task=${NTHREADS_ANALCALC_FV3JEDI}"
+
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc_gsi" ]]; then
 
     export MKL_NUM_THREADS=4
     export MKL_CBWR=AUTO

env/S4.env

@@ -122,7 +122,12 @@ elif [[ "${step}" = "marinebmat" ]]; then
 elif [[ "${step}" = "marinerun" ]]; then
      echo "WARNING: ${step} is not enabled on S4!"
 
-elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+elif [[ "${step}" = "analcalc_fv3jedi" ]]; then
+
+    export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax}
+    export APRUN_ANALCALC_FV3JEDI="${APRUN_default}"
+
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc_gsi" ]]; then
 
     export MKL_NUM_THREADS=4
     export MKL_CBWR=AUTO

env/WCOSS2.env

@@ -136,13 +136,18 @@ elif [[ "${step}" = "atmanlfv3inc" ]]; then
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
     export APRUN_ATMANLFV3INC="${APRUN_default}"
 
-elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+elif [[ "${step}" = "analcalc_fv3jedi" ]]; then
+
+    export NTHREADS_ANALCALC_FV3JEDI=${NTHREADSmax}
+    export APRUN_ANALCALC_FV3JEDI="${APRUN_default}"
+
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc_gsi" ]]; then
 
     export OMP_PLACES=cores
     export OMP_STACKSIZE=1G
     export FI_OFI_RXM_SAR_LIMIT=3145728
 
-    if [[ "${step}" = "analcalc" ]]; then
+    if [[ "${step}" = "analcalc_gsi" ]]; then
       export MPICH_MPIIO_HINTS="*:romio_cb_write=disable"
     fi
 

jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_FV3JEDI

@@ -0,0 +1,55 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "analcalc_fv3jedi" -c "base analcalc_fv3jedi"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+gPDY=${GDATE:0:8}
+gcyc=${GDATE:8:2}
+GDUMP="gdas"
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COM_ATMOS_ANALYSIS
+RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
+    COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL
+
+##############################################
+# Run relevant script
+##############################################
+
+EXSCRIPT=${GDASATMRUNSH:-${SCRgfs}/exglobal_atmos_analysis_calc_fv3jedi.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##############################################
+# Remove the Temporary working directory
+##############################################
+
+cd "${DATAROOT}" || ( echo "FATAL ERROR: ${DATAROOT} does not exist, ABORT!"; exit 1 )
+if [[ ${KEEPDATA} = "NO" ]]; then
+    rm -rf "${DATA}"
+fi
+
+exit 0

jobs/JGLOBAL_ATMOS_ANALYSIS_CALC → jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_GSI

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "analcalc" -c "base anal analcalc"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "analcalc_gsi" -c "base anal analcalc_gsi"
 
 
 ##############################################
@@ -56,7 +56,7 @@ export DOGAUSFCANL=${DOGAUSFCANL:-"YES"}
 ###############################################################
 # Run relevant script
 
-${ANALCALCSH:-${SCRgfs}/exglobal_atmos_analysis_calc.sh}
+${ANALCALCSH:-${SCRgfs}/exglobal_atmos_analysis_calc_gsi.sh}
 status=$?
 [[ ${status} -ne 0 ]] && exit ${status}
 

jobs/rocoto/analcalc_fv3jedi.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="analcalc_fv3jedi"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}"/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_FV3JEDI
+status=$?
+exit "${status}"

jobs/rocoto/analcalc.sh → jobs/rocoto/analcalc_gsi.sh

@@ -8,12 +8,12 @@ source "${HOMEgfs}/ush/preamble.sh"
 status=$?
 [[ ${status} -ne 0 ]] && exit ${status}
 
-export job="analcalc"
+export job="analcalc_gsi"
 export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-${HOMEgfs}/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC
+"${HOMEgfs}"/jobs/JGLOBAL_ATMOS_ANALYSIS_CALC_GSI
 status=$?
 
 

parm/config/gfs/config.analcalc_fv3jedi

@@ -0,0 +1,26 @@
+#! /usr/bin/env bash
+
+########## config.analcalc_fv3jedi ##########
+# Diagnostic amospheric analysis calculation specific
+
+echo "BEGIN: config.analcalc_fv3jedi"
+
+export layout_x_analcalc_fv3jedi=2
+export layout_y_analcalc_fv3jedi=2
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" analcalc_fv3jedi
+
+export JCB_BASE_YAML=${PARMgfs}/gdas/atm/jcb-base.yaml.j2
+export JCB_ALGO=fv3jedi_analcalc
+export JEDIEXE=${EXECgfs}/fv3jedi_analcalc.x
+
+if [[ ${DOHYBVAR} = "YES" ]]; then
+    export CASE_ANL=${CASE_ENS}
+else
+    export CASE_ANL=${CASE}
+fi
+
+export JEDI_FIX_YAML="${PARMgfs}/gdas/atm_jedi_fix.yaml.j2"
+
+echo "END: config.analcalc_fv3jedi"

parm/config/gfs/config.analcalc_gsi

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+########## config.analcalc_gsi ##########
+# GFS post-anal specific (non-diag)
+
+echo "BEGIN: config.analcalc_gsi"
+
+# Get task specific resources
+. "${EXPDIR}"/config.resources analcalc_gsi
+
+echo "END: config.analcalc_gsi"

parm/config/gfs/config.resources

@@ -17,7 +17,7 @@ if (( $# != 1 )); then
     echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal"
     echo "snowanl esnowrecen"
     echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb"
-    echo "anal sfcanl analcalc analdiag fcst echgres"
+    echo "anal sfcanl analcalc_fv3jedi analcalc_gsi analdiag fcst echgres"
     echo "upp atmos_products"
     echo "tracker genesis genesis_fsu"
     echo "verfozn verfrad vminmon fit2obs metp arch cleanup"
@@ -701,7 +701,18 @@ case ${step} in
     export is_exclusive=True
     ;;
 
-  "analcalc")
+  "analcalc_fv3jedi")
+    export layout_x=${layout_x_analcalc_fv3jedi}
+    export layout_y=${layout_y_analcalc_fv3jedi}
+
+    walltime="00:15:00"
+    ntasks=$(( layout_x * layout_y * 6 ))
+    threads_per_task=1
+    tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
+    export is_exclusive=True
+    ;;
+
+  "analcalc_gsi")
     walltime="00:15:00"
     ntasks=127
     export ntasks_calcanl="${ntasks}"

scripts/exglobal_atmos_analysis_calc_fv3jedi.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+# exglobal_atm_calc_analysis.py
+# This script creates an AnalysisCalc object
+# and runs the execute method which executes
+# the diagnostic global analysis calculation
+import os
+
+from wxflow import Logger, cast_strdict_as_dtypedict
+from pygfs.task.analcalc import AnalysisCalc
+
+# Initialize root logger
+logger = Logger(level='DEBUG', colored_log=True)
+
+
+if __name__ == '__main__':
+
+    # Take configuration from environment and cast it as python dictionary
+    config = cast_strdict_as_dtypedict(os.environ)
+
+    # Instantiate the AnalysisCalc task
+    AnalCalc = AnalysisCalc(config, 'fv3jedi_analcalc')
+
+    # Initialize
+    AnalCalc.initialize_jedi()
+    AnalCalc.initialize()
+
+    # Execute JEDI application
+    AnalCalc.execute(config.APRUN_ANALCALC_FV3JEDI)
+
+    # Finalize
+    AnalCalc.finalize()

sorc/link_workflow.sh

@@ -349,6 +349,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then
                        "fv3jedi_plot_field.x" \
                        "gdasapp_chem_diagb.x" \
                        "fv3jedi_fv3inc.x" \
+                       "fv3jedi_analcalc.x" \
                        "gdas_ens_handler.x" \
                        "gdas_incr_handler.x" \
                        "gdas_obsprovider2ioda.x" \

ush/forecast_postdet.sh

@@ -95,6 +95,7 @@ FV3_postdet() {
         read_increment=".true."
         res_latlon_dynamics="atminc.nc"
       fi
+      increment_file_on_native_grid=".false."
       local increment_file
       for inc_file in "${inc_files[@]}"; do
         increment_file="${COMIN_ATMOS_INPUT}/${RUN}.t${cyc}z.${inc_file}"
@@ -158,9 +159,16 @@ EOF
           delimiter=","
         done
       else  # "${DOIAU}" == "NO"
-        inc_files=("atminc.nc")
-        read_increment=".true."
-        res_latlon_dynamics="atminc.nc"
+        read_increment=".true."  
+	if [[ "${DO_JEDIATMVAR:-NO}" == "YES" ]] && [[ "${PREFIX_ATMINC}" != "r" ]] && [[ "${PDY}${cyc}" != "${SDATE}" ]]; then
+	  inc_files=("atminc.tile1.nc" "atminc.tile2.nc" "atminc.tile3.nc" "atminc.tile4.nc" "atminc.tile5.nc" "atminc.tile6.nc")
+          res_latlon_dynamics="atminc"
+	  increment_file_on_native_grid=".true."
+	else
+	  inc_files=("atminc.nc")
+          res_latlon_dynamics="atminc.nc"
+          increment_file_on_native_grid=".false."
+	fi    
         if [[ "${REPLAY_ICS:-NO}" == "YES" ]]; then
           IAU_FHROT=${half_window}  # Replay ICs start at the end of the assimilation window
           # Control member has no perturbation
@@ -169,12 +177,17 @@ EOF
             read_increment=".false."
             res_latlon_dynamics='""'
           fi
+	  increment_file_on_native_grid=".false."
         fi
       fi
 
       local increment_file
       for inc_file in "${inc_files[@]}"; do
-        increment_file="${COMIN_ATMOS_ANALYSIS}/${RUN}.t${cyc}z.${PREFIX_ATMINC}${inc_file}"
+          if [[ "${DO_JEDIATMVAR:-NO}" == "YES" ]] && [[ "${PREFIX_ATMINC}" != "r" ]] && [[ "${PDY}${cyc}" != "${SDATE}" ]]; then
+            increment_file="${COMIN_ATMOS_ANALYSIS}/${RUN}.t${cyc}z.cubed_sphere_grid_${PREFIX_ATMINC}${inc_file}"
+          else
+            increment_file="${COMIN_ATMOS_ANALYSIS}/${RUN}.t${cyc}z.${PREFIX_ATMINC}${inc_file}"
+          fi
         if [[ -f "${increment_file}" ]]; then
           ${NCP} "${increment_file}" "${DATA}/INPUT/${inc_file}"
         else

ush/forecast_predet.sh

@@ -362,6 +362,7 @@ FV3_predet(){
   warm_start=".false."
   read_increment=".false."
   res_latlon_dynamics='""'
+  increment_file_on_native_grid=".true."
 
   # Stochastic Physics Options
   do_skeb=".false."