Add marine hybrid envar

ci/cases/gfsv17/C384mx025_3DVarAOWCDA.yaml

@@ -9,10 +9,11 @@ arguments:
   resdetocean: 0.25
   nens: 0
   interval: 6
-  start: cold
+  start: warm
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  idate: 2021063000
+  idate: 2021063018
   edate: 2021070306
-  icsdir: /scratch1/NCEPDEV/climate/Jessica.Meixner/cycling/IC_2021063000_V2
-  yaml: {{ HOMEgfs }}/ci/cases/gfsv17/ocnanal.yaml
+  #icsdir: /scratch1/NCEPDEV/climate/Jessica.Meixner/cycling/IC_2021063000_V2
+  icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
+  yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marine3dvar.yaml

ci/cases/gfsv17/C384mx025_hybAOWCDA.yaml

@@ -0,0 +1,19 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: S2S
+  resdetatmos: 384
+  resensatmos: 192
+  resdetocean: 0.25
+  nens: 30
+  interval: 0
+  start: warm
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  idate: 2021063018
+  edate: 2021070306
+  icsdir: /work/noaa/da/gvernier/ensda/ictest/1440x1080x75/
+  yaml: {{ HOMEgfs }}/ci/cases/gfsv17/marinehyb.yaml

ci/cases/gfsv17/ocnanal.yaml → ci/cases/gfsv17/marine3dvar.yaml

@@ -20,8 +20,3 @@ marineanl:
   SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca
   SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
   SOCA_NINNER: 100
-
-prepoceanobs:
-  SOCA_OBS_LIST: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obs/obs_list.yaml
-  OBSPREP_YAML: {{ HOMEgfs }}/sorc/gdas.cd/parm/soca/obsprep/obsprep_config.yaml
-  DMPDIR: /scratch1/NCEPDEV/da/common/

ci/cases/gfsv17/marinehyb.yaml

@@ -0,0 +1,21 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+
+base:
+  DOIAU: "YES"
+  DO_JEDIATMVAR: "NO"
+  DO_JEDIATMENS: "NO"
+  DO_JEDIOCNVAR: "YES"
+  DO_JEDISNOWDA: "NO"
+  DO_MERGENSST: "NO"
+  DOHYBVAR: "YES"
+  DO_FIT2OBS: "YES"
+  DO_VERFOZN: "YES"
+  DO_VERFRAD: "YES"
+  DO_VRFY_OCEANDA: "NO"
+  FHMAX_GFS: 240
+  ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
+
+marineanl:
+  SOCA_INPUT_FIX_DIR: {{ HOMEgfs }}/fix/gdas/soca/1440x1080x75/soca
+  SOCA_NINNER: 20  # revert to ~100 after the memory leak is fixed

ci/cases/pr/C48mx500_3DVarAOWCDA.yaml

@@ -21,4 +21,4 @@ skip_ci_on_hosts:
   - wcoss2
   - gaea
   - orion
-  - hercules
+

ci/cases/pr/C48mx500_hybAOWCDA.yaml

@@ -6,21 +6,22 @@ arguments:
   pslot: {{ 'pslot' | getenv }}
   app: S2S
   resdetatmos: 48
-  resdetocean: 5.0
   resensatmos: 48
+  resdetocean: 5.0
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610
-  idate: 2021032412
-  edate: 2021032418
+  # TODO: Stage IC's on HPC
+  #icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610
+  icsdir: /work/noaa/da/gvernier/ensda/ictest
+  #icsdir: /scratch2/NCEPDEV/ocean/Guillaume.Vernieres/data/prepics/ictest/
+  idate: 2021032418
+  edate: 2021032500
   nens: 3
   interval: 0
   start: warm
   yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml
 
 skip_ci_on_hosts:
   - wcoss2
-  - orion
-  - hercules
-  - hera
   - gaea
+  - orion

env/HERCULES.env

@@ -139,14 +139,7 @@ case ${step} in
     export APRUN_MARINEANLVAR="${APRUN_default}"
  ;;
 "ocnanalecen")
-
-    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
-
-    max_threads_per_task=$((max_tasks_per_node / tasks_per_node_ocnanalecen))
-
-    export NTHREADS_OCNANALECEN=${threads_per_task_ocnanalecen:-${max_threads_per_task}}
-    [[ ${NTHREADS_OCNANALECEN} -gt ${max_threads_per_task} ]] && export NTHREADS_OCNANALECEN=${max_threads_per_task}
-    export APRUN_OCNANALECEN="${launcher} -n ${ntasks_ocnanalecen} --cpus-per-task=${NTHREADS_OCNANALECEN}"
+   export APRUN_OCNANALECEN="${APRUN_default}"
 ;;
  "marineanlchkpt")
 

env/WCOSS2.env

@@ -109,17 +109,16 @@ elif [[ "${step}" = "marinebmat" ]]; then
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
     export APRUN_MARINEBMAT="${APRUN_default}"
 
-elif [[ "${step}" = "ocnanalrun" ]]; then
+elif [[ "${step}" = "marineanlvar" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUN_MARINEANLVAR="${APRUN_default}"
 
-    export APRUN_OCNANAL="${APRUN_default}"
-
-elif [[ "${step}" = "ocnanalchkpt" ]]; then
+elif [[ "${step}" = "marineanlchkpt" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
 
-    export APRUN_OCNANAL="${APRUN_default}"
+    export APRUN_MARINEANLCHKPT="${APRUN_default}"
 
 elif [[ "${step}" = "ocnanalecen" ]]; then
 
@@ -130,7 +129,7 @@ elif [[ "${step}" = "marineanlletkf" ]]; then
 
     export NTHREADS_MARINEANLLETKF=${NTHREADSmax}
     export APRUN_MARINEANLLETKF="${APRUN_default}"
-    
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

jobs/JGDAS_GLOBAL_OCEAN_ANALYSIS_ECEN

@@ -1,6 +1,6 @@
 #!/bin/bash
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalecen" -c "base ocnanal ocnanalecen"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "ocnanalecen" -c "base marineanl ocnanalecen"
 
 ##############################################
 # Set variables used in the script
@@ -12,8 +12,13 @@ export gPDY=${GDATE:0:8}
 export gcyc=${GDATE:8:2}
 
 YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-   COM_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \
-   COM_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL
+   COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \
+   COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL
+
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+   COMOUT_OCEAN_ANALYSIS:COM_OCEAN_ANALYSIS_TMPL \
+   COMIN_ICE_RESTART:COM_ICE_RESTART_TMPL \
+   COMOUT_ICE_ANALYSIS:COM_ICE_ANALYSIS_TMPL
 
 ##############################################
 # Begin JOB SPECIFIC work

jobs/JGLOBAL_MARINE_ANALYSIS_CHECKPOINT

@@ -2,6 +2,7 @@
 source "${HOMEgfs}/ush/preamble.sh"
 export WIPE_DATA="NO"
 export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}"
+export DATAens="${DATAjob}/ensdata"
 export DATA="${DATAjob}/marinevariational"
 source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlchkpt" -c "base marineanl marineanlchkpt"
 

jobs/JGLOBAL_MARINE_ANALYSIS_FINALIZE

@@ -2,6 +2,7 @@
 source "${HOMEgfs}/ush/preamble.sh"
 export WIPE_DATA="NO"
 export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}"
+export DATAens="${DATAjob}/ensdata"
 export DATA="${DATAjob}/marinevariational"
 source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlfinal" -c "base marineanl marineanlfinal"
 

jobs/JGLOBAL_MARINE_ANALYSIS_INITIALIZE

@@ -2,6 +2,7 @@
 
 source "${HOMEgfs}/ush/preamble.sh"
 export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}"
+export DATAens="${DATAjob}/ensdata"
 export DATA="${DATAjob}/marinevariational"
 source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlinit" -c "base marineanl marineanlinit"
 

jobs/JGLOBAL_MARINE_ANALYSIS_VARIATIONAL

@@ -3,6 +3,7 @@
 source "${HOMEgfs}/ush/preamble.sh"
 export WIPE_DATA="NO"
 export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}"
+export DATAens="${DATAjob}/ensdata"
 export DATA="${DATAjob}/marinevariational"
 source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlvar" -c "base marineanl marineanlvar"
 

parm/config/gfs/config.base

@@ -356,6 +356,7 @@ export DO_JEDIATMENS="@DO_JEDIATMENS@"
 export DO_JEDIOCNVAR="@DO_JEDIOCNVAR@"
 export DO_JEDISNOWDA="@DO_JEDISNOWDA@"
 export DO_MERGENSST="@DO_MERGENSST@"
+export DO_STARTMEM_FROM_JEDIICE="@DO_STARTMEM_FROM_JEDIICE@"
 
 # Hybrid related
 export DOHYBVAR="@DOHYBVAR@"

parm/config/gfs/yaml/defaults.yaml

@@ -5,6 +5,7 @@ base:
   DO_JEDIOCNVAR: "NO"
   DO_JEDISNOWDA: "NO"
   DO_MERGENSST: "NO"
+  DO_STARTMEM_FROM_JEDIICE: "NO"
   DO_GOES: "NO"
   DO_BUFRSND: "NO"
   DO_GEMPAK: "NO"
@@ -43,7 +44,7 @@ atmensanlobs:
 
 atmensanlsol:
   JCB_ALGO_YAML: "${PARMgfs}/gdas/atm/jcb-prototype_lgetkf_solver.yaml.j2"
-  
+
 aeroanl:
   IO_LAYOUT_X: 1
   IO_LAYOUT_Y: 1

parm/stage/ice.yaml.j2

@@ -1,5 +1,12 @@
+{% set START_ICE_FROM_ANA = False %}
+{% if DO_JEDIOCNVAR == True and RUN == 'gdas' %}
+    {% set START_ICE_FROM_ANA = True %}
+{% endif %}
+{% if DO_STARTMEM_FROM_JEDIICE == False and RUN == 'enkfgdas' %}
+    {% set START_ICE_FROM_ANA = False %}
+{% endif %}
 ice:
-    {% if DO_JEDIOCNVAR == True %}
+    {% if START_ICE_FROM_ANA == True %}
     mkdir:
         {% for mem in range(first_mem, last_mem + 1) %}
         {% set imem = mem - first_mem %}

scripts/exglobal_stage_ic.py

@@ -21,7 +21,7 @@ def main():
     keys = ['RUN', 'MODE', 'EXP_WARM_START', 'NMEM_ENS',
             'assim_freq', 'current_cycle', 'previous_cycle',
             'ROTDIR', 'ICSDIR', 'STAGE_IC_YAML_TMPL', 'DO_JEDIATMVAR',
-            'OCNRES', 'waveGRD', 'ntiles', 'DOIAU', 'DO_JEDIOCNVAR',
+            'OCNRES', 'waveGRD', 'ntiles', 'DOIAU', 'DO_JEDIOCNVAR', 'DO_STARTMEM_FROM_JEDIICE',
             'REPLAY_ICS', 'DO_WAVE', 'DO_OCN', 'DO_ICE', 'DO_NEST']
 
     stage_dict = AttrDict()

ush/forecast_postdet.sh

@@ -466,13 +466,12 @@ MOM6_postdet() {
     fi
 
     # GEFS perturbations
-    if [[ "${RUN}" == "gefs" ]]; then 
-    #     to ensure it does not interfere with the GFS
-      if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then
-        ${NCP} "${COMIN_OCEAN_ANALYSIS}/${RUN}.t${cyc}z.ocninc.nc" "${DATA}/INPUT/mom6_increment.nc" \
-        || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 )
-      fi
-    fi # if [[ "${RUN}" == "gefs" ]]; then 
+    # TODO if [[ $RUN} == "gefs" ]] block maybe be needed
+    #     to ensure it does not interfere with the GFS when ensemble is updated in the GFS
+    if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then
+      ${NCP} "${COMIN_OCEAN_ANALYSIS}/${RUN}.t${cyc}z.ocninc.nc" "${DATA}/INPUT/mom6_increment.nc" \
+      || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 )
+    fi
   fi  # if [[ "${RERUN}" == "NO" ]]; then
 
   # Link output files
@@ -592,7 +591,12 @@ CICE_postdet() {
   else  # "${RERUN}" == "NO"
     restart_date="${model_start_date_current_cycle}"
     cice_restart_file="${COMIN_ICE_RESTART_PREV}/${restart_date:0:8}.${restart_date:8:2}0000.cice_model.res.nc"
-    if [[ "${DO_JEDIOCNVAR:-NO}" == "YES" ]]; then
+    # Start the deterministic from the JEDI/SOCA analysis if the Marine DA in ON
+    if (( MEMBER == 0 )) && [[ "${DO_JEDIOCNVAR:-NO}" == "YES" ]]; then
+      cice_restart_file="${COMIN_ICE_ANALYSIS}/${restart_date:0:8}.${restart_date:8:2}0000.cice_model_anl.res.nc"
+    fi
+    # Ignore the JEDI/SOCA ensemble analysis for the ensemble members if DO_START_FROM_JEDIICE is OFF
+    if (( MEMBER > 0 )) && [[ "${DO_JEDIOCNVAR:-NO}" == "YES" ]] && [[ "${DO_STARTMEM_FROM_JEDIICE:-NO}" == "YES" ]]; then
       cice_restart_file="${COMIN_ICE_ANALYSIS}/${restart_date:0:8}.${restart_date:8:2}0000.cice_model_anl.res.nc"
     fi
   fi

ush/python/pygfs/task/marine_analysis.py

@@ -51,9 +51,9 @@ def __init__(self, config):
         _window_begin = add_to_datetime(self.task_config.current_cycle, -to_timedelta(f"{self.task_config.assim_freq}H") / 2)
         _window_end = add_to_datetime(self.task_config.current_cycle, to_timedelta(f"{self.task_config.assim_freq}H") / 2)
 
-        # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert
+        # compute the relative path from self.task_config.DATA to self.task_config.DATAens
         if self.task_config.NMEM_ENS > 0:
-            _enspert_relpath = os.path.relpath(self.task_config.DATAenspert, self.task_config.DATA)
+            _enspert_relpath = os.path.relpath(self.task_config.DATAens, self.task_config.DATA)
         else:
             _enspert_relpath = None
 
@@ -69,7 +69,8 @@ def __init__(self, config):
                 'MARINE_WINDOW_MIDDLE_ISO': self.task_config.current_cycle.strftime('%Y-%m-%dT%H:%M:%SZ'),
                 'ENSPERT_RELPATH': _enspert_relpath,
                 'CALC_SCALE_EXEC': _calc_scale_exec,
-                'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z."
+                'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.",
+                'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z."
             }
         )
 
@@ -110,10 +111,13 @@ def initialize(self: Task) -> None:
         os.symlink('../staticb', 'staticb')
 
         # hybrid EnVAR case
-        if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2:
-            # stage ensemble membersfiles for use in hybrid background error
-            logger.debug(f"Stage ensemble members for the hybrid background error")
-            mdau.stage_ens_mem(self.task_config)
+        if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS >= 3:
+            # stage the ensemble weights
+            logger.debug(f"Stage ensemble weights for the hybrid background error")
+            FileHandler({'copy': [[os.path.join(self.task_config.COMIN_OCEAN_BMATRIX, f'{self.task_config.APREFIX}ocean.ens_weights.nc'),
+                                   os.path.join(self.task_config.DATA, 'ocean.ens_weights.nc')],
+                                  [os.path.join(self.task_config.COMIN_ICE_BMATRIX, f'{self.task_config.APREFIX}ice.ens_weights.nc'),
+                                   os.path.join(self.task_config.DATA, 'ice.ens_weights.nc')]]}).sync()
 
         # prepare the yaml configuration to run the SOCA variational application
         self._prep_variational_yaml()
@@ -137,8 +141,8 @@ def _fetch_observations(self: Task) -> None:
 
         obs_files = []
         for ob in obs_list_config['observations']['observers']:
-            logger.info(f"******** {self.task_config.OPREFIX}{ob['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc}.nc4")
-            obs_files.append(f"{self.task_config.OPREFIX}{ob['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc}.nc4")
+            logger.info(f"******** {self.task_config.OPREFIX}{ob['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4")
+            obs_files.append(f"{self.task_config.OPREFIX}{ob['obs space']['name'].lower()}.{to_YMD(self.task_config.PDY)}{self.task_config.cyc:02d}.nc4")
         obs_list = []
 
         # copy obs from COM_OBS to DATA/obs
@@ -202,15 +206,15 @@ def _prep_variational_yaml(self: Task) -> None:
         envconfig_jcb['PARMgfs'] = self.task_config.PARMgfs
         envconfig_jcb['nmem_ens'] = self.task_config.NMEM_ENS
         envconfig_jcb['berror_model'] = 'marine_background_error_static_diffusion'
-        if self.task_config.NMEM_ENS > 3:
+        if self.task_config.NMEM_ENS >= 3:
             envconfig_jcb['berror_model'] = 'marine_background_error_hybrid_diffusion_diffusion'
         envconfig_jcb['DATA'] = self.task_config.DATA
         envconfig_jcb['OPREFIX'] = self.task_config.OPREFIX
         envconfig_jcb['PDY'] = os.getenv('PDY')
         envconfig_jcb['cyc'] = os.getenv('cyc')
         envconfig_jcb['SOCA_NINNER'] = self.task_config.SOCA_NINNER
         envconfig_jcb['obs_list'] = ['adt_rads_all']
-        envconfig_jcb['MOM6_LEVS'] = mdau.get_mom6_levels(str(self.task_config.OCNRES))
+        envconfig_jcb['MOM6_LEVS'] = mdau.get_mom6_levels(str(self.task_config.OCNRES).zfill(3))
 
         # Write obs_list_short
         save_as_yaml(parse_obs_list_file(self.task_config.MARINE_OBS_LIST_YAML), 'obs_list_short.yaml')
@@ -220,12 +224,8 @@ def _prep_variational_yaml(self: Task) -> None:
         jcb_base_yaml = os.path.join(self.task_config.PARMsoca, 'marine-jcb-base.yaml')
         jcb_algo_yaml = os.path.join(self.task_config.PARMsoca, 'marine-jcb-3dfgat.yaml.j2')
 
-        jcb_base_config = YAMLFile(path=jcb_base_yaml)
-        jcb_base_config = Template.substitute_structure(jcb_base_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get)
-        jcb_base_config = Template.substitute_structure(jcb_base_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get)
-        jcb_algo_config = YAMLFile(path=jcb_algo_yaml)
-        jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOUBLE_CURLY_BRACES, envconfig_jcb.get)
-        jcb_algo_config = Template.substitute_structure(jcb_algo_config, TemplateConstants.DOLLAR_PARENTHESES, envconfig_jcb.get)
+        jcb_base_config = parse_j2yaml(path=jcb_base_yaml, data=envconfig_jcb)
+        jcb_algo_config = parse_j2yaml(path=jcb_algo_yaml, data=envconfig_jcb)
 
         # Override base with the application specific config
         jcb_config = {**jcb_base_config, **jcb_algo_config}
@@ -382,12 +382,10 @@ def list_all_files(dir_in, dir_out, wc='*', fh_list=[]):
                                    os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.{domain}ana.nc')])
 
         # Copy of the ssh diagnostics
-        '''
         if nmem_ens > 2:
             for string in ['ssh_steric_stddev', 'ssh_unbal_stddev', 'ssh_total_stddev', 'steric_explained_variance']:
-                post_file_list.append([os.path.join(anl_dir, 'static_ens', f'ocn.{string}.incr.{bdate}.nc'),
+                post_file_list.append([os.path.join(anl_dir, 'staticb', f'ocn.{string}.incr.{bdate}.nc'),
                                        os.path.join(com_ocean_analysis, f'{RUN}.t{cyc}z.ocn.{string}.nc')])
-        '''
 
         # Copy DA grid (computed for the start of the window)
         post_file_list.append([os.path.join(anl_dir, 'soca_gridspec.nc'),