From 60513ec62747cad85e5231e64259cca6108018e6 Mon Sep 17 00:00:00 2001 From: yuanxue2870 Date: Wed, 4 Dec 2024 16:01:38 +0000 Subject: [PATCH 1/7] add Jiarui's yaml and add difference check filter --- jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml | 90 +++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml diff --git a/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml b/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml new file mode 100644 index 0000000..3c45819 --- /dev/null +++ b/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml @@ -0,0 +1,90 @@ + - obs space: + name: SnowDepthMADIS + simulated variables: [totalSnowDepth] + obsdatain: + engine: + type: H5File + obsfile: MADIS_XXYYYYXXMMXXDDXXHH.nc + obsdataout: + engine: + type: H5File + obsfile: output/DA/hofx/var_hofx_madis_XXYYYYXXMMXXDDXXHH.nc + obs operator: + name: Identity + obs pre filters: + - filter: Perform Action + filter variables: + - name: totalSnowDepth + action: + name: assign error + error parameter: 40.0 + - filter: Variable Assignment + assignments: + - name: GrossErrorProbability/totalSnowDepth + type: float + value: 0.02 + - name: BkgError/totalSnowDepth_background_error + type: float + value: 30.0 + obs prior filters: + - filter: Bounds Check # negative / missing snow + filter variables: + - name: totalSnowDepth + minvalue: 0.0 + maxvalue: 2000.0 + - filter: Domain Check # missing station elevation (-999.9) + where: + - variable: + name: MetaData/stationElevation + minvalue: -999.0 + - filter: Domain Check # land only + where: + - variable: + name: GeoVaLs/slmsk + minvalue: 0.5 + maxvalue: 1.5 + - filter: RejectList # no land-ice + where: + - variable: + name: GeoVaLs/vtype + minvalue: 14.5 + maxvalue: 15.5 + - filter: Difference Check # elevation check + reference: MetaData/stationElevation + value: GeoVaLs/filtered_orography + threshold: 200. + obs post filters: + - filter: Background Check # gross error check + filter variables: + - name: totalSnowDepth + threshold: 6.25 + action: + name: reject + - filter: Temporal Thinning + min_spacing: PT6H + seed_time: 'XXYYYY-XXMM-XXDDTXXHH:00:00Z' + category_variable: + name: MetaData/stationIdentification + - filter: Met Office Buddy Check + filter variables: + - name: totalSnowDepth + rejection_threshold: 0.5 + traced_boxes: # trace all observations + - min_latitude: -90 + max_latitude: 90 + min_longitude: -180 + max_longitude: 180 + search_radius: 150 # km + station_id_variable: + name: MetaData/stationIdentification + num_zonal_bands: 24 + sort_by_pressure: false + max_total_num_buddies: 15 + max_num_buddies_from_single_band: 10 + max_num_buddies_with_same_station_id: 5 + use_legacy_buddy_collector: false + horizontal_correlation_scale: { "-90": 150, "90": 150 } + temporal_correlation_scale: PT6H + damping_factor_1: 1.0 + damping_factor_2: 1.0 + background_error_group: BkgError From 29fa5865756dc7a20815eb51d0be6e3625f8f2e9 Mon Sep 17 00:00:00 2001 From: yuanxue2870 Date: Wed, 4 Dec 2024 16:17:28 +0000 Subject: [PATCH 2/7] add MADIS obs loc --- do_landDA.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/do_landDA.sh b/do_landDA.sh index c2c3c95..ada954f 100755 --- a/do_landDA.sh +++ b/do_landDA.sh @@ -157,6 +157,8 @@ do # get the obs file name if [ ${OBS_TYPES[$ii]} == "GTS" ]; then obsfile=$OBSDIR/snow_depth/GTS/data_proc/${YYYY}${MM}/sfcsno_snow_${YYYY}${MM}${DD}${HH}.nc4 + elif [ ${OBS_TYPES[$ii]} == "MADIS" ]; then + obsfile=$OBSDIR/snow_depth/MADIS/data_proc/v3/${YYYY}/madis_snow_${YYYY}${MM}${DD}_${HH}00.nc4 elif [ ${OBS_TYPES[$ii]} == "GHCN" ]; then # GHCN are time-stamped at 18. If assimilating at 00, need to use previous day's obs, so that # obs are within DA window. From 3f8cf142b326a4aeba94ffdea1a163d178137e18 Mon Sep 17 00:00:00 2001 From: yuanxue2870 Date: Wed, 4 Dec 2024 17:49:28 +0000 Subject: [PATCH 3/7] continue edits of MADIS data loc --- do_landDA.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/do_landDA.sh b/do_landDA.sh index ada954f..f3e9c2c 100755 --- a/do_landDA.sh +++ b/do_landDA.sh @@ -158,7 +158,7 @@ do if [ ${OBS_TYPES[$ii]} == "GTS" ]; then obsfile=$OBSDIR/snow_depth/GTS/data_proc/${YYYY}${MM}/sfcsno_snow_${YYYY}${MM}${DD}${HH}.nc4 elif [ ${OBS_TYPES[$ii]} == "MADIS" ]; then - obsfile=$OBSDIR/snow_depth/MADIS/data_proc/v3/${YYYY}/madis_snow_${YYYY}${MM}${DD}_${HH}00.nc4 + obsfile=$OBSDIR/snow_depth/MADIS/data_proc/v3/${YYYY}/madis_snow_${YYYY}${MM}${DD}_${HH}00.nc elif [ ${OBS_TYPES[$ii]} == "GHCN" ]; then # GHCN are time-stamped at 18. If assimilating at 00, need to use previous day's obs, so that # obs are within DA window. From 01478d488dc9b1571c2ba361560c483339d73aba Mon Sep 17 00:00:00 2001 From: yuanxue2870 <136842224+yuanxue2870@users.noreply.github.com> Date: Fri, 13 Dec 2024 15:06:52 -0500 Subject: [PATCH 4/7] Update jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml Co-authored-by: Jiarui Dong --- jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml b/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml index 3c45819..18a86a3 100644 --- a/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml +++ b/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml @@ -31,7 +31,7 @@ filter variables: - name: totalSnowDepth minvalue: 0.0 - maxvalue: 2000.0 + maxvalue: 20000.0 - filter: Domain Check # missing station elevation (-999.9) where: - variable: From 627ce79de5205d1102d5f49189dd0990341213a1 Mon Sep 17 00:00:00 2001 From: yuanxue2870 Date: Tue, 17 Dec 2024 19:15:31 +0000 Subject: [PATCH 5/7] change 'GTS' to 'SFCSNO' --- do_landDA.sh | 2 +- jedi/fv3-jedi/yaml_files/2DVar/{GTS.yaml => SFCSNO.yaml} | 6 +++--- settings_DA_template | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) rename jedi/fv3-jedi/yaml_files/2DVar/{GTS.yaml => SFCSNO.yaml} (95%) diff --git a/do_landDA.sh b/do_landDA.sh index f3e9c2c..e1489c8 100755 --- a/do_landDA.sh +++ b/do_landDA.sh @@ -155,7 +155,7 @@ for ii in "${!OBS_TYPES[@]}"; # loop through requested obs do # get the obs file name - if [ ${OBS_TYPES[$ii]} == "GTS" ]; then + if [ ${OBS_TYPES[$ii]} == "SFCSNO" ]; then obsfile=$OBSDIR/snow_depth/GTS/data_proc/${YYYY}${MM}/sfcsno_snow_${YYYY}${MM}${DD}${HH}.nc4 elif [ ${OBS_TYPES[$ii]} == "MADIS" ]; then obsfile=$OBSDIR/snow_depth/MADIS/data_proc/v3/${YYYY}/madis_snow_${YYYY}${MM}${DD}_${HH}00.nc diff --git a/jedi/fv3-jedi/yaml_files/2DVar/GTS.yaml b/jedi/fv3-jedi/yaml_files/2DVar/SFCSNO.yaml similarity index 95% rename from jedi/fv3-jedi/yaml_files/2DVar/GTS.yaml rename to jedi/fv3-jedi/yaml_files/2DVar/SFCSNO.yaml index 38a568e..5e70e38 100644 --- a/jedi/fv3-jedi/yaml_files/2DVar/GTS.yaml +++ b/jedi/fv3-jedi/yaml_files/2DVar/SFCSNO.yaml @@ -1,14 +1,14 @@ - obs space: - name: SnowDepthGTS + name: SnowDepthSFCSNO simulated variables: [totalSnowDepth] obsdatain: engine: type: H5File - obsfile: GTS_XXYYYYXXMMXXDDXXHH.nc + obsfile: SFCSNO_XXYYYYXXMMXXDDXXHH.nc obsdataout: engine: type: H5File - obsfile: output/DA/hofx/var_hofx_gts_XXYYYYXXMMXXDDXXHH.nc + obsfile: output/DA/hofx/var_hofx_sfcsno_XXYYYYXXMMXXDDXXHH.nc obs operator: name: Identity obs pre filters: diff --git a/settings_DA_template b/settings_DA_template index e953b9e..3657d1b 100644 --- a/settings_DA_template +++ b/settings_DA_template @@ -21,7 +21,7 @@ LANDDADIR=${CYCLEDIR}/DA_update/ # if calling from submit_cycle.sh # options: "letkfoi_snow" , "letkf_snow" DAtype= -# JEDI input obs. options : IMS, GHCN, GTS, SYNTH +# JEDI input obs. options : IMS, GHCN, SFCSNO, SYNTH OBS_TYPES=() # format: ("OBS1" "OBS2") # JEDI call type for each obs_type above. options: DA, HOFX JEDI_TYPES=() # format ("DA" "HOFX") From 30b08a4d3d9e02ca1efd697dbbe98803f9a449e0 Mon Sep 17 00:00:00 2001 From: yuanxue2870 Date: Thu, 19 Dec 2024 19:12:26 +0000 Subject: [PATCH 6/7] add all three elevation related filters for MADIS and SCFSNO --- jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml | 10 ++++++++-- jedi/fv3-jedi/yaml_files/2DVar/SFCSNO.yaml | 5 +++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml b/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml index 18a86a3..0cbbe19 100644 --- a/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml +++ b/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml @@ -32,11 +32,17 @@ - name: totalSnowDepth minvalue: 0.0 maxvalue: 20000.0 - - filter: Domain Check # missing station elevation (-999.9) + - filter: Domain Check # missing station elevation where: - variable: name: MetaData/stationElevation - minvalue: -999.0 + value: is_valid + - filter: Domain Check # invalid station elevation + where: + - variable: + name: MetaData/stationElevation + minvalue: -200.0 + maxvalue: 9900.0 - filter: Domain Check # land only where: - variable: diff --git a/jedi/fv3-jedi/yaml_files/2DVar/SFCSNO.yaml b/jedi/fv3-jedi/yaml_files/2DVar/SFCSNO.yaml index 5e70e38..4680b4a 100644 --- a/jedi/fv3-jedi/yaml_files/2DVar/SFCSNO.yaml +++ b/jedi/fv3-jedi/yaml_files/2DVar/SFCSNO.yaml @@ -33,6 +33,11 @@ minvalue: 0.0 maxvalue: 20000.0 - filter: Domain Check # missing station elevation + where: + - variable: + name: MetaData/stationElevation + value: is_valid + - filter: Domain Check # invalid station elevation where: - variable: name: MetaData/stationElevation From c428a2dba53d27fae885ae19261ada92a71d776f Mon Sep 17 00:00:00 2001 From: yuanxue2870 Date: Fri, 20 Dec 2024 16:37:39 +0000 Subject: [PATCH 7/7] continue edits of MADIS yaml --- jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml b/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml index 0cbbe19..53381a4 100644 --- a/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml +++ b/jedi/fv3-jedi/yaml_files/2DVar/MADIS.yaml @@ -32,7 +32,7 @@ - name: totalSnowDepth minvalue: 0.0 maxvalue: 20000.0 - - filter: Domain Check # missing station elevation + - filter: Domain Check # missing station elevation where: - variable: name: MetaData/stationElevation