Add namelist and ctest files/yamls for SEVIRI UKV (#214)

* Add CX and VarObs files for SEVIRI UKV * Add test data for SEVIRI UKV * Update unit test python script for SEVIRI UKV, and add newly generated nc4 files for SEVIRIASR and SEVIRIClr * Upate test CMakeList to actually run SEVIRI UKV tests * Remove old `is_not_defined:` syntax in ctests * Mistaken syntax fix * Remove extra test from SEVIRI and update values * Sorting out ctests * Fix lats * fix lons * Fix lats, missed "" before * Fix obs source --------- Co-authored-by: Matt Shin <[email protected]>
MetOffice · Jul 16, 2024 · 3c6673d · 3c6673d
1 parent 0c512bd
commit 3c6673d
Show file tree

Hide file tree

Showing 14 changed files with 252 additions and 0 deletions.
diff --git a/etc/ukv/cx/SEVIRIASR.nl b/etc/ukv/cx/SEVIRIASR.nl
@@ -0,0 +1,3 @@
+&CXControlNL
+CxFields=4,10,12,24,31,33,254,266,267,268,407,409,3209,3210,3236,3245,16222
+/
diff --git a/etc/ukv/cx/SEVIRIClr.nl b/etc/ukv/cx/SEVIRIClr.nl
@@ -0,0 +1,3 @@
+&CXControlNL
+CxFields=4,10,12,24,31,33,254,266,267,268,407,409,3209,3210,3236,3245,16222
+/
diff --git a/etc/ukv/varobs/SEVIRIASR.nl b/etc/ukv/varobs/SEVIRIASR.nl
@@ -0,0 +1,3 @@
+&VarobsControlNL
+Varfields=10,11,18,19,21,28,31,34,54,55,80
+/
diff --git a/etc/ukv/varobs/SEVIRIClr.nl b/etc/ukv/varobs/SEVIRIClr.nl
@@ -0,0 +1,3 @@
+&VarobsControlNL
+Varfields=10,11,18,19,21,28,31,34,54,55,80
+/
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -356,6 +356,14 @@ ADD_WRITER_TEST(NAME     varobswriter_globalnamelist_aircraft
 
 # Tests the UKV namelist files in the etc directory
 
+ADD_WRITER_TEST(NAME     varobswriter_ukvnamelist_seviviasr
+                YAML     varobswriter_ukvnamelist_seviriasr.yaml
+                NAMELIST ../../etc/ukv/varobs/SEVIRIASR.nl
+                DATA     varobs_ukvnamelist_seviriasr.nc4)
+ADD_WRITER_TEST(NAME     varobswriter_ukvnamelist_seviviclr
+                YAML     varobswriter_ukvnamelist_seviriclr.yaml
+                NAMELIST ../../etc/ukv/varobs/SEVIRIClr.nl
+                DATA     varobs_ukvnamelist_seviriclr.nc4)
 ADD_WRITER_TEST(NAME     varobswriter_ukvnamelist_sonde
                 YAML     varobswriter_ukvnamelist_sonde.yaml
                 NAMELIST ../../etc/ukv/varobs/Sonde.nl
@@ -662,6 +670,14 @@ ADD_WRITER_TEST(NAME     cxwriter_globalnamelist_screen
 
 # Tests the ukv namelist files in the etc directory
 
+ADD_WRITER_TEST(NAME     cxwriter_ukvnamelist_seviriasr
+                YAML     cxwriter_ukvnamelist_seviriasr.yaml
+                NAMELIST ../../etc/ukv/cx/SEVIRIASR.nl
+                DATA     cx_ukvnamelist_seviriasr.nc4 dummy.nc4)
+ADD_WRITER_TEST(NAME     cxwriter_ukvnamelist_seviriclr
+                YAML     cxwriter_ukvnamelist_seviriclr.yaml
+                NAMELIST ../../etc/ukv/cx/SEVIRIClr.nl
+                DATA     cx_ukvnamelist_seviriclr.nc4 dummy.nc4)
 ADD_WRITER_TEST(NAME     cxwriter_ukvnamelist_sonde
                 YAML     cxwriter_ukvnamelist_sonde.yaml
                 NAMELIST ../../etc/ukv/cx/Sonde.nl

diff --git a/test/generate_unittest_netcdfs.py b/test/generate_unittest_netcdfs.py
@@ -853,6 +853,26 @@ def copy_var_to_var(Group, invarname, outvarname, filename):
                                  ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier'],
                                  'testinput/varobs_globalnamelist_seviriclr.nc4')
 
+    # SEVIRIClr UKV
+    output_full_varobs_to_netcdf(['MetaData/latitude', 'MetaData/longitude', 'OneDVar/skinTemperature', 'MetaData/sensorZenithAngle',
+                                  'MetaData/solarZenithAngle', 'Emiss/emissivityIR', 'MetaData/ozoneTotal'],
+                                 ['ObsValue/brightnessTemperature', 'ObsError/brightnessTemperature',
+                                  'BiasCorrObsValue/brightnessTemperature', 'thickness_850_300hPa_satid_13Predictor/brightnessTemperature',
+                                  'thickness_850_300hPa_satid_17Predictor/brightnessTemperature',
+                                  'ObsValue/cloud_amount', 'ObsValue/cloud_top_temperature', 'ObsValue/lower_layer_pressure'],
+                                 ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier'],
+                                 'testinput/varobs_ukvnamelist_seviriclr.nc4')
+
+    # SEVIRIASR UKV
+    output_full_varobs_to_netcdf(['MetaData/latitude', 'MetaData/longitude', 'OneDVar/skinTemperature', 'MetaData/sensorZenithAngle',
+                                  'MetaData/solarZenithAngle', 'Emiss/emissivityIR', 'MetaData/ozoneTotal'],
+                                 ['ObsValue/brightnessTemperature', 'ObsError/brightnessTemperature',
+                                  'BiasCorrObsValue/brightnessTemperature', 'thickness_850_300hPa_satid_13Predictor/brightnessTemperature',
+                                  'thickness_850_300hPa_satid_17Predictor/brightnessTemperature',
+                                  'ObsValue/cloud_amount', 'ObsValue/cloud_top_temperature', 'ObsValue/lower_layer_pressure'],
+                                 ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier'],
+                                 'testinput/varobs_ukvnamelist_seviriasr.nc4')
+
     # Aircraft
     output_full_varobs_to_netcdf(['MetaData/latitude',
                                   'MetaData/longitude',
@@ -1057,6 +1077,22 @@ def copy_var_to_var(Group, invarname, outvarname, filename):
                               'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'],
                              'testinput/cx_globalnamelist_seviriclr.nc4')
 
+    # SEVIRIClr UKV
+    output_full_cx_to_netcdf(['skin_temperature', 'ice_area_fraction', 'surface_altitude', 'surface_pressure', 'uwind_at_10m',
+                              'vwind_at_10m', 'surface_temperature', 'relative_humidity_2m', 'surface_pressure_at_mean_sea_level'],
+                             ['potential_temperature', 'specific_humidity', 'mass_content_of_cloud_ice_in_atmosphere_layer',
+                              'mass_content_of_cloud_liquid_water_in_atmosphere_layer', 'air_pressure_levels',
+                              'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'],
+                             'testinput/cx_ukvnamelist_seviriclr.nc4')
+
+    # SEVIRIASR UKV
+    output_full_cx_to_netcdf(['skin_temperature', 'ice_area_fraction', 'surface_altitude', 'surface_pressure', 'uwind_at_10m',
+                              'vwind_at_10m', 'surface_temperature', 'relative_humidity_2m', 'surface_pressure_at_mean_sea_level'],
+                             ['potential_temperature', 'specific_humidity', 'mass_content_of_cloud_ice_in_atmosphere_layer',
+                              'mass_content_of_cloud_liquid_water_in_atmosphere_layer', 'air_pressure_levels',
+                              'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'],
+                             'testinput/cx_ukvnamelist_seviriasr.nc4')
+
     # GroundGPS
     output_full_cx_to_netcdf(['skin_temperature', 'ice_area_fraction', 'surface_altitude', 'surface_pressure', 'uwind_at_10m',
                               'vwind_at_10m', 'surface_temperature', 'relative_humidity_2m'],

diff --git a/test/testinput/cx_ukvnamelist_seviriasr.nc4 b/test/testinput/cx_ukvnamelist_seviriasr.nc4
diff --git a/test/testinput/cx_ukvnamelist_seviriclr.nc4 b/test/testinput/cx_ukvnamelist_seviriclr.nc4
diff --git a/test/testinput/cxwriter_ukvnamelist_seviriasr.yaml b/test/testinput/cxwriter_ukvnamelist_seviriasr.yaml
@@ -0,0 +1,36 @@
+time window:
+  begin: 2018-01-01T00:00:00Z
+  end: 2018-01-01T01:00:00Z
+
+observations:
+  - obs space:
+      name: SEVIRIASR
+      obsdatain:
+        engine:
+          type: H5File
+          obsfile: Data/dummy.nc4
+      simulated variables: [dummy]
+    geovals:
+      filename: Data/cx_ukvnamelist_seviriasr.nc4
+    obs filters:
+    # Set the flag of observations with missing values to "pass": we want to check if these
+    # values are encoded correctly in the Cx file.
+    - filter: Reset Flags to Pass
+      flags_to_reset: [10, 15]  # missing, Hfailed
+    - filter: Cx Writer
+      namelist_directory: ../etc/ukv/cx
+      reject_obs_with_any_variable_failing_qc: true
+      general_mode: debug
+      IC_PLevels: 5
+    - filter: Cx Checker
+      expected_surface_variables: ["1","2","3","4","5","6","13","16","17"]
+      expected_upper_air_variables: ["1","5","11","29","30","31","34","35"]
+      expected_main_table_columns:
+      - # batch 1 - 9 1dvalues first ; then 2d values in expected order
+        - ["27.10","37.10","67.10","77.10","47.10","57.10","7.10","87.10","17.10","1.10","1.20","1.30","11.10","11.20","11.30","41.10","41.20","41.30","21.10","21.20","21.30","31.10","31.20","31.30","51.10","51.20","51.30","71.10","71.20","71.30","61.10","61.20","61.30"]  # column 1 - 1st observation
+        - ["**********","**********","**********","**********","**********","**********","**********","**********","**********","2.10","**********","2.30","12.10","**********","12.30","42.10","**********","42.30","22.10","**********","22.30","32.10","**********","32.30","52.10","**********","52.30","72.10","**********","72.30","62.10","**********","62.30"]  # column 2 - 2nd observation
+        - ["27.30","37.30","67.30","77.30","47.30","57.30","7.30","87.30","17.30","3.10","3.20","3.30","13.10","13.20","13.30","43.10","43.20","43.30","23.10","23.20","23.30","33.10","33.20","33.30","53.10","53.20","53.30","73.10","73.20","73.30","63.10","63.20","63.30"] # column3 - observation 3
+        - ["27.40","37.40","67.40","77.40","47.40","57.40","7.40","87.40","17.40","4.10","4.20","4.30","14.10","14.20","14.30","44.10","44.20","44.30","24.10","24.20","24.30","34.10","34.20","34.30","54.10","54.20","54.30","74.10","74.20","74.30","64.10","64.20","64.30"] # column4 - observation 4
+    HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter.
+    benchmarkFlag: 1000 # just to keep the ObsFilters test happy
+    flaggedBenchmark: 0
diff --git a/test/testinput/cxwriter_ukvnamelist_seviriclr.yaml b/test/testinput/cxwriter_ukvnamelist_seviriclr.yaml
@@ -0,0 +1,36 @@
+time window:
+  begin: 2018-01-01T00:00:00Z
+  end: 2018-01-01T01:00:00Z
+
+observations:
+  - obs space:
+      name: SEVIRIClr
+      obsdatain:
+        engine:
+          type: H5File
+          obsfile: Data/dummy.nc4
+      simulated variables: [dummy]
+    geovals:
+      filename: Data/cx_ukvnamelist_seviriclr.nc4
+    obs filters:
+    # Set the flag of observations with missing values to "pass": we want to check if these
+    # values are encoded correctly in the Cx file.
+    - filter: Reset Flags to Pass
+      flags_to_reset: [10, 15]  # missing, Hfailed
+    - filter: Cx Writer
+      namelist_directory: ../etc/ukv/cx
+      reject_obs_with_any_variable_failing_qc: true
+      general_mode: debug
+      IC_PLevels: 5
+    - filter: Cx Checker
+      expected_surface_variables: ["1","2","3","4","5","6","13","16","17"]
+      expected_upper_air_variables: ["1","5","11","29","30","31","34","35"]
+      expected_main_table_columns:
+      - # batch 1 - 9 1dvalues first ; then 2d values in expected order
+        - ["27.10","37.10","67.10","77.10","47.10","57.10","7.10","87.10","17.10","1.10","1.20","1.30","11.10","11.20","11.30","41.10","41.20","41.30","21.10","21.20","21.30","31.10","31.20","31.30","51.10","51.20","51.30","71.10","71.20","71.30","61.10","61.20","61.30"]  # column 1 - 1st observation
+        - ["**********","**********","**********","**********","**********","**********","**********","**********","**********","2.10","**********","2.30","12.10","**********","12.30","42.10","**********","42.30","22.10","**********","22.30","32.10","**********","32.30","52.10","**********","52.30","72.10","**********","72.30","62.10","**********","62.30"]  # column 2 - 2nd observation
+        - ["27.30","37.30","67.30","77.30","47.30","57.30","7.30","87.30","17.30","3.10","3.20","3.30","13.10","13.20","13.30","43.10","43.20","43.30","23.10","23.20","23.30","33.10","33.20","33.30","53.10","53.20","53.30","73.10","73.20","73.30","63.10","63.20","63.30"] # column3 - observation 3
+        - ["27.40","37.40","67.40","77.40","47.40","57.40","7.40","87.40","17.40","4.10","4.20","4.30","14.10","14.20","14.30","44.10","44.20","44.30","24.10","24.20","24.30","34.10","34.20","34.30","54.10","54.20","54.30","74.10","74.20","74.30","64.10","64.20","64.30"] # column4 - observation 4
+    HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter.
+    benchmarkFlag: 1000 # just to keep the ObsFilters test happy
+    flaggedBenchmark: 0
diff --git a/test/testinput/varobs_ukvnamelist_seviriasr.nc4 b/test/testinput/varobs_ukvnamelist_seviriasr.nc4
diff --git a/test/testinput/varobs_ukvnamelist_seviriclr.nc4 b/test/testinput/varobs_ukvnamelist_seviriclr.nc4