diff --git a/etc/ukv/cx/SEVIRIASR.nl b/etc/ukv/cx/SEVIRIASR.nl new file mode 100644 index 00000000..7e3c5a63 --- /dev/null +++ b/etc/ukv/cx/SEVIRIASR.nl @@ -0,0 +1,3 @@ +&CXControlNL +CxFields=4,10,12,24,31,33,254,266,267,268,407,409,3209,3210,3236,3245,16222 +/ diff --git a/etc/ukv/cx/SEVIRIClr.nl b/etc/ukv/cx/SEVIRIClr.nl new file mode 100644 index 00000000..7e3c5a63 --- /dev/null +++ b/etc/ukv/cx/SEVIRIClr.nl @@ -0,0 +1,3 @@ +&CXControlNL +CxFields=4,10,12,24,31,33,254,266,267,268,407,409,3209,3210,3236,3245,16222 +/ diff --git a/etc/ukv/varobs/SEVIRIASR.nl b/etc/ukv/varobs/SEVIRIASR.nl new file mode 100644 index 00000000..1c3a2959 --- /dev/null +++ b/etc/ukv/varobs/SEVIRIASR.nl @@ -0,0 +1,3 @@ +&VarobsControlNL +Varfields=10,11,18,19,21,28,31,34,54,55,80 +/ diff --git a/etc/ukv/varobs/SEVIRIClr.nl b/etc/ukv/varobs/SEVIRIClr.nl new file mode 100644 index 00000000..1c3a2959 --- /dev/null +++ b/etc/ukv/varobs/SEVIRIClr.nl @@ -0,0 +1,3 @@ +&VarobsControlNL +Varfields=10,11,18,19,21,28,31,34,54,55,80 +/ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c76a9732..26d19ebd 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -359,6 +359,14 @@ ADD_WRITER_TEST(NAME varobswriter_globalnamelist_aircraft # Tests the UKV namelist files in the etc directory +ADD_WRITER_TEST(NAME varobswriter_ukvnamelist_seviviasr + YAML varobswriter_ukvnamelist_seviriasr.yaml + NAMELIST ../../etc/ukv/varobs/SEVIRIASR.nl + DATA varobs_ukvnamelist_seviriasr.nc4) +ADD_WRITER_TEST(NAME varobswriter_ukvnamelist_seviviclr + YAML varobswriter_ukvnamelist_seviriclr.yaml + NAMELIST ../../etc/ukv/varobs/SEVIRIClr.nl + DATA varobs_ukvnamelist_seviriclr.nc4) ADD_WRITER_TEST(NAME varobswriter_ukvnamelist_sonde YAML varobswriter_ukvnamelist_sonde.yaml NAMELIST ../../etc/ukv/varobs/Sonde.nl @@ -669,6 +677,14 @@ ADD_WRITER_TEST(NAME cxwriter_globalnamelist_oceanwinds # Tests the ukv namelist files in the etc directory +ADD_WRITER_TEST(NAME cxwriter_ukvnamelist_seviriasr + YAML cxwriter_ukvnamelist_seviriasr.yaml + NAMELIST ../../etc/ukv/cx/SEVIRIASR.nl + DATA cx_ukvnamelist_seviriasr.nc4 dummy.nc4) +ADD_WRITER_TEST(NAME cxwriter_ukvnamelist_seviriclr + YAML cxwriter_ukvnamelist_seviriclr.yaml + NAMELIST ../../etc/ukv/cx/SEVIRIClr.nl + DATA cx_ukvnamelist_seviriclr.nc4 dummy.nc4) ADD_WRITER_TEST(NAME cxwriter_ukvnamelist_sonde YAML cxwriter_ukvnamelist_sonde.yaml NAMELIST ../../etc/ukv/cx/Sonde.nl diff --git a/test/generate_unittest_netcdfs.py b/test/generate_unittest_netcdfs.py index 5396da32..01a34045 100644 --- a/test/generate_unittest_netcdfs.py +++ b/test/generate_unittest_netcdfs.py @@ -854,6 +854,26 @@ def copy_var_to_var(Group, invarname, outvarname, filename): ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier'], 'testinput/varobs_globalnamelist_seviriclr.nc4') + # SEVIRIClr UKV + output_full_varobs_to_netcdf(['MetaData/latitude', 'MetaData/longitude', 'OneDVar/skinTemperature', 'MetaData/sensorZenithAngle', + 'MetaData/solarZenithAngle', 'Emiss/emissivityIR', 'MetaData/ozoneTotal'], + ['ObsValue/brightnessTemperature', 'ObsError/brightnessTemperature', + 'BiasCorrObsValue/brightnessTemperature', 'thickness_850_300hPa_satid_13Predictor/brightnessTemperature', + 'thickness_850_300hPa_satid_17Predictor/brightnessTemperature', + 'ObsValue/cloud_amount', 'ObsValue/cloud_top_temperature', 'ObsValue/lower_layer_pressure'], + ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier'], + 'testinput/varobs_ukvnamelist_seviriclr.nc4') + + # SEVIRIASR UKV + output_full_varobs_to_netcdf(['MetaData/latitude', 'MetaData/longitude', 'OneDVar/skinTemperature', 'MetaData/sensorZenithAngle', + 'MetaData/solarZenithAngle', 'Emiss/emissivityIR', 'MetaData/ozoneTotal'], + ['ObsValue/brightnessTemperature', 'ObsError/brightnessTemperature', + 'BiasCorrObsValue/brightnessTemperature', 'thickness_850_300hPa_satid_13Predictor/brightnessTemperature', + 'thickness_850_300hPa_satid_17Predictor/brightnessTemperature', + 'ObsValue/cloud_amount', 'ObsValue/cloud_top_temperature', 'ObsValue/lower_layer_pressure'], + ['MetaData/surfaceQualifier', 'MetaData/satelliteIdentifier'], + 'testinput/varobs_ukvnamelist_seviriasr.nc4') + # Aircraft output_full_varobs_to_netcdf(['MetaData/latitude', 'MetaData/longitude', @@ -1058,6 +1078,22 @@ def copy_var_to_var(Group, invarname, outvarname, filename): 'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'], 'testinput/cx_globalnamelist_seviriclr.nc4') + # SEVIRIClr UKV + output_full_cx_to_netcdf(['skin_temperature', 'ice_area_fraction', 'surface_altitude', 'surface_pressure', 'uwind_at_10m', + 'vwind_at_10m', 'surface_temperature', 'relative_humidity_2m', 'surface_pressure_at_mean_sea_level'], + ['potential_temperature', 'specific_humidity', 'mass_content_of_cloud_ice_in_atmosphere_layer', + 'mass_content_of_cloud_liquid_water_in_atmosphere_layer', 'air_pressure_levels', + 'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'], + 'testinput/cx_ukvnamelist_seviriclr.nc4') + + # SEVIRIASR UKV + output_full_cx_to_netcdf(['skin_temperature', 'ice_area_fraction', 'surface_altitude', 'surface_pressure', 'uwind_at_10m', + 'vwind_at_10m', 'surface_temperature', 'relative_humidity_2m', 'surface_pressure_at_mean_sea_level'], + ['potential_temperature', 'specific_humidity', 'mass_content_of_cloud_ice_in_atmosphere_layer', + 'mass_content_of_cloud_liquid_water_in_atmosphere_layer', 'air_pressure_levels', + 'cloud_volume_fraction_in_atmosphere_layer', 'liquid_cloud_volume_fraction_in_atmosphere_layer', 'ice_cloud_volume_fraction_in_atmosphere_layer'], + 'testinput/cx_ukvnamelist_seviriasr.nc4') + # GroundGPS output_full_cx_to_netcdf(['skin_temperature', 'ice_area_fraction', 'surface_altitude', 'surface_pressure', 'uwind_at_10m', 'vwind_at_10m', 'surface_temperature', 'relative_humidity_2m'], diff --git a/test/testinput/cx_ukvnamelist_seviriasr.nc4 b/test/testinput/cx_ukvnamelist_seviriasr.nc4 new file mode 100644 index 00000000..2cef2914 Binary files /dev/null and b/test/testinput/cx_ukvnamelist_seviriasr.nc4 differ diff --git a/test/testinput/cx_ukvnamelist_seviriclr.nc4 b/test/testinput/cx_ukvnamelist_seviriclr.nc4 new file mode 100644 index 00000000..2cef2914 Binary files /dev/null and b/test/testinput/cx_ukvnamelist_seviriclr.nc4 differ diff --git a/test/testinput/cxwriter_ukvnamelist_seviriasr.yaml b/test/testinput/cxwriter_ukvnamelist_seviriasr.yaml new file mode 100644 index 00000000..b57b9322 --- /dev/null +++ b/test/testinput/cxwriter_ukvnamelist_seviriasr.yaml @@ -0,0 +1,36 @@ +time window: + begin: 2018-01-01T00:00:00Z + end: 2018-01-01T01:00:00Z + +observations: + - obs space: + name: SEVIRIASR + obsdatain: + engine: + type: H5File + obsfile: Data/dummy.nc4 + simulated variables: [dummy] + geovals: + filename: Data/cx_ukvnamelist_seviriasr.nc4 + obs filters: + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the Cx file. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + - filter: Cx Writer + namelist_directory: ../etc/ukv/cx + reject_obs_with_any_variable_failing_qc: true + general_mode: debug + IC_PLevels: 5 + - filter: Cx Checker + expected_surface_variables: ["1","2","3","4","5","6","13","16","17"] + expected_upper_air_variables: ["1","5","11","29","30","31","34","35"] + expected_main_table_columns: + - # batch 1 - 9 1dvalues first ; then 2d values in expected order + - ["27.10","37.10","67.10","77.10","47.10","57.10","7.10","87.10","17.10","1.10","1.20","1.30","11.10","11.20","11.30","41.10","41.20","41.30","21.10","21.20","21.30","31.10","31.20","31.30","51.10","51.20","51.30","71.10","71.20","71.30","61.10","61.20","61.30"] # column 1 - 1st observation + - ["**********","**********","**********","**********","**********","**********","**********","**********","**********","2.10","**********","2.30","12.10","**********","12.30","42.10","**********","42.30","22.10","**********","22.30","32.10","**********","32.30","52.10","**********","52.30","72.10","**********","72.30","62.10","**********","62.30"] # column 2 - 2nd observation + - ["27.30","37.30","67.30","77.30","47.30","57.30","7.30","87.30","17.30","3.10","3.20","3.30","13.10","13.20","13.30","43.10","43.20","43.30","23.10","23.20","23.30","33.10","33.20","33.30","53.10","53.20","53.30","73.10","73.20","73.30","63.10","63.20","63.30"] # column3 - observation 3 + - ["27.40","37.40","67.40","77.40","47.40","57.40","7.40","87.40","17.40","4.10","4.20","4.30","14.10","14.20","14.30","44.10","44.20","44.30","24.10","24.20","24.30","34.10","34.20","34.30","54.10","54.20","54.30","74.10","74.20","74.30","64.10","64.20","64.30"] # column4 - observation 4 + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 diff --git a/test/testinput/cxwriter_ukvnamelist_seviriclr.yaml b/test/testinput/cxwriter_ukvnamelist_seviriclr.yaml new file mode 100644 index 00000000..35fa9dca --- /dev/null +++ b/test/testinput/cxwriter_ukvnamelist_seviriclr.yaml @@ -0,0 +1,36 @@ +time window: + begin: 2018-01-01T00:00:00Z + end: 2018-01-01T01:00:00Z + +observations: + - obs space: + name: SEVIRIClr + obsdatain: + engine: + type: H5File + obsfile: Data/dummy.nc4 + simulated variables: [dummy] + geovals: + filename: Data/cx_ukvnamelist_seviriclr.nc4 + obs filters: + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the Cx file. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + - filter: Cx Writer + namelist_directory: ../etc/ukv/cx + reject_obs_with_any_variable_failing_qc: true + general_mode: debug + IC_PLevels: 5 + - filter: Cx Checker + expected_surface_variables: ["1","2","3","4","5","6","13","16","17"] + expected_upper_air_variables: ["1","5","11","29","30","31","34","35"] + expected_main_table_columns: + - # batch 1 - 9 1dvalues first ; then 2d values in expected order + - ["27.10","37.10","67.10","77.10","47.10","57.10","7.10","87.10","17.10","1.10","1.20","1.30","11.10","11.20","11.30","41.10","41.20","41.30","21.10","21.20","21.30","31.10","31.20","31.30","51.10","51.20","51.30","71.10","71.20","71.30","61.10","61.20","61.30"] # column 1 - 1st observation + - ["**********","**********","**********","**********","**********","**********","**********","**********","**********","2.10","**********","2.30","12.10","**********","12.30","42.10","**********","42.30","22.10","**********","22.30","32.10","**********","32.30","52.10","**********","52.30","72.10","**********","72.30","62.10","**********","62.30"] # column 2 - 2nd observation + - ["27.30","37.30","67.30","77.30","47.30","57.30","7.30","87.30","17.30","3.10","3.20","3.30","13.10","13.20","13.30","43.10","43.20","43.30","23.10","23.20","23.30","33.10","33.20","33.30","53.10","53.20","53.30","73.10","73.20","73.30","63.10","63.20","63.30"] # column3 - observation 3 + - ["27.40","37.40","67.40","77.40","47.40","57.40","7.40","87.40","17.40","4.10","4.20","4.30","14.10","14.20","14.30","44.10","44.20","44.30","24.10","24.20","24.30","34.10","34.20","34.30","54.10","54.20","54.30","74.10","74.20","74.30","64.10","64.20","64.30"] # column4 - observation 4 + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 diff --git a/test/testinput/varobs_ukvnamelist_seviriasr.nc4 b/test/testinput/varobs_ukvnamelist_seviriasr.nc4 new file mode 100644 index 00000000..025e7058 Binary files /dev/null and b/test/testinput/varobs_ukvnamelist_seviriasr.nc4 differ diff --git a/test/testinput/varobs_ukvnamelist_seviriclr.nc4 b/test/testinput/varobs_ukvnamelist_seviriclr.nc4 new file mode 100644 index 00000000..025e7058 Binary files /dev/null and b/test/testinput/varobs_ukvnamelist_seviriclr.nc4 differ diff --git a/test/testinput/varobswriter_ukvnamelist_seviriasr.yaml b/test/testinput/varobswriter_ukvnamelist_seviriasr.yaml new file mode 100644 index 00000000..b624e530 --- /dev/null +++ b/test/testinput/varobswriter_ukvnamelist_seviriasr.yaml @@ -0,0 +1,58 @@ +time window: + begin: 2018-01-01T00:00:00Z + end: 2018-01-01T02:00:00Z + +observations: +# This ob space shows what happens if the following are left as defaults +# reject_obs_with_all_variables_failing_qc: false +# reject_obs_with_any_variables_failing_qc: false +# This means that despite the blacklist the entries still end up in the varobs file + - obs space: + name: SEVIRIASR + obsdatain: + engine: + type: H5File + obsfile: Data/varobs_ukvnamelist_seviriasr.nc4 + simulated variables: [brightnessTemperature] + channels: 1, 3 + obs filters: + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + # Reject observation 3: the blacklist doesn't stop the ob getting into the varobs because of the above "bad" settings + - filter: BlackList + where: + - variable: + name: MetaData/latitude + value: is_not_valid + - filter: VarObs Writer + namelist_directory: ../etc/ukv/varobs + general_mode: debug + - filter: VarObs Checker + expected_main_table_columns: + # Only channels 1 and 3 are simulated and hence written to the VarObs file. + # In the arrays below, rows denote locations and columns are the varobs channels are consecutive. + field: ["10","10","11","18","19","21","28","31","34","54","55","55","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80", + "10","10","11","18","19","21","28","31","34","54","55","55","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80", + "10","10","11","18","19","21","28","31","34","54","55","55","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80", + "10","10","11","18","19","21","28","31","34","54","55","55","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80"] + level: ["1","2","1","1","1","1","1","1","1","1","1","2","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31", + "1","2","1","1","1","1","1","1","1","1","1","2","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31", + "1","2","1","1","1","1","1","1","1","1","1","2","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31", + "1","2","1","1","1","1","1","1","1","1","1","2","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31"] + ob value: ["21.10000","21.30000","27.10000","67.10000","37.10000","3.00000","13.00000","47.10000","57.10000","2.00000","1.00000","2.00000","0.00000","72.20000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000", + "22.10000","22.30000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-32768.00000","-32768.00000","-1073741824.00000","-1073741824.00000","0.00000","0.00000","0.00000","0.00000","74.20000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000", + "23.10000","23.30000","27.30000","67.30000","37.30000","7.00000","17.00000","47.30000","57.30000","2.00000","1.00000","2.00000","0.00000","76.20000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000", + "24.10000","24.30000","27.40000","67.40000","37.40000","9.00000","19.00000","47.40000","57.40000","2.00000","1.00000","2.00000","0.00000","78.20000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000"] + lat: ["7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000", + "-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000", + "7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000", + "7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000"] + lon: ["17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000", + "-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000", + "17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000", + "17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000"] + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 diff --git a/test/testinput/varobswriter_ukvnamelist_seviriclr.yaml b/test/testinput/varobswriter_ukvnamelist_seviriclr.yaml new file mode 100644 index 00000000..808b1034 --- /dev/null +++ b/test/testinput/varobswriter_ukvnamelist_seviriclr.yaml @@ -0,0 +1,58 @@ +time window: + begin: 2018-01-01T00:00:00Z + end: 2018-01-01T02:00:00Z + +observations: +# This ob space shows what happens if the following are left as defaults +# reject_obs_with_all_variables_failing_qc: false +# reject_obs_with_any_variables_failing_qc: false +# This means that despite the blacklist the entries still end up in the varobs file + - obs space: + name: SEVIRIClr + obsdatain: + engine: + type: H5File + obsfile: Data/varobs_ukvnamelist_seviriclr.nc4 + simulated variables: [brightnessTemperature] + channels: 1, 3 + obs filters: + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + # Reject observation 3: the blacklist doesn't stop the ob getting into the varobs because of the above "bad" settings + - filter: BlackList + where: + - variable: + name: MetaData/latitude + value: is_not_valid + - filter: VarObs Writer + namelist_directory: ../etc/ukv/varobs + general_mode: debug + - filter: VarObs Checker + expected_main_table_columns: + # Only channels 1 and 3 are simulated and hence written to the VarObs file. + # In the arrays below, rows denote locations and columns are the varobs channels are consecutive. + field: ["10","10","11","18","19","21","28","31","34","54","55","55","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80", + "10","10","11","18","19","21","28","31","34","54","55","55","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80", + "10","10","11","18","19","21","28","31","34","54","55","55","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80", + "10","10","11","18","19","21","28","31","34","54","55","55","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80","80"] + level: ["1","2","1","1","1","1","1","1","1","1","1","2","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31", + "1","2","1","1","1","1","1","1","1","1","1","2","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31", + "1","2","1","1","1","1","1","1","1","1","1","2","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31", + "1","2","1","1","1","1","1","1","1","1","1","2","1","2","3","4","5","6","7","8","9","10","11","12","13","14","15","16","17","18","19","20","21","22","23","24","25","26","27","28","29","30","31"] + ob value: ["21.10000","21.30000","27.10000","67.10000","37.10000","3.00000","13.00000","47.10000","57.10000","2.00000","1.00000","2.00000","0.00000","72.20000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000", + "22.10000","22.30000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-32768.00000","-32768.00000","-1073741824.00000","-1073741824.00000","0.00000","0.00000","0.00000","0.00000","74.20000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000", + "23.10000","23.30000","27.30000","67.30000","37.30000","7.00000","17.00000","47.30000","57.30000","2.00000","1.00000","2.00000","0.00000","76.20000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000", + "24.10000","24.30000","27.40000","67.40000","37.40000","9.00000","19.00000","47.40000","57.40000","2.00000","1.00000","2.00000","0.00000","78.20000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000","0.00000"] + lat: ["7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.10000", + "-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000", + "7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000", + "7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000"] + lon: ["17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.10000", + "-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000","-1073741824.00000", + "17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000", + "17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000"] + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0