diff --git a/etc/ukv/cx/Radar.nl b/etc/ukv/cx/Radar.nl new file mode 100644 index 00000000..10cd24bd --- /dev/null +++ b/etc/ukv/cx/Radar.nl @@ -0,0 +1,8 @@ +&CXControlNL +! These STASH codes correspond to the following variables: +! 2: upper-air u +! 3: upper-air v +! 33: orography +! 150: upper-air w +CxFields=2,3,33,150 +/ diff --git a/etc/ukv/varobs/Radar.nl b/etc/ukv/varobs/Radar.nl new file mode 100644 index 00000000..7dd44fc0 --- /dev/null +++ b/etc/ukv/varobs/Radar.nl @@ -0,0 +1,10 @@ +&VarobsControlNL +! These indices correspond to the following variables: +! 63: Radial velocity +! 64: Beam tilt (elevation in OPS terminology) +! 65: Gate range +! 66: Gate azimuth +! 69: Station identifier +! 75: Station elevation (altitude above MSL in OPS terminology) +Varfields=63,64,65,66,69,75 +/ diff --git a/src/opsinputs/opsinputs_cxfields_mod.F90 b/src/opsinputs/opsinputs_cxfields_mod.F90 index 86743976..9361680d 100644 --- a/src/opsinputs/opsinputs_cxfields_mod.F90 +++ b/src/opsinputs/opsinputs_cxfields_mod.F90 @@ -63,7 +63,7 @@ module opsinputs_cxfields_mod character(len=*), parameter, public :: opsinputs_cxfields_rh = var_rh character(len=*), parameter, public :: opsinputs_cxfields_u = var_u character(len=*), parameter, public :: opsinputs_cxfields_v = var_v -character(len=*), parameter, public :: opsinputs_cxfields_w = opsinputs_cxfields_unknown +character(len=*), parameter, public :: opsinputs_cxfields_w = var_w character(len=*), parameter, public :: opsinputs_cxfields_q = var_q character(len=*), parameter, public :: opsinputs_cxfields_qc = opsinputs_cxfields_unknown character(len=*), parameter, public :: opsinputs_cxfields_p_bar = var_prs diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index e016154e..f2404abc 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -65,6 +65,7 @@ module opsinputs_fill_mod opsinputs_fill_fillelementtypefromsimulatedvariable, & opsinputs_fill_fillelementtype2dfromsimulatedvariable, & opsinputs_fill_fillinteger, & + opsinputs_fill_fillinteger2d, & opsinputs_fill_fillreal, & opsinputs_fill_fillreal2d, & opsinputs_fill_fillrealfromgeoval, & @@ -580,11 +581,11 @@ end subroutine opsinputs_fill_fillelementtype2dfromsimulatedvariable !> \param[in] JediValueVarName !> Name of the JEDI variable containing observation values. !> \param[in] JediValueGroup -!> Group of the JEDI variable containing observation values. +!> Group name of the JEDI variable containing observation values. !> \param[in] JediErrorVarName !> (Optional) Name of the JEDI variable containing observation errors. !> \param[in] JediErrorGroup -!> (Optional) Group of the JEDI variable containing observation errors. +!> (Optional) Group name of the JEDI variable containing observation errors. !> \param[in] PackPGEs !> Optional; true by default. If set to false, PGEs won't be stored in packed form. !> The Ops_VarobPGEs subroutine expects PGEs to be stored in packed form for most varobs fields, @@ -693,11 +694,11 @@ end subroutine opsinputs_fill_fillelementtypefromnormalvariable !> \param[in] JediValueVarName !> Name of the JEDI variable containing observation values. !> \param[in] JediValueGroup -!> Group of the JEDI variable containing observation values. +!> Group name of the JEDI variable containing observation values. !> \param[in] JediErrorVarName !> (Optional) Name of the JEDI variable containing observation errors. !> \param[in] JediErrorGroup -!> (Optional) Group of the JEDI variable containing observation errors. +!> (Optional) Group name of the JEDI variable containing observation errors. !> \param[in] PackPGEs !> Optional; true by default. If set to false, PGEs won't be stored in packed form. !> The Ops_VarobPGEs subroutine expects PGEs to be stored in packed form for most varobs fields, @@ -829,13 +830,13 @@ end subroutine opsinputs_fill_fillelementtype2dfromnormalvariable !> \param[in] JediValueVarName !> Name of the JEDI variable containing observation values. !> \param[in] JediValueGroup -!> Group of the JEDI variable containing observation values. +!> Group name of the JEDI variable containing observation values. !> \param[in] LevelsAreTopToBottom !> A logical to specify if the levels being passed in are top to bottom in the atmosphere. !> \param[in] JediErrorVarName !> (Optional) Name of the JEDI variable containing observation errors. !> \param[in] JediErrorGroup -!> (Optional) Group of the JEDI variable containing observation errors. +!> (Optional) Group name of the JEDI variable containing observation errors. !> !> \note This function returns early (without a warning) if the specified JEDI variable is not found. !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield @@ -950,7 +951,7 @@ end subroutine opsinputs_fill_fillelementtype2dfromnormalvariablewithlevels !> \param[in] JediVarName !> Name of the JEDI variable used to populate \p Real1. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p Real1. +!> Group name of the JEDI variable used to populate \p Real1. !> !> \note This function returns early (without a warning) if the specified JEDI variable is not found. !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield @@ -1030,7 +1031,7 @@ end subroutine opsinputs_fill_fillreal !> variable with no channel suffix (in which case \p Real2 will have only a single row) or a set !> of variables with suffixes corresponding to the indices specified in \p Channels. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p Real2. +!> Group name of the JEDI variable used to populate \p Real2. !> \param[in] compressVarChannels !> Whether to apply var channel compression (No NaN spaces between channels) !> \param[in] sizeOfVarobsArray @@ -1165,7 +1166,7 @@ end subroutine opsinputs_fill_fillreal2d_norecords !> \param[in] JediVarName !> Name of the JEDI variable used to populate \p Real2. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p Real2. +!> Group name of the JEDI variable used to populate \p Real2. !> !> \note This function returns early (without a warning) if the specified JEDI variable is not found. !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield @@ -1235,7 +1236,7 @@ end subroutine opsinputs_fill_fillreal2d_records !> \param[inout] Hdr !> Header to be populated. !> \param[in] OpsVarName -!> Name of the OB_type field to which \p Real1 corresponds. +!> Name of the OB_type field to which \p Real2 corresponds. !> \param[in] JediToOpsLayoutMapping !> Data needed to map JEDI locations stored on the current PE to OPS observations. !> \param[inout] Real2 @@ -1254,7 +1255,7 @@ end subroutine opsinputs_fill_fillreal2d_records !> suffix (in which case \p Real2 will have only a single row) or a set of variables with !> suffixes corresponding to the indices specified in \p Channels. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p Real2. +!> Group name of the JEDI variable used to populate \p Real2. !> !> \note This function returns early (without a warning) if the specified JEDI variable is not found. !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield @@ -1764,7 +1765,7 @@ end subroutine opsinputs_fill_fillreal2dfromgeovalformultilevelobs !> \param[in] JediVarName !> Name of the JEDI variable used to populate \p Int1. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p Int1. +!> Group name of the JEDI variable used to populate \p Int1. !> !> \note This function returns early (without a warning) if the specified JEDI variable is not found. !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield @@ -1817,6 +1818,87 @@ subroutine opsinputs_fill_fillinteger( & end if end subroutine opsinputs_fill_fillinteger + +! ------------------------------------------------------------------------------ +!> Populate a 2D array of integers and its header from a JEDI variable. +!> +!> \param[inout] Hdr +!> Header to be populated. +!> \param[in] OpsVarName +!> Name of the OB_type field to which \p Int2 corresponds. +!> \param[in] JediToOpsLayoutMapping +!> Data needed to map JEDI locations stored on the current PE to OPS observations. +!> \param[inout] Int2 +!> Pointer to the array to be populated. +!> \param[in] ObsSpace +!> Pointer to ioda::ObsSpace object containing the specified JEDI variable. The variable can +!> have either no channel suffix (in which case \p Int2 will have only a single row) or suffixes +!> representing the indices specified in \p Channels. +!> \param[in] Channels +!> Indices returned by ioda::ObsSpace::obsvariables().channels(). +!> \param[in] VarobsLength +!> Length of varobs profile. +!> \param[in] JediVarName +!> Name of the JEDI variable used to populate \p Int2. If each JEDI location needs to be mapped +!> to a separate OPS observation, this can represent either a single variable with no channel +!> suffix (in which case \p Int2 will have only a single row) or a set of variables with +!> suffixes corresponding to the indices specified in \p Channels. +!> \param[in] JediGroup +!> Group name of the JEDI variable used to populate \p Int2. +!> +!> \note This function returns early (without a warning) if the specified JEDI variable is not found. +!> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield +!> are not found. +subroutine opsinputs_fill_fillinteger2d( & + Hdr, OpsVarName, JediToOpsLayoutMapping, Int2, ObsSpace, Channels, & + VarobsLength, JediVarName, JediVarGroup) +implicit none + +! Subroutine arguments: +type(ElementHeader_Type), intent(inout) :: Hdr +character(len=*), intent(in) :: OpsVarName +type(opsinputs_jeditoopslayoutmapping), intent(in) :: JediToOpsLayoutMapping +integer(integer64), pointer, intent(out) :: Int2(:,:) +type(c_ptr), value, intent(in) :: ObsSpace +integer(c_int), intent(in) :: Channels(:) +integer(integer64), intent(in) :: VarobsLength +character(len=*), intent(in) :: JediVarName +character(len=*), intent(in) :: JediVarGroup +! todo(someone): add optional arguments used in opsinputs_fill_fillreal2d if there is a need. + +! Local declarations: +integer(kind=c_int) :: VarValue(JediToOpsLayoutMapping % NumJediObs) +integer(kind=c_int) :: CurrentVarValue +integer(kind=c_int) :: MissingInt +integer :: i +integer :: numchans + +! Body: + +MissingInt = missing_value(0_c_int32_t) + +! todo(someone): add this if needed +if (JediToOpsLayoutMapping % ConvertRecordsToMultilevelObs) then + call abor1_ftn("must extend opsinputs_fill_fillreal2d to deal with multi-level observations") +end if + +! todo(someone): make this configurable if required +numchans = 1 + +if (obsspace_has(ObsSpace, JediVarGroup, JediVarName)) then + ! Retrieve data from JEDI + call obsspace_get_db(ObsSpace, JediVarGroup, JediVarName, VarValue) + + ! Fill the OPS data structures + call Ops_Alloc(Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Int2, & + num_levels = int(numchans, kind=integer64)) + do i = 1, JediToOpsLayoutMapping % NumOpsObs + CurrentVarValue = VarValue(i) + if (CurrentVarValue /= MissingInt) Int2(i, 1) = CurrentVarValue + end do +end if +end subroutine opsinputs_fill_fillinteger2d + ! ------------------------------------------------------------------------------ !> Populate a 1D array of strings and its header from a JEDI variable. @@ -1837,7 +1919,7 @@ end subroutine opsinputs_fill_fillinteger !> \param[in] JediVarName !> Name of the JEDI variable used to populate \p String1. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p String1. +!> Group name of the JEDI variable used to populate \p String1. !> \param[in] ConvertIntToSTring !> Convert an integer-valued ObsSpace vector to a string. !> \note This function returns early (without a warning) if the specified JEDI variable is not found. @@ -1934,7 +2016,7 @@ end subroutine opsinputs_fill_fillstring !> \param[in] JediVarName !> Name of the JEDI variable used to populate \p Real1. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p Real1. +!> Group name of the JEDI variable used to populate \p Real1. !> \param[in] ReferenceTime !> Reference time. JEDI datetimes will be converted into offsets from this time. !> @@ -2010,7 +2092,7 @@ end subroutine opsinputs_fill_filltimeoffsets !> variable with no channel suffix (in which case \p Real2 will have only a single row) or a set !> of variables with suffixes corresponding to the indices specified in \p Channels. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p Real2. +!> Group name of the JEDI variable used to populate \p Real2. !> \param[in] ReferenceTime !> Reference time. JEDI datetimes will be converted into offsets from this time. !> @@ -2085,7 +2167,7 @@ end subroutine opsinputs_fill_filltimeoffsets2d_norecords !> \param[in] JediVarName !> Name of the JEDI variable used to populate \p Real2. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p Real2. +!> Group name of the JEDI variable used to populate \p Real2. !> \param[in] ReferenceTime !> Reference time. JEDI datetimes will be converted into offsets from this time. !> @@ -2181,7 +2263,7 @@ end subroutine opsinputs_fill_filltimeoffsets2d_records !> suffix (in which case \p Real2 will have only a single row) or a set of variables with !> suffixes corresponding to the indices specified in \p Channels. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p Real2. +!> Group name of the JEDI variable used to populate \p Real2. !> \param[in] ReferenceTime !> Reference time. JEDI datetimes will be converted into offsets from this time. !> @@ -2249,7 +2331,7 @@ end subroutine opsinputs_fill_filltimeoffsets2d !> variable with no channel suffix (in which case \p Coord2 will have only a single row) or a set !> of variables with suffixes corresponding to the indices specified in \p Channels. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p Coord2. +!> Group name of the JEDI variable used to populate \p Coord2. !> !> \note This function returns early (without a warning) if the specified JEDI variable is not found. !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield @@ -2318,7 +2400,7 @@ end subroutine opsinputs_fill_fillcoord2d_norecords !> \param[in] JediVarName !> Name of the JEDI variable used to populate \p Coord2. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p Coord2. +!> Group name of the JEDI variable used to populate \p Coord2. !> !> \note This function returns early (without a warning) if the specified JEDI variable is not found. !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield @@ -2390,7 +2472,7 @@ end subroutine opsinputs_fill_fillcoord2d_records !> suffix (in which case \p Coord2 will have only a single row) or a set of variables with !> suffixes corresponding to the indices specified in \p Channels. !> \param[in] JediGroup -!> Group of the JEDI variable used to populate \p Coord2. +!> Group name of the JEDI variable used to populate \p Coord2. !> !> \note This function returns early (without a warning) if the specified JEDI variable is not found. !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 3de03cef..acfd3aa3 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -35,6 +35,7 @@ module opsinputs_varobswriter_mod opsinputs_fill_fillelementtypefromsimulatedvariable, & opsinputs_fill_fillelementtype2dfromsimulatedvariable, & opsinputs_fill_fillinteger, & + opsinputs_fill_fillinteger2d, & opsinputs_fill_fillreal, & opsinputs_fill_fillreal2d, & opsinputs_fill_fillrealfromgeoval, & @@ -1017,8 +1018,17 @@ subroutine opsinputs_varobswriter_populateobservations( & ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % SBUVozone, "SBUVozone", Ob % Header % NumObsLocal, Ob % SBUVozone) case (VarField_RadialVelocity) - ! TODO(someone): handle this varfield - ! call Ops_Alloc(Ob % Header % RadialVelocSO, "RadialVelocSO", Ob % Header % NumObsLocal, Ob % RadialVelocSO) + ! Write DerivedObsValue/radialVelocity to both Ob % RadialVelocSO and Ob % RadialVelocity. + ! This ensures that the code in deps/ops/stubs/OpsMod_Varobs/Ops_VarobPGEs.inc works correctly. + ! The logical `RadWind_SuperOb` is always false in opsinputs, but is true in operational OPS. + ! By default that results in PGEs not being filled correctly. Writing out both OPS variables + ! fixes the problem. + call opsinputs_fill_fillelementtype2dfromsimulatedvariable( & + Ob % Header % RadialVelocSO, "RadialVelocSO", JediToOpsLayoutMapping, Ob % RadialVelocSO, & + ObsSpace, self % channels, Flags, ObsErrors, self % VarobsLength, "radialVelocity", "ObsValue") + call opsinputs_fill_fillelementtype2dfromsimulatedvariable( & + Ob % Header % RadialVelocity, "RadialVelocity", JediToOpsLayoutMapping, Ob % RadialVelocity, & + ObsSpace, self % channels, Flags, ObsErrors, self % VarobsLength, "radialVelocity", "ObsValue") case (VarField_Reflectivity) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % ReflectivitySO, "ReflectivitySO", Ob % Header % NumObsLocal, Ob % ReflectivitySO) @@ -1029,21 +1039,25 @@ subroutine opsinputs_varobswriter_populateobservations( & ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % ReflectivityI, "ReflectivityI", Ob % Header % NumObsLocal, Ob % ReflectivityI) case (VarField_RadarBeamElev) - ! TODO(someone): handle this varfield - ! call Ops_Alloc(Ob % Header % RadarBeamElev, "RadarBeamElev", Ob % Header % NumObsLocal, Ob % RadarBeamElev) + call opsinputs_fill_fillreal2d( & + Ob % Header % RadarBeamElev, "RadarBeamElev", JediToOpsLayoutMapping, Ob % RadarBeamElev, & + ObsSpace, self % channels, self % VarobsLength, "beamTiltAngle", "MetaData") case (VarField_RadarObRange) - ! TODO(someone): handle this varfield - ! call Ops_Alloc(Ob % Header % RadarObRange, "RadarObRange", Ob % Header % NumObsLocal, Ob % RadarObRange) + call opsinputs_fill_fillreal2d( & + Ob % Header % RadarObRange, "RadarObRange", JediToOpsLayoutMapping, Ob % RadarObRange, & + ObsSpace, self % channels, self % VarobsLength, "gateRange", "MetaData") case (VarField_RadarObAzim) call opsinputs_fill_fillreal2d( & - Ob % Header % RadarObAzim, "RadarObAzim", JediToOpsLayoutMapping, Ob % RadarObAzim, & - ObsSpace, self % channels, self % VarobsLength, "radarAzimuth", "MetaData") + Ob % Header % RadarObAzim, "RadarObAzim", JediToOpsLayoutMapping, Ob % RadarObAzim, & + ObsSpace, self % channels, self % VarobsLength, "beamAzimuthAngle", "MetaData") case (VarField_RadIdent) - ! TODO(someone): handle this varfield - ! call Ops_Alloc(Ob % Header % RadIdent, "RadIdent", Ob % Header % NumObsLocal, Ob % RadIdent) + call opsinputs_fill_fillinteger2d( & + Ob % Header % RadIdent, "RadIdent", JediToOpsLayoutMapping, Ob % RadIdent, & + ObsSpace, self % channels, self % VarobsLength, "stationIdentification", "MetaData") case (VarField_RadAltAboveMSL) - ! TODO(someone): handle this varfield - ! call Ops_Alloc(Ob % Header % RadAltAboveMSL, "RadAltAboveMSL", Ob % Header % NumObsLocal, Ob % RadAltAboveMSL) + call opsinputs_fill_fillreal2d( & + Ob % Header % RadAltAboveMSL, "RadAltAboveMSL", JediToOpsLayoutMapping, Ob % RadAltAboveMSL, & + ObsSpace, self % channels, self % VarobsLength, "stationElevation", "MetaData") case (VarField_RadNoiseLvl) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % RadNoiseLvl, "RadNoiseLvl", Ob % Header % NumObsLocal, Ob % RadNoiseLvl) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 82b66d4c..c76a9732 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -220,9 +220,10 @@ ADD_WRITER_TEST(NAME varobswriter_057_VarField_emissivity YAML 057_VarField_emissivity.yaml NAMELIST VarObsWriterNamelists_057_VarField_emissivity/IASI.nl DATA 057_VarField_emissivity.nc4) -ADD_WRITER_TEST(NAME varobswriter_066_VarField_radarobazim - YAML 066_VarField_radarobazim.yaml - DATA 066_VarField_radarobazim.nc4) +ADD_WRITER_TEST(NAME varobswriter_063_VarField_radialVelocity + YAML 063_VarField_radialVelocity.yaml + NAMELIST VarObsWriterNamelists_063_VarField_radialVelocity/Radar.nl + DATA 063_VarField_radialVelocity.nc4) ADD_WRITER_TEST(NAME varobswriter_067_VarField_GPS_Station_Height YAML 067_VarField_GPS_Station_Height.yaml DATA 067_VarField_GPS_Station_Height.nc4) @@ -366,6 +367,10 @@ ADD_WRITER_TEST(NAME varobswriter_ukvnamelist_surfacecloud YAML varobswriter_ukvnamelist_surfacecloud.yaml NAMELIST ../../etc/ukv/varobs/SurfaceCloud.nl DATA varobs_ukvnamelist_surfacecloud.nc4) +ADD_WRITER_TEST(NAME varobswriter_ukvnamelist_radar_doppler_wind + YAML varobswriter_ukvnamelist_radar_doppler_wind.yaml + NAMELIST ../../etc/ukv/varobs/Radar.nl + DATA varobs_ukvnamelist_radar_doppler_wind.nc4) ### CxWriter tests @@ -676,6 +681,10 @@ ADD_WRITER_TEST(NAME cxwriter_ukvnamelist_surface YAML cxwriter_ukvnamelist_surface.yaml NAMELIST ../../etc/ukv/cx/Surface.nl DATA cx_ukvnamelist_surface.nc4 dummy.nc4) +ADD_WRITER_TEST(NAME cxwriter_ukvnamelist_radar_doppler_wind + YAML cxwriter_ukvnamelist_radar_doppler_wind.yaml + NAMELIST ../../etc/ukv/cx/Radar.nl + DATA cx_ukvnamelist_radar_doppler_wind.nc4 dummy.nc4) ### Tests of auxiliary classes diff --git a/test/generate_unittest_netcdfs.py b/test/generate_unittest_netcdfs.py index f21bb3e0..5396da32 100644 --- a/test/generate_unittest_netcdfs.py +++ b/test/generate_unittest_netcdfs.py @@ -16,7 +16,7 @@ missing_float_nc = 9.969209968386869e+36 -def output_1d_simulated_var_to_netcdf(var_name, file_name, with_bias=False): +def output_1d_simulated_var_to_netcdf(var_name, file_name, with_bias=False, radar_doppler_wind=False): f = nc4.Dataset(file_name, 'w', format="NETCDF4") nlocs = 4 @@ -34,9 +34,31 @@ def output_1d_simulated_var_to_netcdf(var_name, file_name, with_bias=False): minute = 1 / 60. var[:] = [1 * minute, 2 * minute, 3 * minute, 4 * minute] - var = f.createVariable('MetaData/stationIdentification', str, ('Location')) - for i, s in enumerate(["station_1", "station_2", "station_3", "station_4"]): - var[i] = s + # Station ID + if radar_doppler_wind: + # The radar Doppler wind processing uses integers for station identification because + # `MetaData/stationIdentification` is mapped to the ODB variable `rad_ident`, + # which is an integer. + var = f.createVariable('MetaData/stationIdentification', 'i', ('Location')) + var[:] = [1, 2, 3, 4] + else: + # In all other cases, `MetaData/stationIdentification` is a string. + var = f.createVariable('MetaData/stationIdentification', str, ('Location')) + for i, s in enumerate(["station_1", "station_2", "station_3", "station_4"]): + var[i] = s + + # Extra variables for radar + if radar_doppler_wind: + var = f.createVariable('MetaData/radar_family', 'i', ('Location',)) + var[:] = [11, 12, 13, 14] + var = f.createVariable('MetaData/beamTiltAngle', 'i', ('Location',)) + var[:] = [11, 12, 13, 14] + var = f.createVariable('MetaData/gateRange', 'i', ('Location',)) + var[:] = [11, 12, 13, 14] + var = f.createVariable('MetaData/beamAzimuthAngle', 'i', ('Location',)) + var[:] = [11, 12, 13, 14] + var = f.createVariable('MetaData/stationElevation', 'i', ('Location',)) + var[:] = [11, 12, 13, 14] var = f.createVariable('ObsValue/' + var_name, 'f', ('Location',)) obsVal = [1.1, missing_float, 1.3, 1.4] @@ -64,6 +86,7 @@ def output_1d_simulated_var_to_netcdf(var_name, file_name, with_bias=False): if biasVal[i] == missing_float: biascorr[i] = obsVal[i] var[:] = biascorr + f.date_time = 2018010100 f.close() @@ -695,7 +718,7 @@ def copy_var_to_var(Group, invarname, outvarname, filename): output_2d_simulated_var_to_netcdf('probability', 'testinput/053_VarField_awpriorpcorrect.nc4') output_2d_normal_var_to_netcdf ('emissivity', 'OneDVar', 'testinput/057_VarField_emissivity.nc4', use_chans=True) # 54 VarField_NumChans and 55 VarField_ChanNum: separate files not necessary - output_2d_normal_var_to_netcdf ('radarAzimuth', 'MetaData', 'testinput/066_VarField_radarobazim.nc4', with_radar_family=True) + output_1d_simulated_var_to_netcdf ('radialVelocity', 'testinput/063_VarField_radialVelocity.nc4', radar_doppler_wind=True) output_2d_normal_var_to_netcdf ('liquidWaterContent', 'OneDVar', 'testinput/068_VarField_clw.nc4', use_levs=True) output_2d_normal_var_to_netcdf ('brightnessTemperature', ['constant_satid_5Predictor', 'constant_satid_8Predictor', 'thickness_850_300hPa_satid_5Predictor','thickness_850_300hPa_satid_8Predictor', @@ -845,6 +868,21 @@ def copy_var_to_var(Group, invarname, outvarname, filename): copy_var_to_var('ObsValue', 'potentialTemperature', 'airTemperature', 'testinput/varobs_globalnamelist_aircraft.nc4') copy_var_to_var('ObsError', 'potentialTemperature', 'airTemperature', 'testinput/varobs_globalnamelist_aircraft.nc4') + # Radar doppler wind - UKV + output_full_varobs_to_netcdf(['MetaData/latitude', + 'MetaData/longitude', + 'MetaData/beamTiltAngle', + 'MetaData/gateRange', + 'MetaData/beamAzimuthAngle', + 'MetaData/stationElevation', + 'ObsValue/radialVelocity', 'ObsError/radialVelocity'], + [], + ['MetaData/stationIdentification'], + 'testinput/varobs_ukvnamelist_radar_doppler_wind.nc4') + + + + # Cx output_1d_simulated_var_to_netcdf('dummy', 'testinput/dummy.nc4') output_1d_geoval_to_netcdf ('surface_altitude', 'testinput/001_SurfaceCxField_Orog.nc4') @@ -1086,3 +1124,9 @@ def copy_var_to_var(Group, invarname, outvarname, filename): 'uwind_at_10m', 'vwind_at_10m'], ['air_pressure_levels'], 'testinput/cx_globalnamelist_oceanwinds.nc4') + + # Radar doppler wind - UKV + output_full_cx_to_netcdf(['surface_altitude'], + ['eastward_wind', 'northward_wind', 'upward_air_velocity'], + 'testinput/cx_ukvnamelist_radar_doppler_wind.nc4') + diff --git a/test/testinput/001_VarField_pstar_stationID_integer.nc4 b/test/testinput/001_VarField_pstar_stationID_integer.nc4 index cdc81217..03038252 100644 Binary files a/test/testinput/001_VarField_pstar_stationID_integer.nc4 and b/test/testinput/001_VarField_pstar_stationID_integer.nc4 differ diff --git a/test/testinput/063_VarField_radialVelocity.nc4 b/test/testinput/063_VarField_radialVelocity.nc4 new file mode 100644 index 00000000..72a20cfd Binary files /dev/null and b/test/testinput/063_VarField_radialVelocity.nc4 differ diff --git a/test/testinput/063_VarField_radialVelocity.yaml b/test/testinput/063_VarField_radialVelocity.yaml new file mode 100644 index 00000000..1d98369f --- /dev/null +++ b/test/testinput/063_VarField_radialVelocity.yaml @@ -0,0 +1,44 @@ +time window: + begin: 2018-01-01T00:00:00Z + end: 2018-01-01T02:00:00Z + +observations: + - obs space: + name: Radar + obsdatain: + engine: + type: H5File + obsfile: Data/063_VarField_radialVelocity.nc4 + simulated variables: [radialVelocity] + obs filters: + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + # Reject observation 3: we want to check if it is omitted from the VarObs file, as expected. + - filter: Domain Check + where: + - variable: + name: MetaData/latitude + minvalue: 0.0 + - filter: VarObs Writer + reject_obs_with_any_variable_failing_qc: true + namelist_directory: testinput/VarObsWriterNamelists_021_VarField_surface + general_mode: debug + station_ID_int_to_string: true + - filter: VarObs Checker + expected_main_table_columns: + # Only observations 1, 2 and 4 are passed; observation 3 is rejected by the domain check + field: ["63","64","65","66","69","75","63","64","65","66","69","75","63","64","65","66","69","75"] + ob value: ["1.10000","11.00000","11.00000","11.00000","1.00000","11.00000","-1073741824.00000", + "12.00000","12.00000","12.00000","2.00000","12.00000","1.40000","14.00000","14.00000","14.00000","4.00000","14.00000"] + lat: ["21.00000","21.00000","21.00000","21.00000","21.00000","21.00000","22.00000","22.00000", + "22.00000","22.00000","22.00000","22.00000","24.00000","24.00000","24.00000","24.00000","24.00000","24.00000"] + lon: ["31.00000","31.00000","31.00000","31.00000","31.00000","31.00000","32.00000","32.00000", + "32.00000","32.00000","32.00000","32.00000","34.00000","34.00000","34.00000","34.00000","34.00000","34.00000"] + time: ["-3540.00000","-3540.00000","-3540.00000","-3540.00000","-3540.00000","-3540.00000","-3480.00000","-3480.00000", + "-3480.00000","-3480.00000","-3480.00000","-3480.00000","-3360.00000","-3360.00000","-3360.00000","-3360.00000","-3360.00000","-3360.00000"] + Callsign: ["1","1","1","1","1","1","2","2","2","2","2","2","4","4","4","4","4","4"] + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 diff --git a/test/testinput/066_VarField_radarobazim.yaml b/test/testinput/066_VarField_radarobazim.yaml deleted file mode 100644 index 0a0a3a3c..00000000 --- a/test/testinput/066_VarField_radarobazim.yaml +++ /dev/null @@ -1,69 +0,0 @@ -time window: - begin: 2018-01-01T00:00:00Z - end: 2018-01-01T02:00:00Z - -observations: - - obs space: - name: Radar - obsdatain: - engine: - type: H5File - obsfile: Data/066_VarField_radarobazim.nc4 - simulated variables: [dummy] - channels: 1, 3 - obs filters: - # Double all observation errors: we want to check if error changes made by filters are - # propagated to VarObs files - - filter: BlackList - action: - name: inflate error - inflation factor: 2.0 - # Set the flag of observations with missing values to "pass": we want to check if these - # values are encoded correctly in the VarObsFile. - - filter: Reset Flags to Pass - flags_to_reset: [10, 15] # missing, Hfailed - # Reject observation 3: we want to check if it is omitted from the VarObs file, as expected. - - filter: Domain Check - where: - - variable: - name: MetaData/latitude - minvalue: 0.0 - - filter: VarObs Writer - reject_obs_with_any_variable_failing_qc: true - use_radar_family: true - general_mode: debug - - filter: VarObs Checker - expected_main_table_columns: - # Only observations 1, 2 and 4 are passed; observation 3 is rejected by the domain check. - # Only levels 1 and 3 are simulated and hence written to the VarObs file. - # In the arrays below, rows denote locations and columns levels. - field: ["66", "66", - "66", "66", - "66", "66"] - level: ["1", "2", - "1", "2", - "1", "2"] - ob value: ["4.10000", "6.10000", - "-1073741824.00000", "6.20000", - "4.40000", "6.40000"] - ob error: ["-1073741824.00000", "-1073741824.00000", - "-1073741824.00000", "-1073741824.00000", - "-1073741824.00000", "-1073741824.00000"] - lat: ["21.00000", "21.00000", - "22.00000", "22.00000", - "24.00000", "24.00000"] - lon: ["31.00000", "31.00000", - "32.00000", "32.00000", - "34.00000", "34.00000"] - time: ["-3540.00000", "-3540.00000", - "-3480.00000", "-3480.00000", - "-3360.00000", "-3360.00000"] - family: ["11", "11", - "12", "12", - "14", "14"] - Callsign: ["station_1", "station_1", - "station_2", "station_2", - "station_4", "station_4"] - HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. - benchmarkFlag: 1000 # just to keep the ObsFilters test happy - flaggedBenchmark: 0 diff --git a/test/testinput/VarObsWriterNamelists_063_VarField_radialVelocity/Radar.nl b/test/testinput/VarObsWriterNamelists_063_VarField_radialVelocity/Radar.nl new file mode 100644 index 00000000..a7990a74 --- /dev/null +++ b/test/testinput/VarObsWriterNamelists_063_VarField_radialVelocity/Radar.nl @@ -0,0 +1,3 @@ +&VarobsControlNL +Varfields=63,64,65,66,69,75 +/ diff --git a/test/testinput/cx_ukvnamelist_radar_doppler_wind.nc4 b/test/testinput/cx_ukvnamelist_radar_doppler_wind.nc4 new file mode 100644 index 00000000..2460718b Binary files /dev/null and b/test/testinput/cx_ukvnamelist_radar_doppler_wind.nc4 differ diff --git a/test/testinput/cxwriter_ukvnamelist_radar_doppler_wind.yaml b/test/testinput/cxwriter_ukvnamelist_radar_doppler_wind.yaml new file mode 100644 index 00000000..6fa2e780 --- /dev/null +++ b/test/testinput/cxwriter_ukvnamelist_radar_doppler_wind.yaml @@ -0,0 +1,41 @@ +time window: + begin: 2018-01-01T00:00:00Z + end: 2018-01-01T01:00:00Z + +observations: + - obs space: + name: Radar + obsdatain: + engine: + type: H5File + obsfile: Data/dummy.nc4 + simulated variables: [dummy] + geovals: + filename: Data/cx_ukvnamelist_radar_doppler_wind.nc4 + obs filters: + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the Cx file. + - filter: Reset Flags to Pass + flags_to_reset: [10] # missing + # Reject observation 3: we want to check if it is omitted from the Cx file, as expected. + - filter: Domain Check + where: + - variable: + name: MetaData/latitude + minvalue: 0.0 + - filter: Cx Writer + namelist_directory: ../etc/ukv/cx + reject_obs_with_any_variable_failing_qc: true + general_mode: debug + geovals_are_top_to_bottom: false + - filter: Cx Checker + expected_surface_variables: ["1"] + expected_upper_air_variables: ["3","4","32"] + expected_main_table_columns: + - # observation 3 is rejected by the tests above hence only 3 (1,2,4) columns + - ["7.10","1.30","1.20","1.10","11.30","11.20","11.10","21.30","21.20","21.10"] + - ["**********","2.30","**********","2.10","12.30","**********","12.10","22.30","**********","22.10"] + - ["7.40","4.30","4.20","4.10","14.30","14.20","14.10","24.30","24.20","24.10"] + HofX: ObsValue # just a placeholder -- not used""but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 diff --git a/test/testinput/varobs_ukvnamelist_radar_doppler_wind.nc4 b/test/testinput/varobs_ukvnamelist_radar_doppler_wind.nc4 new file mode 100644 index 00000000..cb9df160 Binary files /dev/null and b/test/testinput/varobs_ukvnamelist_radar_doppler_wind.nc4 differ diff --git a/test/testinput/varobswriter_ukvnamelist_radar_doppler_wind.yaml b/test/testinput/varobswriter_ukvnamelist_radar_doppler_wind.yaml new file mode 100644 index 00000000..8dc2d502 --- /dev/null +++ b/test/testinput/varobswriter_ukvnamelist_radar_doppler_wind.yaml @@ -0,0 +1,47 @@ +time window: + begin: 2018-01-01T00:00:00Z + end: 2018-01-01T02:00:00Z + +observations: + - obs space: + name: Radar + obsdatain: + engine: + type: H5File + obsfile: Data/varobs_ukvnamelist_radar_doppler_wind.nc4 + simulated variables: [radialVelocity] + observed variables: [radialVelocity] + obs filters: + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10] # missing + # Blacklist all missing entries in the original profiles (i.e. observation 2) + # This must be run after the Reset Flags to Pass filter, + # which sets the flags of any missing values in the original profiles + # to 'pass'. Therefore this filter is run with the 'defer to post' option set to true. + - filter: BlackList + where: + - variable: + name: MetaData/latitude + value: is_not_valid + defer to post: true + - filter: VarObs Writer + namelist_directory: ../etc/ukv/varobs + general_mode: debug + IC_PLevels: 1 + reject_obs_with_all_variables_failing_qc: true + station_ID_int_to_string: true + - filter: VarObs Checker + expected_main_table_columns: + # Rows are locations, columns are filter variables + # The blacklist prevents the missing ob from being written out + field: [63,64,65,66,69,75, + 63,64,65,66,69,75, + 63,64,65,66,69,75] + ob value: ["67.10000","27.10000","37.10000","47.10000","3.00000","57.10000","67.30000","27.30000","37.30000","47.30000","7.00000","57.30000","67.40000","27.40000","37.40000","47.40000","9.00000","57.40000"] + lat: ["7.10000","7.10000","7.10000","7.10000","7.10000","7.10000","7.30000","7.30000","7.30000","7.30000","7.30000","7.30000","7.40000","7.40000","7.40000","7.40000","7.40000","7.40000"] + lon: ["17.10000","17.10000","17.10000","17.10000","17.10000","17.10000","17.30000","17.30000","17.30000","17.30000","17.30000","17.30000","17.40000","17.40000","17.40000","17.40000","17.40000","17.40000"] + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0