From bcd0f0317e03b5c7990138d1635f4580e6ff1fd9 Mon Sep 17 00:00:00 2001 From: Chris Thomas <32307951+ctgh@users.noreply.github.com> Date: Mon, 20 May 2024 14:21:09 +0100 Subject: [PATCH] Account for integer `MetaData/stationIdentification` (#206) * integer MetaData/stationIdentification * remove output * add new files --- src/opsinputs/VarObsWriterParameters.h | 3 ++ src/opsinputs/opsinputs_fill_mod.F90 | 44 +++++++++++++++--- src/opsinputs/opsinputs_varobswriter_mod.F90 | 6 ++- test/CMakeLists.txt | 3 ++ test/generate_unittest_netcdfs.py | 25 ++++++++++ .../001_VarField_pstar_stationID_integer.nc4 | Bin 0 -> 10072 bytes .../001_VarField_pstar_stationID_integer.yaml | 27 +++++++++++ 7 files changed, 100 insertions(+), 8 deletions(-) create mode 100644 test/testinput/001_VarField_pstar_stationID_integer.nc4 create mode 100644 test/testinput/001_VarField_pstar_stationID_integer.yaml diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 0e6b9926..3020f63d 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -76,6 +76,9 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// length of each output profile is set to the length of the profiles in the ObsSpace. oops::Parameter varobsLengthIsIC_PLevels{"varobs_length_is_IC_PLevels", false, this}; + /// Set to true if MetaData/stationIdentification is an integer instead of a string. + oops::Parameter StationIDIntToString{"station_ID_int_to_string", false, this}; + /// Update OPS flag to output the varbc predictors oops::Parameter outputVarBCPredictors{"output_varbc_predictors", false, this}; diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 49bef460..9f544501 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1837,12 +1837,14 @@ end subroutine opsinputs_fill_fillinteger !> Name of the JEDI variable used to populate \p String1. !> \param[in] JediGroup !> Group of the JEDI variable used to populate \p String1. -!> +!> \param[in] ConvertIntToSTring +!> Convert an integer-valued ObsSpace vector to a string. !> \note This function returns early (without a warning) if the specified JEDI variable is not found. !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield !> are not found. subroutine opsinputs_fill_fillstring( & - Hdr, OpsVarName, JediToOpsLayoutMapping, StringLen, String1, ObsSpace, JediVarName, JediVarGroup) + Hdr, OpsVarName, JediToOpsLayoutMapping, StringLen, String1, ObsSpace, JediVarName, JediVarGroup, & + ConvertIntToSTring) implicit none ! Subroutine arguments: @@ -1854,17 +1856,33 @@ subroutine opsinputs_fill_fillstring( & type(c_ptr), value, intent(in) :: ObsSpace character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup +logical, optional, intent(in) :: ConvertIntToString ! Local declarations: character(len=StringLen) :: VarValue(JediToOpsLayoutMapping % NumJediObs) +integer(integer64) :: IntVarValue(JediToOpsLayoutMapping % NumJediObs) integer :: i +logical :: IntToString +character(len=20) :: IntAsString ! Body: + + if (obsspace_has(ObsSpace, JediVarGroup, JediVarName)) then + if (present(ConvertIntToString)) then + IntToString = ConvertIntToString + else + IntToString = .false. + end if + ! Retrieve data from JEDI - call opsinputs_obsspace_get_db_string(ObsSpace, JediVarGroup, JediVarName, & - int(StringLen, kind=c_int), VarValue) + if (IntToString) then + call obsspace_get_db(ObsSpace, JediVarGroup, JediVarName, IntVarValue) + else + call opsinputs_obsspace_get_db_string(ObsSpace, JediVarGroup, JediVarName, & + int(StringLen, kind=c_int), VarValue) + end if ! Fill the OPS data structures call Ops_Alloc(Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, String1) @@ -1872,11 +1890,23 @@ subroutine opsinputs_fill_fillstring( & if (JediToOpsLayoutMapping % ConvertRecordsToMultilevelObs) then if (JediToOpsLayoutMapping % RecordStarts(i + 1) > JediToOpsLayoutMapping % RecordStarts(i)) then ! This record is non-empty. Use the first location from that record. - String1(i) = VarValue(JediToOpsLayoutMapping % LocationsOrderedByRecord( & - JediToOpsLayoutMapping % RecordStarts(i))) + if (IntToString) then + write(IntAsString,"(I0)") & + IntVarValue(JediToOpsLayoutMapping % LocationsOrderedByRecord( & + JediToOpsLayoutMapping % RecordStarts(i))) + String1(i) = IntAsString + else + String1(i) = VarValue(JediToOpsLayoutMapping % LocationsOrderedByRecord( & + JediToOpsLayoutMapping % RecordStarts(i))) + end if end if else - String1(i) = VarValue(i) + if (IntToString) then + write(IntAsString,"(I0)") IntVarValue(i) + String1(i) = IntAsString + else + String1(i) = VarValue(i) + end if end if end do end if diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index c112d709..fd5b6593 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -155,6 +155,7 @@ module opsinputs_varobswriter_mod logical :: RequireTForTheta logical :: FillObsTypeFromOpsSubType logical :: VarobsLengthIsIC_PLevels + logical :: StationIDIntToString character(len=100) :: latitudeName character(len=100) :: longitudeName @@ -313,6 +314,8 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe call f_conf % get_or_die("varobs_length_is_IC_PLevels", self % VarobsLengthIsIC_PLevels) +call f_conf % get_or_die("station_ID_int_to_string", self % StationIDIntToString) + call f_conf % get_or_die("size_of_varobs_array", self % size_of_varobs_array) call f_conf % get_or_die("compress_var_channels", self % compressVarChannels) @@ -738,7 +741,8 @@ subroutine opsinputs_varobswriter_populateobservations( & if (obsspace_has(ObsSpace, "MetaData", "stationIdentification")) then call opsinputs_fill_fillstring(Ob % Header % Callsign, "Callsign", JediToOpsLayoutMapping, & - LenCallSign, Ob % Callsign, ObsSpace, "stationIdentification", "MetaData") + LenCallSign, Ob % Callsign, ObsSpace, "stationIdentification", "MetaData", & + self % StationIDIntToString) else if (obsspace_has(ObsSpace, "MetaData", "satelliteIdentifier")) then call opsinputs_varobswriter_fillsatid(Ob, ObsSpace, JediToOpsLayoutMapping) call Ops_Alloc(Ob % Header % Callsign, "Callsign", JediToOpsLayoutMapping % NumOpsObs, Ob % Callsign) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a85ce1f1..4fa7d9d7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -106,6 +106,9 @@ set_tests_properties(test_opsinputs_varobswriter_setup PROPERTIES FIXTURES_SETUP ADD_WRITER_TEST(NAME varobswriter_001_VarField_pstar YAML 001_VarField_pstar.yaml DATA 001_VarField_pstar.nc4) +ADD_WRITER_TEST(NAME varobswriter_001_VarField_pstar_stationID_integer + YAML 001_VarField_pstar_stationID_integer.yaml + DATA 001_VarField_pstar_stationID_integer.nc4) ADD_WRITER_TEST(NAME varobswriter_002_VarField_temperature_Surface YAML 002_VarField_temperature_Surface.yaml DATA 002_VarField_temperature_Surface.nc4) diff --git a/test/generate_unittest_netcdfs.py b/test/generate_unittest_netcdfs.py index ecb3d207..303e69b7 100644 --- a/test/generate_unittest_netcdfs.py +++ b/test/generate_unittest_netcdfs.py @@ -69,6 +69,30 @@ def output_1d_simulated_var_to_netcdf(var_name, file_name, with_bias=False): f.close() +def output_1d_simulated_var_to_netcdf_stationID_integer(var_name, file_name): + f = nc4.Dataset(file_name, 'w', format="NETCDF4") + + nlocs = 4 + f.createDimension('Location', nlocs) + + var = f.createVariable('MetaData/latitude', 'f', ('Location',)) + var[:] = [21, 22, -23, 24] + var = f.createVariable('MetaData/longitude', 'f', ('Location',)) + var[:] = [31, 32, 33, 34] + var = f.createVariable('MetaData/time', 'f', ('Location',)) + minute = 1 / 60. + var[:] = [1 * minute, 2 * minute, 3 * minute, 4 * minute] + var = f.createVariable('MetaData/stationIdentification', 'i', ('Location',)) + var[:] = [1, 2, 3, 4] + var = f.createVariable('ObsValue/' + var_name, 'f', ('Location',)) + obsVal = [1.1, missing_float, 1.3, 1.4] + var[:] = obsVal + var = f.createVariable('ObsError/' + var_name, 'f', ('Location',)) + var[:] = [0.1, missing_float, 0.3, 0.4] + f.date_time = 2018010100 + + f.close() + def output_1d_simulated_vars_to_netcdf(var_name_1, var_name_2, file_name): f = nc4.Dataset(file_name, 'w', format="NETCDF4") @@ -640,6 +664,7 @@ def copy_var_to_var(Group, invarname, outvarname, filename): if __name__ == "__main__": # VarObs output_1d_simulated_var_to_netcdf('surfacePressure', 'testinput/001_VarField_pstar.nc4') # Surface + output_1d_simulated_var_to_netcdf_stationID_integer('surfacePressure', 'testinput/001_VarField_pstar_stationID_integer.nc4') output_1d_simulated_var_to_netcdf('airTemperatureAt2M', 'testinput/002_VarField_temperature_Surface.nc4') output_2d_simulated_var_to_netcdf('airTemperature', 'testinput/002_VarField_temperature_RadarZ.nc4') output_1d_simulated_var_to_netcdf('relativeHumidityAt2M', 'testinput/003_VarField_rh_Surface.nc4') diff --git a/test/testinput/001_VarField_pstar_stationID_integer.nc4 b/test/testinput/001_VarField_pstar_stationID_integer.nc4 new file mode 100644 index 0000000000000000000000000000000000000000..03038252af23786944c23ed6431905b224310075 GIT binary patch literal 10072 zcmeHNZ)jUp6hFyZv$yzWI@j2#lf70pMQ7-`IwqR6S(2{Z(xfC~3_loOn-}*W%bPE+ zRfcX35n3$xdHP|B%s)6pa36#U>Oh##A|m?DevtWTqB2FqKh$&Hz4xWf*2<&-3HOk^ z^WMGZo_p@^oOko*_ojzZgB!zlhJ&gKLKe9ky_dE0@Q&NhY=0*=l*$LSi@|QMO4A4zY%pM0?#Jx)o@&5`@M%#5HsA7N4UaT!;|i zt9Nht`agyrx{JI8d{=74a4mPDWL^uhX+m3>fwx#Z-B7WsPkI)A@Ij$YX2A!*I@daG ze$3a)LXUFOJVkGU^4{-a#$E=!;FGH`IZZb8fmk^O^&9bZd*o3QBZMWVvX*iZK zRxx9`kUpS*g6RKN&?ul{)^x{9<-U}4+$upkV#hGiz2I<8luSo{2g9)Q-!as0fMJ-> zRqkh=e|}cIjl3!W*g~2?L0X`>a-%;p+svg z*1TOEE}2!=DqG{;)g|5QgwVndv)-u28)d|Fjg;XUKQTKHcapdb)ZAF@h*6m^&vi44 zbNTa6aa*oVT|D^(`PS0ZK>79xu$WDbI`#?EaV-Rk3XwdJ-$~PPLI3>R zT%4|7IeT^=p6^2wDG~vRfJ8tdAQ6xV{Lc{}%15x8KC#YY)@F#YS0DmCL^kHrgXw%a zn@kryvT+Z7Eg%vB`5mKXx^>pkr*%Q4exH>^E>f@uLJK-wfYeC&2)0d!&9@co%&IZFx8CX(b)r6Lf0N_(h45GhsWH+6}*`Jv!@yKV|WH9#vV(4M)(ALf`C$b4g!b?0zBf=I?*pcB&akA8n1K6U%$`wO@W)HR&G_ YYrK|87u>SYYn^m|KjcO5`W1cgAE3Lq`v3p{ literal 0 HcmV?d00001 diff --git a/test/testinput/001_VarField_pstar_stationID_integer.yaml b/test/testinput/001_VarField_pstar_stationID_integer.yaml new file mode 100644 index 00000000..30e798d7 --- /dev/null +++ b/test/testinput/001_VarField_pstar_stationID_integer.yaml @@ -0,0 +1,27 @@ +time window: + begin: 2018-01-01T00:00:00Z + end: 2018-01-01T02:00:00Z + +observations: + - obs space: + name: Surface + obsdatain: + engine: + type: H5File + obsfile: Data/001_VarField_pstar_stationID_integer.nc4 + simulated variables: [surfacePressure] + obs filters: + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + - filter: VarObs Writer + # Convert integer station ID to string. + station_ID_int_to_string: true + general_mode: debug + - filter: VarObs Checker + expected_main_table_columns: + Callsign: ["1", "2", "3", "4"] + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0