From cf664d5c5b3e8fbf29ea6f7a44fb1f90593562eb Mon Sep 17 00:00:00 2001 From: Owen Lewis <65953787+orlewis@users.noreply.github.com> Date: Thu, 7 Dec 2023 10:24:51 +0000 Subject: [PATCH] Map channels to Var Channels (#181) * Trying combining offset with use_actual_channels * A commit * Commit some changes * A push for testing purposes. * Some more changes, bu failing at call to fillchan * Strip out the offset channels and use_actual_channels. * This is an example of a setup which works. I think the Ob % ChanNum = ChannelIndices might have been in the wrong place. * This is the setup that I think is required but this crashes at line 1151 in opsinputs_varobswriter_mod.F90 when going to the opsinputs_varobswriter_fillchannumandandnumchans function. * Change output dir to mine (plevens) and remove / at end of output_directory line * Change path of another file to my own directory due to permission problems * remove output_directory line * Change to CallSign in yaml * Change to fields in yaml * Changes to lat, lon, and time for ctest * Change level field in ctest yaml * Update ob value field in ctest yaml * Add size_of_varobs_array back to opsinputs_varobswriter_mod.f90 and add it as an explicit entry in opsinputs_varobswriter type instead of being inside the channel_offset type (which was removed) * This is a bit messy and may need to be reverted but handy to see what changes have been made. * A por version which has just reintroduced previous options under new names. Not really what we want but want to save a copy. * All the ctests are now passing. * Change to the logic to work for var channels not compressed. * Creating an option to increase the size of the channel array to the size of the varobs array. * remove some of the write statements * Some tidy up, removing write statements etc. * Removing outdated comments * Restore develop versions, nothing should have changed with these files. * Adding local variables to handle optional better. Dealing with review comments. * Remove own drietories and correct typo * re include emissivity * Missed brackets off if statement * Fix missing brackets for if statements, remove extra variables from the brightness temp ctest. * MCC changes * Switch compress_var_channels to be default true * Fix if statement logic * More changes to the if logic so that the ctests pass. * Remove merge issues * Return emissivity setting to original. * empty commit * Another edit to the if statement logic. * Improve logic * Update descriptions for new options. * Change sizeVarObs to sizeOfVarobsArray * Remove the emissivity part from https://rom-saf.eumetsat.int/romsaf_ropp_ug_1dvar.pdf, change name of sizeVarObs and add comments. * Update to use IMDI --------- Co-authored-by: PJLevensMO Co-authored-by: mikecooke77 Co-authored-by: Michael Cooke <48374999+mikecooke77@users.noreply.github.com> --- src/opsinputs/VarObsWriter.cc | 8 + src/opsinputs/VarObsWriter.h | 3 + src/opsinputs/VarObsWriterParameters.h | 22 +- src/opsinputs/opsinputs_fill_mod.F90 | 133 +++++++---- src/opsinputs/opsinputs_varobswriter_mod.F90 | 177 ++++++++++----- test/testinput/010_VarField_britemp.yaml | 214 +++++++++++++++++- test/testinput/055_VarField_channum.yaml | 3 +- test/testinput/057_VarField_emissivity.yaml | 2 +- .../076_VarField_britempvarerror.yaml | 2 +- .../varobswriter_globalnamelist_atovs.yaml | 2 +- 10 files changed, 453 insertions(+), 113 deletions(-) diff --git a/src/opsinputs/VarObsWriter.cc b/src/opsinputs/VarObsWriter.cc index 51c790f2..69e350e2 100644 --- a/src/opsinputs/VarObsWriter.cc +++ b/src/opsinputs/VarObsWriter.cc @@ -17,6 +17,7 @@ #include "ioda/ObsVector.h" #include "oops/base/Variables.h" #include "oops/mpi/mpi.h" +#include "oops/util/IntSetParser.h" #include "oops/util/Logger.h" #include "opsinputs/LocalEnvironment.h" #include "opsinputs/VarObsWriterParameters.h" @@ -70,6 +71,13 @@ VarObsWriter::VarObsWriter(ioda::ObsSpace & obsdb, const Parameters_ & params, const int fallbackChannels = 0; // Avoid passing a null pointer to Fortran. const int *channelsData = channels.empty() ? &fallbackChannels : channels.data(); + + // Want to also set up an ordered list of channels numbered for VAR. + std::set varchanset = oops::parseIntSet(parameters_.varChannels.value()); + varchannels_.assign(varchanset.begin(), varchanset.end()); + conf.set("varChannels", varchannels_); + conf.set("NumVarChannels", sizeof(varchannels_)); + if (!opsinputs_varobswriter_create_f90(key_, &conf, fortranMpiCommunicatorIsValid, fortranMpiCommunicator, diff --git a/src/opsinputs/VarObsWriter.h b/src/opsinputs/VarObsWriter.h index 80fe226f..585dbb42 100644 --- a/src/opsinputs/VarObsWriter.h +++ b/src/opsinputs/VarObsWriter.h @@ -8,7 +8,9 @@ #include #include +#include #include +#include #include "ioda/ObsDataVector.h" #include "oops/base/Variables.h" @@ -87,6 +89,7 @@ class VarObsWriter : public oops::interface::ObsFilterBase, oops::Variables extradiagvars_; std::shared_ptr> flags_; std::shared_ptr> obsErrors_; + std::vector varchannels_; VarObsWriterParameters parameters_; }; diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 631e79e5..0e6b9926 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -79,21 +79,25 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// Update OPS flag to output the varbc predictors oops::Parameter outputVarBCPredictors{"output_varbc_predictors", false, this}; - /// This contains the offset that needs to be added to the channel number in order to - /// index the output arrays correctly. - oops::Parameter channel_offset{"channel_offset", 0, this}; - /// This is the size of the varobs array for output. The default is zero and the size /// of the array will be used. /// For atovs, jedi has 20 brightness temperatures but var expects 40. /// Therefore for atovs brightness_tmperatuere => size_of_varobs_array = 40. oops::Parameter size_of_varobs_array{"size_of_varobs_array", 0, this}; - /// This matches the channel number with the array index. This is useful when - /// there are skipped channels. e.g. channels 5,6,7,9,10,11 are required from a possible 12 - /// channels. This would fill a size 12 array with the array indexes matching the channel number - /// [NaN,NaN,NaN,Nan,5,6,7,Nan,9,10,11,NaN]. - oops::Parameter use_actual_channels{"use_actual_channels", false, this}; + /// List of channels which are expected for Var. This enables mapping of JOPA + /// channel numbers to VAR numbers. Default is empty and the JOPA channels will be used. + oops::Parameter varChannels{"varChannels", "", this}; + + /// If set to false (default true) this matches the channel number with the array index. + /// This is useful when there are skipped channels. e.g. channels 5,6,7,9,10,11 are + /// required from a possible 12 channels. This would fill a size 12 array with the array + /// indexes matching the channel number [NaN,NaN,NaN,Nan,5,6,7,Nan,9,10,11,NaN]. + /// Compressed example could be [2,8,50,100], no missing values between channel numbers. + oops::Parameter compressVarChannels{"compress_var_channels", true, this}; + + /// Increase the channel array size to the same size as the varobs array + oops::Parameter increaseChanArray{"increase_chan_array", false, this}; /// If this list of ufo::variable is defined in the yaml a subset of the flags /// will be made with just these variables present. This will allow Fortran calls such-as diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index c9befbfb..2899218d 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1030,12 +1030,19 @@ end subroutine opsinputs_fill_fillreal !> of variables with suffixes corresponding to the indices specified in \p Channels. !> \param[in] JediGroup !> Group of the JEDI variable used to populate \p Real2. +!> \param[in] compressVarChannels +!> Whether to apply var channel compression (No NaN spaces between channels) +!> \param[in] sizeOfVarobsArray +!> The size of the varobs array which the output data will be stored in. +!> \param[in] varChannels +!> A list of the var channel numbers which the channels will be mapped to. !> !> \note This function returns early (without a warning) if the specified JEDI variable is not found. !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield !> are not found. subroutine opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, NumObs, Real2, ObsSpace, Channels, JediVarName, JediVarGroup, OffsetChans, useActualChans) + Hdr, OpsVarName, NumObs, Real2, ObsSpace, Channels, JediVarName, & + JediVarGroup, compressVarChannels, sizeOfVarobsArray, varChannels) implicit none ! Subroutine arguments: @@ -1047,8 +1054,10 @@ subroutine opsinputs_fill_fillreal2d_norecords( & integer(c_int), intent(in) :: Channels(:) character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup -type(opsinputs_channeloffset), optional, intent(in) :: OffsetChans -logical, optional, intent(in) :: useActualChans +logical, optional, intent(in) :: compressVarChannels +integer(integer64), optional, intent(in) :: sizeOfVarobsArray +integer(c_int), optional, intent(in) :: varChannels(:) + ! Local declarations: real(kind=c_double) :: VarValue(NumObs) @@ -1057,29 +1066,32 @@ subroutine opsinputs_fill_fillreal2d_norecords( & integer :: iChannel integer :: offset integer :: numchans -logical :: localUseActualChans +integer :: offsetsize +integer :: arrayindex +logical :: compressChannels ! Body: MissingDouble = missing_value(0.0_c_double) +compressChannels = .true. +if (present(compressVarChannels)) then + compressChannels = compressVarChannels +end if + JediVarNamesWithChannels = opsinputs_fill_varnames_with_channels(JediVarName, Channels) !take into account offsetting of 2nd dimension if required !designed to be used to pack where multiple satellite instruments expected !e.g. HIRS in ATOVS stream -offset = 0 -numchans = size(JediVarNamesWithChannels) -if (present(OffsetChans)) then - offset = OffsetChans % channel_offset - if (OffsetChans % size_of_varobs_array > 0) & - numchans = OffsetChans % size_of_varobs_array -end if -!Setup for channels needing to match array index -localUseActualChans = .false. -if (present(useActualChans)) then - localUseActualChans = useActualChans +numchans = size(JediVarNamesWithChannels) +!sizeOfVarobsArray comes from intitial setting of size_of_varobs_array +! used to define the size of the channel array to fill. +if (present(sizeOfVarobsArray)) then + if (sizeOfVarobsArray > 0) then + numchans = sizeOfVarobsArray + end if end if if (obsspace_has(ObsSpace, JediVarGroup, JediVarNamesWithChannels(1))) then @@ -1087,22 +1099,44 @@ subroutine opsinputs_fill_fillreal2d_norecords( & call Ops_Alloc(Hdr, OpsVarName, NumObs, Real2, & num_levels = int(numchans, kind=integer64)) do iChannel = 1, size(JediVarNamesWithChannels) - ! Retrieve data from JEDI call obsspace_get_db(ObsSpace, JediVarGroup, JediVarNamesWithChannels(iChannel), VarValue) - - ! Fill the OPS data structures - if (localUseActualChans) then - where (VarValue /= MissingDouble) - Real2(:, Channels(iChannel)) = VarValue - end where + arrayindex = iChannel + + ! if VAR channels have been assigned then jopa channels will be mapped to these var channels + ! Set up the size of the array, if channels are being pushed together an offset between + ! the var and jopa channel numbers is added onto the size of the array. + ! If not compressed the positions in the array are based on the actual channel number. + + if (present(varChannels)) then + if (size(varChannels) > 0) then + if (iChannel <= size(varChannels)) then + if (compressChannels) then + offsetsize = abs(varChannels(1) - channels(1)) + arrayindex = arrayindex + offsetsize + else + arrayindex = varChannels(iChannel) + end if + end if + else + if (.not. compressChannels) then + arrayindex = Channels(iChannel) + end if + end if else - ! Fill the OPS data structures - where (VarValue /= MissingDouble) - Real2(:, iChannel+offset) = VarValue - end where + if (present(sizeOfVarobsArray)) then + if (sizeOfVarobsArray > size(channels)) then + if (.not. compressChannels) then + arrayindex = Channels(iChannel) + end if + end if + end if ! the end end if + where (VarValue /= MissingDouble) + Real2(:, arrayindex) = VarValue + end where end do end if ! Data not present? OPS will produce a warning -- we don't need to duplicate it. + end subroutine opsinputs_fill_fillreal2d_norecords ! ------------------------------------------------------------------------------ @@ -1223,8 +1257,8 @@ end subroutine opsinputs_fill_fillreal2d_records !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield !> are not found. subroutine opsinputs_fill_fillreal2d( & - Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, Channels, VarobsLength, JediVarName, JediVarGroup, OffsetChans, & - useActualChans) + Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, Channels, & + VarobsLength, JediVarName, JediVarGroup, compressVarChannels, sizeOfVarobsArray, varChannels) implicit none ! Subroutine arguments: @@ -1237,28 +1271,44 @@ subroutine opsinputs_fill_fillreal2d( & integer(integer64), intent(in) :: VarobsLength character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup -type(opsinputs_channeloffset), optional, intent(in) :: OffsetChans -logical, optional, intent(in) :: useActualChans +logical, optional, intent(in) :: compressVarChannels +integer(integer64), optional, intent(in) :: sizeOfVarobsArray +integer(c_int), optional, intent(in) :: varChannels(:) + +! local variables +logical :: compressChannels +integer(integer64) :: sizeOfVarobsArray_local +integer(c_int), allocatable :: localvarChannels(:) ! Body: + +compressChannels = .true. +if (present(compressVarChannels)) then + compressChannels = compressVarChannels +end if + +if (present(varChannels)) then + allocate(localvarChannels(size(varChannels))) + localvarChannels = varChannels +end if + +sizeOfVarobsArray_local = 0 +if (present(sizeOfVarobsArray)) then + sizeOfVarobsArray_local = sizeOfVarobsArray +end if + if (JediToOpsLayoutMapping % ConvertRecordsToMultilevelObs) then call opsinputs_fill_fillreal2d_records( & Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, VarobsLength, JediVarName, JediVarGroup) else - if (Present(OffsetChans)) then - if (Present(useActualChans)) then - call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, OffsetChans, useActualChans) - else - call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, OffsetChans) - end if + if (allocated(localvarChannels)) then + call opsinputs_fill_fillreal2d_norecords( & + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & + JediVarName, JediVarGroup, compressChannels, sizeOfVarobsArray_local, varChannels) else call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup) + JediVarName, JediVarGroup, compressChannels, sizeOfVarobsArray_local) end if end if @@ -2386,6 +2436,7 @@ function opsinputs_fill_varnames_with_channels(VarName, Channels) result(VarName write (VarNames(ichan),'(A,"_",I0)') VarName, Channels(ichan) end do end if + end function opsinputs_fill_varnames_with_channels diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 0784a2ff..6a6c4907 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -187,15 +187,16 @@ module opsinputs_varobswriter_mod integer(integer64) :: VarobsLength integer(c_int), allocatable :: channels(:) + integer(c_int), allocatable :: varChannels(:) + integer(integer64) :: size_of_varobs_array + integer(integer64) :: NumVarChannels + + logical :: compressVarChannels + logical :: increaseChanArray !this stores the atmospheric levels we wish to pass to varobs integer(c_int), allocatable :: modlevs(:) - !this stores the offset for channels when storing britemp - type(opsinputs_channeloffset) :: channel_offset - - logical :: useActualChannels - type(ufo_geovals), pointer :: GeoVals type(ufo_geovals), pointer :: ObsDiags end type opsinputs_varobswriter @@ -236,6 +237,13 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe allocate(self % channels(size(channels))) self % channels(:) = channels(:) +call f_conf % get_or_die("NumVarChannels", self % NumVarChannels) +allocate(self % varChannels(self % NumVarChannels)) +call f_conf % get_or_die("varChannels", self % varChannels) + +call f_conf % get_or_die("compress_var_channels", self % compressVarChannels) + +call f_conf % get_or_die("increase_chan_array", self % increaseChanArray) ! Setup OPS call f_conf % get_or_die("general_mode", StringValue) @@ -305,25 +313,9 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe call f_conf % get_or_die("varobs_length_is_IC_PLevels", self % VarobsLengthIsIC_PLevels) -! This contains the offset that needs to be added to the channel number in order to -! index the output arrays correctly. -call f_conf % get_or_die("channel_offset", self % channel_offset % channel_offset) +call f_conf % get_or_die("size_of_varobs_array", self % size_of_varobs_array) -! This is the size of the varobs array for output. The default is zero and the size of the array will be used. -! For atovs, jedi has 20 brightness temperatures but var expects 40. -! Therefore for atovs brightness_tmperatuere => size_of_varobs_array = 40. -call f_conf % get_or_die("size_of_varobs_array", self % channel_offset % size_of_varobs_array) - -! This defines if to use the channel numbers as the indices to fill an array by -! This prevents channels which are seperated by NaNs from being bunched together in the write -call f_conf % get_or_die("use_actual_channels", self % useActualChannels) - -if ((self % useActualChannels) .and. (self % channel_offset % channel_offset/=0)) then - write (ErrorMessage, '(A)') "OffsetChans and UseActualChans cannot both be set" - call gen_warn(RoutineName, ErrorMessage) - opsinputs_varobswriter_create = .false. - return -end if +call f_conf % get_or_die("compress_var_channels", self % compressVarChannels) ! Updates the varbc flag passedaround by a module in OPS call f_conf % get_or_die("output_varbc_predictors", BoolValue) @@ -870,8 +862,8 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_britemp) call opsinputs_fill_fillreal2d( & Ob % Header % CorBriTemp, "CorBriTemp", JediToOpsLayoutMapping, Ob % CorBriTemp, & - ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue", self % channel_offset, & - self % useActualChannels) + ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue", & + self % compressVarChannels, self % size_of_varobs_array, self % varChannels) case (VarField_tskin) call opsinputs_fill_fillelementtypefromnormalvariable( & Ob % Header % Tskin, "Tskin", Ob % Header % NumObsLocal, Ob % Tskin, & @@ -1004,7 +996,8 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_Emissivity) call opsinputs_fill_fillreal2d( & Ob % Header % Emissivity, "Emissivity", JediToOpsLayoutMapping, Ob % Emissivity, & - ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar", self % channel_offset) + ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar", & + self % compressVarChannels, self % size_of_varobs_array, self % varChannels) case (VarField_QCinfo) ! TODO(someone): This will come from a variable generated by the 1D-Var filter. Its name and ! group are not known yet. Once they are, replace the placeholders in the call below. @@ -1103,7 +1096,8 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_BriTempVarError) call opsinputs_fill_fillreal2d( & Ob % Header % BriTempVarError, "BriTempVarError", JediToOpsLayoutMapping, Ob % BriTempVarError, & - ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature", "ObsErrorData", self % channel_offset) + ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature",& + "ObsErrorData", self % compressVarChannels, self % size_of_varobs_array, self % varChannels) case (VarField_CloudRTError) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % CloudRTError, "CloudRTError", Ob % Header % NumObsLocal, Ob % CloudRTError) @@ -1176,8 +1170,9 @@ subroutine opsinputs_varobswriter_populateobservations( & if (FillChanNum .or. FillNumChans) then call opsinputs_varobswriter_fillchannumandnumchans( & - Ob, ObsSpace, self % channels, Flags, FillChanNum, & - FillNumChans, self % channel_offset % channel_offset, self % useActualChannels) + Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & + FillNumChans, self % compressVarChannels, self % size_of_varobs_array, & + self % increaseChanArray) end if end do @@ -1222,54 +1217,122 @@ end subroutine opsinputs_varobswriter_fillreportflags !> e.g. for HIRS & AMSUA !> the number of these channels is stored in Ob % NumChans. subroutine opsinputs_varobswriter_fillchannumandnumchans( & - Ob, ObsSpace, Channels, Flags, FillChanNum, FillNumChans, & - OffsetChans,UseActualChans) + Ob, ObsSpace, channels, varChannels, Flags, FillChanNum, FillNumChans, compressVarChannels, & + varObsSize, increaseChanArray) + implicit none ! Subroutine arguments: type(OB_type), intent(inout) :: Ob type(c_ptr), value, intent(in) :: ObsSpace -integer(c_int), intent(in) :: Channels(:) +integer(c_int), intent(in) :: channels(:) +integer(c_int), intent(in) :: varChannels(:) type(c_ptr), value, intent(in) :: Flags logical, intent(in) :: FillChanNum, FillNumChans -integer, intent(in) :: OffsetChans -logical, optional, intent(in) :: UseActualChans +logical, optional, intent(in) :: compressVarChannels +integer(integer64), optional, intent(in) :: varObsSize +logical, optional, intent(in) :: increaseChanArray + ! Local declarations: -integer(integer64) :: NumChannels -integer(integer64) :: ChannelIndices(Ob % Header % NumObsLocal, size(Channels)) -integer(integer64) :: ChannelCounts(Ob % Header % NumObsLocal) -integer :: iChannel -integer :: iObs -logical :: localUseActualChans +integer(integer64) :: NumChannels +integer(integer64), allocatable :: ChannelIndicesVar(:,:) +integer(integer64) :: ChannelIndices(Ob % Header % NumObsLocal, size(channels)) +integer(integer64) :: ChannelCounts(Ob % Header % NumObsLocal) +integer :: iChannel +integer :: iObs +real(kind=c_double) :: MissingDouble +integer(integer64) :: array_loop +integer :: offsetsize +logical :: compressChannels +integer(integer64) :: varObsSize_loc +logical :: increaseChanArray_loc ! Body: -NumChannels = size(Channels) + +compressChannels = .true. +if (present(compressVarChannels)) then + compressChannels = compressVarChannels +end if + +varObsSize_loc = 0 +if (present(varObsSize)) then + varObsSize_loc = varObsSize +end if + +increaseChanArray_loc = .false. +if (present(increaseChanArray)) then + increaseChanArray_loc = increaseChanArray +end if + + +MissingDouble = missing_value(0.0_c_double) + +NumChannels = size(channels) if (NumChannels == 0) return -! Setup for useActualChans for array indices -localUseActualchans = .false. -if (Present(UseActualChans)) then - localUseActualChans = UseActualChans +if (increaseChanArray_loc) then + NumChannels = varObsSize_loc + allocate(ChannelIndicesVar(Ob % Header % NumObsLocal, varObsSize_loc)) +else + allocate(ChannelIndicesVar(Ob % Header % NumObsLocal, size(varChannels))) end if call opsinputs_varobswriter_findchannelspassingqc( & - Ob % Header % NumObsLocal, ObsSpace, Channels, Flags, ChannelIndices, ChannelCounts) + Ob % Header % NumObsLocal, ObsSpace, channels, Flags, ChannelIndices, ChannelCounts) + +ChannelIndicesVar(:,:) = 0 + if (FillChanNum) then call Ops_Alloc(Ob % Header % ChanNum, "ChanNum", Ob % Header % NumObsLocal, Ob % ChanNum, & num_levels = NumChannels) - - if (localUseActualChans) then - do iChannel=1, size(Channels) - ChannelIndices(:,iChannel) = Channels(iChannel) - end do - Ob % ChanNum = ChannelIndices + if ((varObsSize_loc /= 0)) then + if (varObsSize_loc > size(channels)) then + if (size(varChannels) > 0 .and. (size(channels) == size(varChannels))) then + if (compressChannels) then + offsetsize = abs(varChannels(1) - channels(1)) + Ob % ChanNum = ChannelIndices + where (Ob % ChanNum > 0) + Ob % ChanNum = Ob % ChanNum + int(offsetsize, kind=integer64) + end where + else + do iChannel=1, NumChannels + if (iChannel > size(varChannels)) then + ChannelIndicesVar(:, iChannel) = IMDI + else + ChannelIndicesVar(:, iChannel) = varChannels(iChannel) + end if + end do + Ob % ChanNum = ChannelIndicesVar + end if + else + if (compressChannels) then + Ob % ChanNum = ChannelIndices + else + do iChannel=1, size(Channels) + ChannelIndices(:,iChannel) = Channels(iChannel) + end do + Ob % ChanNum = ChannelIndices + end if + end if + else if (varObsSize_loc == size(channels)) then + if (size(varChannels) > 0) then + do iChannel=1, NumChannels + if (compressChannels) then + do iObs=1, Ob % Header % NumObsLocal + if (ChannelIndices(iObs,iChannel) /= 0) then + ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) + end if + end do + end if + end do + Ob % ChanNum = ChannelIndicesVar + else + Ob % ChanNum = ChannelIndices + end if + end if else - Ob % ChanNum = ChannelIndices - !only apply offset to actual channels in list, not missing data - where (Ob % ChanNum > 0) - Ob % ChanNum = Ob % ChanNum + int(OffsetChans, kind=integer64) - end where + Ob % ChanNum = ChannelIndices end if end if diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index cb4e3091..205506ca 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -59,7 +59,6 @@ observations: HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 - # # Now we test effect of actual channel # @@ -90,7 +89,7 @@ observations: namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp general_mode: debug size_of_varobs_array: 3 - use_actual_channels: true + compress_var_channels: false - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. @@ -125,3 +124,214 @@ observations: HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 +# +# Now we test effect varChannels and compress var channels +# + - obs space: + name: AMSUB + obsdatain: + engine: + type: H5File + obsfile: Data/010_VarField_britemp.nc4 + simulated variables: [brightnessTemperature] + channels: 1,3 + obs filters: + # Double all observation errors: we want to check if error changes made by filters are + # propagated to VarObs files + - filter: BlackList + action: + name: inflate error + inflation factor: 2.0 + # Reject the first level of the first observation and the last level of the last observation + - filter: Bounds Check + minvalue: 1.15 + maxvalue: 3.35 + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + - filter: VarObs Writer + variables_for_quality_control: + - name: brightnessTemperature + channels: 1,3 + namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp + general_mode: debug + size_of_varobs_array: 3 + varChannels: 2, 4 #[5,7] + - filter: VarObs Checker + expected_main_table_columns: + # In the arrays below, rows denote locations and columns levels. + field: ["10", "10", "10", + "10", "10", "10", + "10", "10", "10", + "10", "10", "10"] + level: ["1","2","3", + "1","2","3", + "1","2","3", + "1","2","3"] + ob value: ["-1073741824.00000","1.20000","3.11000", + "-1073741824.00000","-1073741824.00000","3.22000", + "-1073741824.00000","1.60000","3.33000", + "-1073741824.00000","1.80000", "3.44000"] + lat: ["21.00000", "21.00000", "21.00000", + "22.00000", "22.00000", "22.00000", + "-23.00000", "-23.00000", "-23.00000", + "24.00000", "24.00000", "24.00000"] + lon: ["31.00000", "31.00000", "31.00000", + "32.00000", "32.00000", "32.00000", + "33.00000", "33.00000", "33.00000", + "34.00000", "34.00000", "34.00000"] + time: ["-3540.00000", "-3540.00000", "-3540.00000", + "-3480.00000", "-3480.00000", "-3480.00000", + "-3420.00000", "-3420.00000", "-3420.00000", + "-3360.00000", "-3360.00000", "-3360.00000"] + Callsign: ["station_1", "station_1", "station_1", + "station_2", "station_2", "station_2", + "station_3", "station_3", "station_3", + "station_4", "station_4", "station_4"] + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 + + +# +# Now we test effect of var channels +# + - obs space: + name: AMSUB + obsdatain: + engine: + type: H5File + obsfile: Data/010_VarField_britemp.nc4 + simulated variables: [brightnessTemperature] + channels: 1,3 + obs filters: + # Double all observation errors: we want to check if error changes made by filters are + # propagated to VarObs files + - filter: BlackList + action: + name: inflate error + inflation factor: 2.0 + # Reject the first level of the first observation and the last level of the last observation + - filter: Bounds Check + minvalue: 1.15 + maxvalue: 3.35 + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + - filter: VarObs Writer + variables_for_quality_control: + - name: brightnessTemperature + channels: 1,3 + namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp + general_mode: debug + compress_var_channels: false + size_of_varobs_array: 4 + varChannels: 2, 4 #[5,7] + - filter: VarObs Checker + expected_main_table_columns: + # In the arrays below, rows denote locations and columns levels. + field: ["10", "10", "10", "10", + "10", "10", "10", "10", + "10", "10", "10", "10", + "10", "10", "10", "10"] + level: ["1","2","3","4", + "1","2","3","4", + "1","2","3","4", + "1","2","3","4"] + ob value: ["-1073741824.00000","1.20000","-1073741824.00000","3.11000", + "-1073741824.00000","-1073741824.00000","-1073741824.00000","3.22000", + "-1073741824.00000","1.60000","-1073741824.00000","3.33000", + "-1073741824.00000","1.80000", "-1073741824.00000","3.44000"] + lat: ["21.00000", "21.00000", "21.00000", "21.00000", + "22.00000", "22.00000", "22.00000", "22.00000", + "-23.00000", "-23.00000", "-23.00000","-23.00000", + "24.00000", "24.00000", "24.00000","24.00000"] + lon: ["31.00000","31.00000", "31.00000", "31.00000", + "32.00000", "32.00000", "32.00000","32.00000", + "33.00000", "33.00000","33.00000", "33.00000", + "34.00000", "34.00000", "34.00000","34.00000"] + time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000", + "-3480.00000", "-3480.00000", "-3480.00000","-3480.00000", + "-3420.00000", "-3420.00000", "-3420.00000","-3420.00000", + "-3360.00000", "-3360.00000", "-3360.00000","-3360.00000"] + Callsign: ["station_1", "station_1","station_1", "station_1", + "station_2", "station_2", "station_2","station_2", + "station_3", "station_3", "station_3","station_3", + "station_4", "station_4", "station_4","station_4"] + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 + + +# +# Now we test effect of varchannels, compress var channels and increase channel array size. +# + - obs space: + name: AMSUB + obsdatain: + engine: + type: H5File + obsfile: Data/010_VarField_britemp.nc4 + simulated variables: [brightnessTemperature] + channels: 1,3 + obs filters: + # Double all observation errors: we want to check if error changes made by filters are + # propagated to VarObs files + - filter: BlackList + action: + name: inflate error + inflation factor: 2.0 + # Reject the first level of the first observation and the last level of the last observation + - filter: Bounds Check + minvalue: 1.15 + maxvalue: 3.35 + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + - filter: VarObs Writer + variables_for_quality_control: + - name: brightnessTemperature + channels: 1,3 + namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp + general_mode: debug + compress_var_channels: false + size_of_varobs_array: 4 + varChannels: 2, 4 #[5,7] + increase_chan_array: true + - filter: VarObs Checker + expected_main_table_columns: + # In the arrays below, rows denote locations and columns levels. + field: ["10", "10", "10", "10", + "10", "10", "10", "10", + "10", "10", "10", "10", + "10", "10", "10", "10"] + level: ["1","2","3","4", + "1","2","3","4", + "1","2","3","4", + "1","2","3","4"] + ob value: ["-1073741824.00000","1.20000","-1073741824.00000","3.11000", + "-1073741824.00000","-1073741824.00000","-1073741824.00000","3.22000", + "-1073741824.00000","1.60000","-1073741824.00000","3.33000", + "-1073741824.00000","1.80000", "-1073741824.00000","3.44000"] + lat: ["21.00000", "21.00000", "21.00000", "21.00000", + "22.00000", "22.00000", "22.00000", "22.00000", + "-23.00000", "-23.00000", "-23.00000","-23.00000", + "24.00000", "24.00000", "24.00000","24.00000"] + lon: ["31.00000","31.00000", "31.00000", "31.00000", + "32.00000", "32.00000", "32.00000","32.00000", + "33.00000", "33.00000","33.00000", "33.00000", + "34.00000", "34.00000", "34.00000","34.00000"] + time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000", + "-3480.00000", "-3480.00000", "-3480.00000","-3480.00000", + "-3420.00000", "-3420.00000", "-3420.00000","-3420.00000", + "-3360.00000", "-3360.00000", "-3360.00000","-3360.00000"] + Callsign: ["station_1", "station_1","station_1", "station_1", + "station_2", "station_2", "station_2","station_2", + "station_3", "station_3", "station_3","station_3", + "station_4", "station_4", "station_4","station_4"] + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 diff --git a/test/testinput/055_VarField_channum.yaml b/test/testinput/055_VarField_channum.yaml index 544aad15..5ed585df 100644 --- a/test/testinput/055_VarField_channum.yaml +++ b/test/testinput/055_VarField_channum.yaml @@ -91,7 +91,8 @@ observations: - filter: VarObs Writer namelist_directory: testinput/VarObsWriterNamelists_055_VarField_channum general_mode: debug - channel_offset: 15 + size_of_varobs_array: 2 + varChannels: 16, 17 - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. diff --git a/test/testinput/057_VarField_emissivity.yaml b/test/testinput/057_VarField_emissivity.yaml index 6ef78508..f4338dbd 100644 --- a/test/testinput/057_VarField_emissivity.yaml +++ b/test/testinput/057_VarField_emissivity.yaml @@ -24,7 +24,7 @@ observations: - filter: VarObs Writer reject_obs_with_all_variables_failing_qc: true namelist_directory: testinput/VarObsWriterNamelists_057_VarField_emissivity - channel_offset: 1 + varChannels: 2-4 size_of_varobs_array: 4 general_mode: verbose - filter: VarObs Checker diff --git a/test/testinput/076_VarField_britempvarerror.yaml b/test/testinput/076_VarField_britempvarerror.yaml index 76a6ceab..2f1ce339 100644 --- a/test/testinput/076_VarField_britempvarerror.yaml +++ b/test/testinput/076_VarField_britempvarerror.yaml @@ -24,7 +24,7 @@ observations: - filter: VarObs Writer reject_obs_with_all_variables_failing_qc: true namelist_directory: testinput/VarObsWriterNamelists_076_VarField_britempvarerror - channel_offset: 1 + varChannels: 2-4 size_of_varobs_array: 4 general_mode: verbose - filter: VarObs Checker diff --git a/test/testinput/varobswriter_globalnamelist_atovs.yaml b/test/testinput/varobswriter_globalnamelist_atovs.yaml index 44657b3f..402ad92b 100644 --- a/test/testinput/varobswriter_globalnamelist_atovs.yaml +++ b/test/testinput/varobswriter_globalnamelist_atovs.yaml @@ -25,7 +25,7 @@ observations: - filter: VarObs Writer namelist_directory: ../etc/global/varobs general_mode: verbose - channel_offset: 1 + varChannels: 2,4 size_of_varobs_array: 3 - filter: VarObs Checker expected_main_table_columns: