From 4516268bb17de03d18177cd46782810064509a5f Mon Sep 17 00:00:00 2001 From: orlewis Date: Tue, 18 Jul 2023 13:58:00 +0100 Subject: [PATCH 01/44] Trying combining offset with use_actual_channels --- src/opsinputs/opsinputs_varobswriter_mod.F90 | 37 ++++++++++++++++---- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 78a20089..d58b99bf 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -317,12 +317,12 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe ! This prevents channels which are seperated by NaNs from being bunched together in the write call f_conf % get_or_die("use_actual_channels", self % useActualChannels) -if ((self % useActualChannels) .and. (self % channel_offset % channel_offset/=0)) then - write (ErrorMessage, '(A)') "OffsetChans and UseActualChans cannot both be set" - call gen_warn(RoutineName, ErrorMessage) - opsinputs_varobswriter_create = .false. - return -end if +!if ((self % useActualChannels) .and. (self % channel_offset % channel_offset/=0)) then +! write (ErrorMessage, '(A)') "OffsetChans and UseActualChans cannot both be set" +! call gen_warn(RoutineName, ErrorMessage) +! opsinputs_varobswriter_create = .false. +! return +!end if ! Updates the varbc flag passedaround by a module in OPS call f_conf % get_or_die("output_varbc_predictors", BoolValue) @@ -1230,6 +1230,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & integer :: iObs logical :: localUseActualChans + ! Body: NumChannels = size(Channels) if (NumChannels == 0) return @@ -1240,17 +1241,39 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & localUseActualChans = UseActualChans end if + call opsinputs_varobswriter_findchannelspassingqc( & Ob % Header % NumObsLocal, ObsSpace, Channels, Flags, ChannelIndices, ChannelCounts) if (FillChanNum) then call Ops_Alloc(Ob % Header % ChanNum, "ChanNum", Ob % Header % NumObsLocal, Ob % ChanNum, & num_levels = NumChannels) +! if (localUseActualChans) then +! do iChannel=1, size(Channels) +! ChannelIndices(:,iChannel) = Channels(iChannel) +! end do + if (localUseActualChans) then do iChannel=1, size(Channels) - ChannelIndices(:,iChannel) = Channels(iChannel) + if (OffsetChans/=0) then + ChannelIndices(:,iChannel) = Channels(iChannel+OffsetChans) + if ((iChannel+OffsetChans)==size(Channels)) exit + else + ChannelIndices(:,iChannel) = Channels(iChannel) + end if end do + +! if (OffsetChans/=0) then +! Ob % ChanNum = ChannelIndices(:,OffSetChans+1) +! else +! Ob % ChanNum = ChannelIndices +! end if + + WRITE(*,*) "ChannelIndices=", ChannelIndices + WRITE(*,*) "##################", ChannelIndices(:, OffSetChans+1) + Ob % ChanNum = ChannelIndices + else Ob % ChanNum = ChannelIndices !only apply offset to actual channels in list, not missing data From 267400b3e019a835d6132636e5375530a7885cf3 Mon Sep 17 00:00:00 2001 From: orlewis Date: Mon, 31 Jul 2023 11:55:46 +0100 Subject: [PATCH 02/44] A commit --- src/opsinputs/VarObsWriterParameters.h | 5 ++ src/opsinputs/opsinputs_fill_mod.F90 | 53 +++++++------ src/opsinputs/opsinputs_varobswriter_mod.F90 | 81 +++++++------------- 3 files changed, 63 insertions(+), 76 deletions(-) diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 631e79e5..259e5307 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -94,6 +94,11 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// channels. This would fill a size 12 array with the array indexes matching the channel number /// [NaN,NaN,NaN,Nan,5,6,7,Nan,9,10,11,NaN]. oops::Parameter use_actual_channels{"use_actual_channels", false, this}; + + + oops::Parameter jopaChannels{"jopaChannels", "", this}; + + oops::Parameter varChannels{"varChannels", "", this}; /// If this list of ufo::variable is defined in the yaml a subset of the flags /// will be made with just these variables present. This will allow Fortran calls such-as diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 50680bef..73ff711c 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1034,7 +1034,8 @@ end subroutine opsinputs_fill_fillreal !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield !> are not found. subroutine opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, NumObs, Real2, ObsSpace, Channels, JediVarName, JediVarGroup, OffsetChans, useActualChans) + Hdr, OpsVarName, NumObs, Real2, ObsSpace, jopaChannels, varChannels, JediVarName, & + JediVarGroup, OffsetChans, useActualChans) implicit none ! Subroutine arguments: @@ -1043,7 +1044,8 @@ subroutine opsinputs_fill_fillreal2d_norecords( & integer(integer64), intent(in) :: NumObs real(real64), pointer, intent(out) :: Real2(:,:) type(c_ptr), value, intent(in) :: ObsSpace -integer(c_int), intent(in) :: Channels(:) +integer(c_int), intent(in) :: jopaChannels(:) +integer(c_int), intent(in) :: varChannels(:) character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup type(opsinputs_channeloffset), optional, intent(in) :: OffsetChans @@ -1052,7 +1054,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & ! Local declarations: real(kind=c_double) :: VarValue(NumObs) real(kind=c_double) :: MissingDouble -character(len=max_varname_with_channel_length) :: JediVarNamesWithChannels(max(size(Channels), 1)) +character(len=max_varname_with_channel_length) :: JediVarNamesWithChannels(max(size(jopaChannels), 1)) integer :: iChannel integer :: offset integer :: numchans @@ -1062,7 +1064,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & MissingDouble = missing_value(0.0_c_double) -JediVarNamesWithChannels = opsinputs_fill_varnames_with_channels(JediVarName, Channels) +JediVarNamesWithChannels = opsinputs_fill_varnames_with_channels(JediVarName, jopaChannels) !take into account offsetting of 2nd dimension if required !designed to be used to pack where multiple satellite instruments expected @@ -1088,18 +1090,21 @@ subroutine opsinputs_fill_fillreal2d_norecords( & do iChannel = 1, size(JediVarNamesWithChannels) ! Retrieve data from JEDI call obsspace_get_db(ObsSpace, JediVarGroup, JediVarNamesWithChannels(iChannel), VarValue) - + where (VarValue /= MissingDouble) + Real2(:, varChannels(iChannel)) = VarValue + end where ! Fill the OPS data structures - if (localUseActualChans) then - where (VarValue /= MissingDouble) - Real2(:, Channels(iChannel)) = VarValue - end where - else - ! Fill the OPS data structures - where (VarValue /= MissingDouble) - Real2(:, iChannel+offset) = VarValue - end where - end if +! if (localUseActualChans) then +! where (VarValue /= MissingDouble) +! Real2(:, Channels(iChannel)) = VarValue +! end where +! else +! ! Fill the OPS data structures +! where (VarValue /= MissingDouble) +! Real2(:, iChannel+offset) = VarValue +! end where +! end if + end do end if ! Data not present? OPS will produce a warning -- we don't need to duplicate it. end subroutine opsinputs_fill_fillreal2d_norecords @@ -1222,7 +1227,8 @@ end subroutine opsinputs_fill_fillreal2d_records !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield !> are not found. subroutine opsinputs_fill_fillreal2d( & - Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, Channels, VarobsLength, JediVarName, JediVarGroup, OffsetChans, & + Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, jopaChannels, varChannels, & + VarobsLength, JediVarName, JediVarGroup, OffsetChans, & useActualChans) implicit none @@ -1232,7 +1238,8 @@ subroutine opsinputs_fill_fillreal2d( & type(opsinputs_jeditoopslayoutmapping), intent(in) :: JediToOpsLayoutMapping real(real64), pointer, intent(out) :: Real2(:,:) type(c_ptr), value, intent(in) :: ObsSpace -integer(c_int), intent(in) :: Channels(:) +integer(c_int), intent(in) :: jopaChannels(:) +integer(c_int), intent(in) :: varChannels(:) integer(integer64), intent(in) :: VarobsLength character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup @@ -1247,17 +1254,17 @@ subroutine opsinputs_fill_fillreal2d( & if (Present(OffsetChans)) then if (Present(useActualChans)) then call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, OffsetChans, useActualChans) + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, jopaChannels, & + varChannels, JediVarName, JediVarGroup, OffsetChans, useActualChans) else call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, OffsetChans) + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, jopaChannels, & + varChannels, JediVarName, JediVarGroup, OffsetChans) end if else call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup) + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, jopaChannels, & + varChannels, JediVarName, JediVarGroup) end if end if diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index d58b99bf..61fa6f52 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -186,6 +186,8 @@ module opsinputs_varobswriter_mod integer(integer64) :: VarobsLength integer(c_int), allocatable :: channels(:) + integer(c_int), allocatable :: jopaChannels(:) + integer(c_int), allocatable :: varChannels(:) !this stores the atmospheric levels we wish to pass to varobs integer(c_int), allocatable :: modlevs(:) @@ -317,6 +319,9 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe ! This prevents channels which are seperated by NaNs from being bunched together in the write call f_conf % get_or_die("use_actual_channels", self % useActualChannels) +call f_conf % get_or_die("varChannels", self % varChannels) + +call f_conf % get_or_die("jopaChannels", self % jopaChannels) !if ((self % useActualChannels) .and. (self % channel_offset % channel_offset/=0)) then ! write (ErrorMessage, '(A)') "OffsetChans and UseActualChans cannot both be set" ! call gen_warn(RoutineName, ErrorMessage) @@ -861,8 +866,8 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_britemp) call opsinputs_fill_fillreal2d( & Ob % Header % CorBriTemp, "CorBriTemp", JediToOpsLayoutMapping, Ob % CorBriTemp, & - ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue", self % channel_offset, & - self % useActualChannels) + ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue", & + self % channel_offset, self % useActualChannels) case (VarField_tskin) call opsinputs_fill_fillelementtypefromnormalvariable( & Ob % Header % Tskin, "Tskin", Ob % Header % NumObsLocal, Ob % Tskin, & @@ -878,7 +883,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_mwemiss) call opsinputs_fill_fillreal2d( & Ob % Header % MwEmiss, "MwEmiss", JediToOpsLayoutMapping, Ob % MwEmiss, & - ObsSpace, self % channels, self % VarobsLength, "emissivity", "Emiss") + ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "emissivity", "Emiss") case (VarField_TCozone) call opsinputs_fill_fillreal( & Ob % Header % TCozone, "TCozone", JediToOpsLayoutMapping, Ob % TCozone, & @@ -991,7 +996,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_Emissivity) call opsinputs_fill_fillreal2d( & Ob % Header % Emissivity, "Emissivity", JediToOpsLayoutMapping, Ob % Emissivity, & - ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar", self % channel_offset) + ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "emissivity", "OneDVar", self % channel_offset) case (VarField_QCinfo) ! TODO(someone): This will come from a variable generated by the 1D-Var filter. Its name and ! group are not known yet. Once they are, replace the placeholders in the call below. @@ -1022,7 +1027,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_RadarObAzim) call opsinputs_fill_fillreal2d( & Ob % Header % RadarObAzim, "RadarObAzim", JediToOpsLayoutMapping, Ob % RadarObAzim, & - ObsSpace, self % channels, self % VarobsLength, "radarAzimuth", "MetaData") + ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "radarAzimuth", "MetaData") case (VarField_RadIdent) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % RadIdent, "RadIdent", Ob % Header % NumObsLocal, Ob % RadIdent) @@ -1090,7 +1095,8 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_BriTempVarError) call opsinputs_fill_fillreal2d( & Ob % Header % BriTempVarError, "BriTempVarError", JediToOpsLayoutMapping, Ob % BriTempVarError, & - ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature", "ObsErrorData", self % channel_offset) + ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "brightnessTemperature",& + "ObsErrorData", self % channel_offset) case (VarField_CloudRTError) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % CloudRTError, "CloudRTError", Ob % Header % NumObsLocal, Ob % CloudRTError) @@ -1108,11 +1114,11 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_LevelLat) call opsinputs_fill_fillreal2d( & Ob % Header % level_lat, "level_lat", JediToOpsLayoutMapping, Ob % level_lat, & - ObsSpace, self % channels, self % VarobsLength, "latitude", "MetaData") + ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "latitude", "MetaData") case (VarField_LevelLon) call opsinputs_fill_fillreal2d( & Ob % Header % level_lon, "level_lon", JediToOpsLayoutMapping, Ob % level_lon, & - ObsSpace, self % channels, self % VarobsLength, "longitude", "MetaData") + ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "longitude", "MetaData") case (VarField_RainAccum) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % RainAccum, "RainAccum", Ob % Header % NumObsLocal, Ob % RainAccum) @@ -1163,7 +1169,7 @@ subroutine opsinputs_varobswriter_populateobservations( & if (FillChanNum .or. FillNumChans) then call opsinputs_varobswriter_fillchannumandnumchans( & - Ob, ObsSpace, self % channels, Flags, FillChanNum, & + Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & FillNumChans, self % channel_offset % channel_offset, self % useActualChannels) end if @@ -1209,14 +1215,15 @@ end subroutine opsinputs_varobswriter_fillreportflags !> e.g. for HIRS & AMSUA !> the number of these channels is stored in Ob % NumChans. subroutine opsinputs_varobswriter_fillchannumandnumchans( & - Ob, ObsSpace, Channels, Flags, FillChanNum, FillNumChans, & + Ob, ObsSpace, jopaChannels, varChannels, Flags, FillChanNum, FillNumChans, & OffsetChans,UseActualChans) implicit none ! Subroutine arguments: type(OB_type), intent(inout) :: Ob type(c_ptr), value, intent(in) :: ObsSpace -integer(c_int), intent(in) :: Channels(:) +integer(c_int), intent(in) :: jopaChannels(:) +integer(c_int), intent(in) :: varChannels(:) type(c_ptr), value, intent(in) :: Flags logical, intent(in) :: FillChanNum, FillNumChans integer, intent(in) :: OffsetChans @@ -1224,7 +1231,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & ! Local declarations: integer(integer64) :: NumChannels -integer(integer64) :: ChannelIndices(Ob % Header % NumObsLocal, size(Channels)) +integer(integer64) :: ChannelIndices(Ob % Header % NumObsLocal, size(jopaChannels)) integer(integer64) :: ChannelCounts(Ob % Header % NumObsLocal) integer :: iChannel integer :: iObs @@ -1232,56 +1239,24 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & ! Body: -NumChannels = size(Channels) +NumChannels = size(jopaChannels) if (NumChannels == 0) return -! Setup for useActualChans for array indices -localUseActualchans = .false. -if (Present(UseActualChans)) then - localUseActualChans = UseActualChans -end if - - call opsinputs_varobswriter_findchannelspassingqc( & - Ob % Header % NumObsLocal, ObsSpace, Channels, Flags, ChannelIndices, ChannelCounts) + Ob % Header % NumObsLocal, ObsSpace, jopaChannels, Flags, ChannelIndices, ChannelCounts) + if (FillChanNum) then call Ops_Alloc(Ob % Header % ChanNum, "ChanNum", Ob % Header % NumObsLocal, Ob % ChanNum, & num_levels = NumChannels) - -! if (localUseActualChans) then -! do iChannel=1, size(Channels) -! ChannelIndices(:,iChannel) = Channels(iChannel) -! end do - - if (localUseActualChans) then - do iChannel=1, size(Channels) - if (OffsetChans/=0) then - ChannelIndices(:,iChannel) = Channels(iChannel+OffsetChans) - if ((iChannel+OffsetChans)==size(Channels)) exit - else - ChannelIndices(:,iChannel) = Channels(iChannel) - end if + if (size(jopaChannels) == size(varChannels)) then + do iChannel=1, NumChannels + ChannelIndices(:,iChannel) = VarChannels(iChannel) end do - -! if (OffsetChans/=0) then -! Ob % ChanNum = ChannelIndices(:,OffSetChans+1) -! else -! Ob % ChanNum = ChannelIndices -! end if - - WRITE(*,*) "ChannelIndices=", ChannelIndices - WRITE(*,*) "##################", ChannelIndices(:, OffSetChans+1) - - Ob % ChanNum = ChannelIndices - - else + Ob % ChanNum = ChannelIndices - !only apply offset to actual channels in list, not missing data - where (Ob % ChanNum > 0) - Ob % ChanNum = Ob % ChanNum + int(OffsetChans, kind=integer64) - end where + end if -end if +end if if (FillNumChans) then call Ops_Alloc(Ob % Header % NumChans, "NumChans", Ob % Header % NumObsLocal, Ob % NumChans) From 5fbcae4bc8b5f528b89602cd3e17e9b7be78c183 Mon Sep 17 00:00:00 2001 From: orlewis Date: Mon, 31 Jul 2023 16:01:15 +0100 Subject: [PATCH 03/44] Commit some changes --- src/opsinputs/VarObsWriterParameters.h | 5 +-- src/opsinputs/opsinputs_fill_mod.F90 | 20 +++++------ src/opsinputs/opsinputs_varobswriter_mod.F90 | 36 +++++++++++--------- 3 files changed, 33 insertions(+), 28 deletions(-) diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 259e5307..4939e9d8 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -96,9 +96,10 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { oops::Parameter use_actual_channels{"use_actual_channels", false, this}; - oops::Parameter jopaChannels{"jopaChannels", "", this}; + /// oops::Parameter jopaChannels{"jopaChannels", "", this}; + + oops::Parameter> varChannels{"varChannels", {}, this}; - oops::Parameter varChannels{"varChannels", "", this}; /// If this list of ufo::variable is defined in the yaml a subset of the flags /// will be made with just these variables present. This will allow Fortran calls such-as diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 73ff711c..5c5a1afa 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1034,7 +1034,7 @@ end subroutine opsinputs_fill_fillreal !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield !> are not found. subroutine opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, NumObs, Real2, ObsSpace, jopaChannels, varChannels, JediVarName, & + Hdr, OpsVarName, NumObs, Real2, ObsSpace, Channels, varChannels, JediVarName, & JediVarGroup, OffsetChans, useActualChans) implicit none @@ -1044,7 +1044,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & integer(integer64), intent(in) :: NumObs real(real64), pointer, intent(out) :: Real2(:,:) type(c_ptr), value, intent(in) :: ObsSpace -integer(c_int), intent(in) :: jopaChannels(:) +integer(c_int), intent(in) :: Channels(:) integer(c_int), intent(in) :: varChannels(:) character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup @@ -1054,7 +1054,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & ! Local declarations: real(kind=c_double) :: VarValue(NumObs) real(kind=c_double) :: MissingDouble -character(len=max_varname_with_channel_length) :: JediVarNamesWithChannels(max(size(jopaChannels), 1)) +character(len=max_varname_with_channel_length) :: JediVarNamesWithChannels(max(size(Channels), 1)) integer :: iChannel integer :: offset integer :: numchans @@ -1064,7 +1064,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & MissingDouble = missing_value(0.0_c_double) -JediVarNamesWithChannels = opsinputs_fill_varnames_with_channels(JediVarName, jopaChannels) +JediVarNamesWithChannels = opsinputs_fill_varnames_with_channels(JediVarName, Channels) !take into account offsetting of 2nd dimension if required !designed to be used to pack where multiple satellite instruments expected @@ -1227,7 +1227,7 @@ end subroutine opsinputs_fill_fillreal2d_records !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield !> are not found. subroutine opsinputs_fill_fillreal2d( & - Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, jopaChannels, varChannels, & + Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, Channels, varChannels, & VarobsLength, JediVarName, JediVarGroup, OffsetChans, & useActualChans) implicit none @@ -1238,8 +1238,8 @@ subroutine opsinputs_fill_fillreal2d( & type(opsinputs_jeditoopslayoutmapping), intent(in) :: JediToOpsLayoutMapping real(real64), pointer, intent(out) :: Real2(:,:) type(c_ptr), value, intent(in) :: ObsSpace -integer(c_int), intent(in) :: jopaChannels(:) -integer(c_int), intent(in) :: varChannels(:) +integer(c_int), intent(in) :: Channels(:) +integer, intent(in) :: varChannels(:) integer(integer64), intent(in) :: VarobsLength character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup @@ -1254,16 +1254,16 @@ subroutine opsinputs_fill_fillreal2d( & if (Present(OffsetChans)) then if (Present(useActualChans)) then call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, jopaChannels, & + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & varChannels, JediVarName, JediVarGroup, OffsetChans, useActualChans) else call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, jopaChannels, & + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & varChannels, JediVarName, JediVarGroup, OffsetChans) end if else call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, jopaChannels, & + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & varChannels, JediVarName, JediVarGroup) end if end if diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 61fa6f52..b01d328c 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -186,7 +186,7 @@ module opsinputs_varobswriter_mod integer(integer64) :: VarobsLength integer(c_int), allocatable :: channels(:) - integer(c_int), allocatable :: jopaChannels(:) + ! integer(c_int), allocatable :: jopaChannels(:) integer(c_int), allocatable :: varChannels(:) !this stores the atmospheric levels we wish to pass to varobs @@ -321,7 +321,8 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe call f_conf % get_or_die("varChannels", self % varChannels) -call f_conf % get_or_die("jopaChannels", self % jopaChannels) +!call f_conf % get_or_die("jopaChannels", self % jopaChannels) + !if ((self % useActualChannels) .and. (self % channel_offset % channel_offset/=0)) then ! write (ErrorMessage, '(A)') "OffsetChans and UseActualChans cannot both be set" ! call gen_warn(RoutineName, ErrorMessage) @@ -866,7 +867,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_britemp) call opsinputs_fill_fillreal2d( & Ob % Header % CorBriTemp, "CorBriTemp", JediToOpsLayoutMapping, Ob % CorBriTemp, & - ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue", & + ObsSpace, self % channels, self % varChannels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue", & self % channel_offset, self % useActualChannels) case (VarField_tskin) call opsinputs_fill_fillelementtypefromnormalvariable( & @@ -883,7 +884,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_mwemiss) call opsinputs_fill_fillreal2d( & Ob % Header % MwEmiss, "MwEmiss", JediToOpsLayoutMapping, Ob % MwEmiss, & - ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "emissivity", "Emiss") + ObsSpace, self % channels, self % varChannels, self % VarobsLength, "emissivity", "Emiss") case (VarField_TCozone) call opsinputs_fill_fillreal( & Ob % Header % TCozone, "TCozone", JediToOpsLayoutMapping, Ob % TCozone, & @@ -996,7 +997,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_Emissivity) call opsinputs_fill_fillreal2d( & Ob % Header % Emissivity, "Emissivity", JediToOpsLayoutMapping, Ob % Emissivity, & - ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "emissivity", "OneDVar", self % channel_offset) + ObsSpace, self % channels, self % varChannels, self % VarobsLength, "emissivity", "OneDVar", self % channel_offset) case (VarField_QCinfo) ! TODO(someone): This will come from a variable generated by the 1D-Var filter. Its name and ! group are not known yet. Once they are, replace the placeholders in the call below. @@ -1027,7 +1028,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_RadarObAzim) call opsinputs_fill_fillreal2d( & Ob % Header % RadarObAzim, "RadarObAzim", JediToOpsLayoutMapping, Ob % RadarObAzim, & - ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "radarAzimuth", "MetaData") + ObsSpace, self % channels, self % varChannels, self % VarobsLength, "radarAzimuth", "MetaData") case (VarField_RadIdent) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % RadIdent, "RadIdent", Ob % Header % NumObsLocal, Ob % RadIdent) @@ -1095,7 +1096,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_BriTempVarError) call opsinputs_fill_fillreal2d( & Ob % Header % BriTempVarError, "BriTempVarError", JediToOpsLayoutMapping, Ob % BriTempVarError, & - ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "brightnessTemperature",& + ObsSpace, self % channels, self % varChannels, self % VarobsLength, "brightnessTemperature",& "ObsErrorData", self % channel_offset) case (VarField_CloudRTError) ! TODO(someone): handle this varfield @@ -1114,11 +1115,11 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_LevelLat) call opsinputs_fill_fillreal2d( & Ob % Header % level_lat, "level_lat", JediToOpsLayoutMapping, Ob % level_lat, & - ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "latitude", "MetaData") + ObsSpace, self % channels, self % varChannels, self % VarobsLength, "latitude", "MetaData") case (VarField_LevelLon) call opsinputs_fill_fillreal2d( & Ob % Header % level_lon, "level_lon", JediToOpsLayoutMapping, Ob % level_lon, & - ObsSpace, self % jopaChannels, self % varChannels, self % VarobsLength, "longitude", "MetaData") + ObsSpace, self % channels, self % varChannels, self % VarobsLength, "longitude", "MetaData") case (VarField_RainAccum) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % RainAccum, "RainAccum", Ob % Header % NumObsLocal, Ob % RainAccum) @@ -1215,15 +1216,16 @@ end subroutine opsinputs_varobswriter_fillreportflags !> e.g. for HIRS & AMSUA !> the number of these channels is stored in Ob % NumChans. subroutine opsinputs_varobswriter_fillchannumandnumchans( & - Ob, ObsSpace, jopaChannels, varChannels, Flags, FillChanNum, FillNumChans, & + Ob, ObsSpace, channels, varChannels, Flags, FillChanNum, FillNumChans, & OffsetChans,UseActualChans) implicit none ! Subroutine arguments: type(OB_type), intent(inout) :: Ob type(c_ptr), value, intent(in) :: ObsSpace -integer(c_int), intent(in) :: jopaChannels(:) -integer(c_int), intent(in) :: varChannels(:) +integer(c_int), intent(in) :: channels(:) +!integer(c_int), intent(in) :: varChannels(:) +integer, intent(in) :: varChannels(:) type(c_ptr), value, intent(in) :: Flags logical, intent(in) :: FillChanNum, FillNumChans integer, intent(in) :: OffsetChans @@ -1231,7 +1233,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & ! Local declarations: integer(integer64) :: NumChannels -integer(integer64) :: ChannelIndices(Ob % Header % NumObsLocal, size(jopaChannels)) +integer(integer64) :: ChannelIndices(Ob % Header % NumObsLocal, size(channels)) integer(integer64) :: ChannelCounts(Ob % Header % NumObsLocal) integer :: iChannel integer :: iObs @@ -1239,17 +1241,19 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & ! Body: -NumChannels = size(jopaChannels) +NumChannels = size(channels) if (NumChannels == 0) return call opsinputs_varobswriter_findchannelspassingqc( & - Ob % Header % NumObsLocal, ObsSpace, jopaChannels, Flags, ChannelIndices, ChannelCounts) + Ob % Header % NumObsLocal, ObsSpace, channels, Flags, ChannelIndices, ChannelCounts) if (FillChanNum) then call Ops_Alloc(Ob % Header % ChanNum, "ChanNum", Ob % Header % NumObsLocal, Ob % ChanNum, & num_levels = NumChannels) - if (size(jopaChannels) == size(varChannels)) then + WRITE(*,*) "#### In if FillChnaNum" + if (size(channels) == size(varChannels)) then do iChannel=1, NumChannels + WRITE(*,*) "VarChannels=", VarChannels(iChannel) ChannelIndices(:,iChannel) = VarChannels(iChannel) end do From 367c5881a27e57a7dc8093701aef0a6dbee01732 Mon Sep 17 00:00:00 2001 From: orlewis Date: Tue, 1 Aug 2023 11:24:26 +0100 Subject: [PATCH 04/44] A push for testing purposes. --- src/opsinputs/opsinputs_fill_mod.F90 | 4 +- src/opsinputs/opsinputs_varobswriter_mod.F90 | 9 +- test/testinput/010_VarField_britemp.yaml | 202 ++++++++++++------ .../AMSUB.nl | 2 +- 4 files changed, 149 insertions(+), 68 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 5c5a1afa..f48538f4 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1045,7 +1045,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & real(real64), pointer, intent(out) :: Real2(:,:) type(c_ptr), value, intent(in) :: ObsSpace integer(c_int), intent(in) :: Channels(:) -integer(c_int), intent(in) :: varChannels(:) +integer, intent(in) :: varChannels(:) character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup type(opsinputs_channeloffset), optional, intent(in) :: OffsetChans @@ -1071,6 +1071,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & !e.g. HIRS in ATOVS stream offset = 0 numchans = size(JediVarNamesWithChannels) +WRITE(*,*) "numchans=", numchans if (present(OffsetChans)) then offset = OffsetChans % channel_offset if (OffsetChans % size_of_varobs_array > 0) & @@ -1089,6 +1090,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & num_levels = int(numchans, kind=integer64)) do iChannel = 1, size(JediVarNamesWithChannels) ! Retrieve data from JEDI + WRITE(*,*) "JediVarNamesWithChannels(iChannel)=", JediVarNamesWithChannels(iChannel) call obsspace_get_db(ObsSpace, JediVarGroup, JediVarNamesWithChannels(iChannel), VarValue) where (VarValue /= MissingDouble) Real2(:, varChannels(iChannel)) = VarValue diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index b01d328c..5cb9f672 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -187,7 +187,7 @@ module opsinputs_varobswriter_mod integer(c_int), allocatable :: channels(:) ! integer(c_int), allocatable :: jopaChannels(:) - integer(c_int), allocatable :: varChannels(:) + integer, allocatable :: varChannels(:) !this stores the atmospheric levels we wish to pass to varobs integer(c_int), allocatable :: modlevs(:) @@ -320,6 +320,7 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe call f_conf % get_or_die("use_actual_channels", self % useActualChannels) call f_conf % get_or_die("varChannels", self % varChannels) +WRITE(*,*) "get or die varChannels", self % varChannels !call f_conf % get_or_die("jopaChannels", self % jopaChannels) @@ -1251,10 +1252,12 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & call Ops_Alloc(Ob % Header % ChanNum, "ChanNum", Ob % Header % NumObsLocal, Ob % ChanNum, & num_levels = NumChannels) WRITE(*,*) "#### In if FillChnaNum" + write(*,*) size(channels) + write(*,*) size(varChannels) if (size(channels) == size(varChannels)) then do iChannel=1, NumChannels - WRITE(*,*) "VarChannels=", VarChannels(iChannel) - ChannelIndices(:,iChannel) = VarChannels(iChannel) + WRITE(*,*) "VarChannels=", varChannels(iChannel) + ChannelIndices(:,iChannel) = varChannels(iChannel) end do Ob % ChanNum = ChannelIndices diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index cb4e3091..cc5aa361 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -2,64 +2,132 @@ window begin: 2018-01-01T00:00:00Z window end: 2018-01-01T02:00:00Z observations: - - obs space: - name: AMSUB - obsdatain: - engine: - type: H5File - obsfile: Data/010_VarField_britemp.nc4 - simulated variables: [brightnessTemperature] - channels: 1, 3 - obs filters: - # Set the flag of observations with missing values to "pass": we want to check if these - # values are encoded correctly in the VarObsFile. - - filter: Reset Flags to Pass - flags_to_reset: [10, 15] # missing, Hfailed - # Reject observation 3: we want to check if it is omitted from the VarObs file, as expected. - - filter: Domain Check - where: - - variable: - name: MetaData/latitude - minvalue: 0.0 - - filter: VarObs Writer - namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp - general_mode: debug - - filter: VarObs Checker - expected_main_table_columns: - # Only levels 1 and 3 are simulated and hence written to the VarObs file. - # In the arrays below, rows denote locations and columns channels. - field: ["10", "10", - "10", "10", - "10", "10", - "10", "10"] - level: ["1", "2", - "1", "2", - "1", "2", - "1", "2"] - ob value: ["1.20000", "3.11000", - "-1073741824.00000", "3.22000", - "1.60000", "3.33000", - "1.80000", "3.44000"] - lat: ["21.00000", "21.00000", - "22.00000", "22.00000", - "-23.00000", "-23.00000", - "24.00000", "24.00000"] - lon: ["31.00000", "31.00000", - "32.00000", "32.00000", - "33.00000", "33.00000", - "34.00000", "34.00000"] - time: ["-3540.00000", "-3540.00000", - "-3480.00000", "-3480.00000", - "-3420.00000", "-3420.00000", - "-3360.00000", "-3360.00000"] - Callsign: ["station_1", "station_1", - "station_2", "station_2", - "station_3", "station_3", - "station_4", "station_4"] - HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. - benchmarkFlag: 1000 # just to keep the ObsFilters test happy - flaggedBenchmark: 0 +# - obs space: +# name: AMSUB +# obsdatain: +# engine: +# type: H5File +# obsfile: Data/010_VarField_britemp.nc4 +# simulated variables: [brightnessTemperature] +# channels: 1, 3 +# obs filters: +# # Set the flag of observations with missing values to "pass": we want to check if these +# # values are encoded correctly in the VarObsFile. +# - filter: Reset Flags to Pass +# flags_to_reset: [10, 15] # missing, Hfailed +# # Reject observation 3: we want to check if it is omitted from the VarObs file, as expected. +# - filter: Domain Check +# where: +# - variable: +# name: MetaData/latitude +# minvalue: 0.0 +# - filter: VarObs Writer +# namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp +# general_mode: debug +# - filter: VarObs Checker +# expected_main_table_columns: +# # Only levels 1 and 3 are simulated and hence written to the VarObs file. +# # In the arrays below, rows denote locations and columns channels. +# field: ["10", "10", +# "10", "10", +# "10", "10", +# "10", "10"] +# level: ["1", "2", +# "1", "2", +# "1", "2", +# "1", "2"] +# ob value: ["1.20000", "3.11000", +# "-1073741824.00000", "3.22000", +# "1.60000", "3.33000", +# "1.80000", "3.44000"] +# lat: ["21.00000", "21.00000", +# "22.00000", "22.00000", +# "-23.00000", "-23.00000", +# "24.00000", "24.00000"] +# lon: ["31.00000", "31.00000", +# "32.00000", "32.00000", +# "33.00000", "33.00000", +# "34.00000", "34.00000"] +# time: ["-3540.00000", "-3540.00000", +# "-3480.00000", "-3480.00000", +# "-3420.00000", "-3420.00000", +# "-3360.00000", "-3360.00000"] +# Callsign: ["station_1", "station_1", +# "station_2", "station_2", +# "station_3", "station_3", +# "station_4", "station_4"] +# HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. +# benchmarkFlag: 1000 # just to keep the ObsFilters test happy +# flaggedBenchmark: 0 +## +## Now we test effect of actual channel +## +# - obs space: +# name: AMSUB +# obsdatain: +# engine: +# type: H5File +# obsfile: Data/010_VarField_britemp.nc4 +# simulated variables: [brightnessTemperature] +# channels: 1,3 +# obs filters: +# # Double all observation errors: we want to check if error changes made by filters are +# # propagated to VarObs files +# - filter: BlackList +# action: +# name: inflate error +# inflation factor: 2.0 +# # Reject the first level of the first observation and the last level of the last observation +# - filter: Bounds Check +# minvalue: 1.15 +# maxvalue: 3.35 +# # Set the flag of observations with missing values to "pass": we want to check if these +# # values are encoded correctly in the VarObsFile. +# - filter: Reset Flags to Pass +# flags_to_reset: [10, 15] # missing, Hfailed +# - filter: VarObs Writer +# namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp +# general_mode: debug +# size_of_varobs_array: 3 +# use_actual_channels: true +# - filter: VarObs Checker +# expected_main_table_columns: +# # In the arrays below, rows denote locations and columns levels. +# field: ["10", "10", "10", +# "10", "10", "10", +# "10", "10", "10", +# "10", "10", "10"] +# level: ["1", "2","3", +# "1", "2","3", +# "1", "2","3", +# "1", "2","3"] +# ob value: ["1.20000","-1073741824.00000","3.11000", +# "-1073741824.00000","-1073741824.00000","3.22000", +# "1.60000","-1073741824.00000","3.33000", +# "1.80000","-1073741824.00000","3.44000"] +# lat: ["21.00000", "21.00000","21.00000", +# "22.00000", "22.00000","22.00000", +# "-23.00000", "-23.00000","-23.00000", +# "24.00000", "24.00000","24.00000"] +# lon: ["31.00000", "31.00000","31.00000", +# "32.00000", "32.00000","32.00000", +# "33.00000", "33.00000","33.00000", +# "34.00000", "34.00000","34.00000"] +# time: ["-3540.00000", "-3540.00000","-3540.00000", +# "-3480.00000", "-3480.00000","-3480.00000", +# "-3420.00000", "-3420.00000","-3420.00000", +# "-3360.00000", "-3360.00000","-3360.00000"] +# Callsign: ["station_1", "station_1","station_1", +# "station_2", "station_2","station_2", +# "station_3", "station_3","station_3", +# "station_4", "station_4","station_4"] +# HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. +# benchmarkFlag: 1000 # just to keep the ObsFilters test happy +# flaggedBenchmark: 0 + + + # # Now we test effect of actual channel # @@ -69,6 +137,10 @@ observations: engine: type: H5File obsfile: Data/010_VarField_britemp.nc4 + obsdataout: + engine: + type: H5File + obsfile: /home/h01/olewis/map_chans.nc4 simulated variables: [brightnessTemperature] channels: 1,3 obs filters: @@ -87,10 +159,14 @@ observations: - filter: Reset Flags to Pass flags_to_reset: [10, 15] # missing, Hfailed - filter: VarObs Writer + variables_for_quality_control: + - name: brightnessTemperature + channels: 1,3 namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp general_mode: debug - size_of_varobs_array: 3 - use_actual_channels: true + # size_of_varobs_array: 3 + varChannels: [5,7] + output_directory: /home/h01/olewis/ - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. @@ -98,10 +174,10 @@ observations: "10", "10", "10", "10", "10", "10", "10", "10", "10"] - level: ["1", "2","3", - "1", "2","3", - "1", "2","3", - "1", "2","3"] + level: ["5", "6","7", + "5", "6","7", + "5", "6","7", + "5", "6","7"] ob value: ["1.20000","-1073741824.00000","3.11000", "-1073741824.00000","-1073741824.00000","3.22000", "1.60000","-1073741824.00000","3.33000", diff --git a/test/testinput/VarObsWriterNamelists_010_VarField_britemp/AMSUB.nl b/test/testinput/VarObsWriterNamelists_010_VarField_britemp/AMSUB.nl index 15d2b7a2..3eb07827 100644 --- a/test/testinput/VarObsWriterNamelists_010_VarField_britemp/AMSUB.nl +++ b/test/testinput/VarObsWriterNamelists_010_VarField_britemp/AMSUB.nl @@ -1,3 +1,3 @@ &VarobsControlNL -Varfields=10 +Varfields=10,54,55 / From 06935974863a1b18c213cf74afe946ba758f8c08 Mon Sep 17 00:00:00 2001 From: orlewis Date: Wed, 2 Aug 2023 15:46:53 +0100 Subject: [PATCH 05/44] Some more changes, bu failing at call to fillchan --- src/opsinputs/VarObsWriter.cc | 9 ++++ src/opsinputs/VarObsWriter.h | 1 + src/opsinputs/VarObsWriterParameters.h | 4 +- src/opsinputs/opsinputs_fill_mod.F90 | 14 +++--- src/opsinputs/opsinputs_varobswriter_mod.F90 | 51 ++++++++++++-------- test/testinput/010_VarField_britemp.yaml | 2 +- 6 files changed, 50 insertions(+), 31 deletions(-) diff --git a/src/opsinputs/VarObsWriter.cc b/src/opsinputs/VarObsWriter.cc index 52644ceb..f813fdb9 100644 --- a/src/opsinputs/VarObsWriter.cc +++ b/src/opsinputs/VarObsWriter.cc @@ -18,6 +18,7 @@ #include "oops/base/Variables.h" #include "oops/mpi/mpi.h" #include "oops/util/Logger.h" +#include "oops/util/IntSetParser.h" #include "opsinputs/LocalEnvironment.h" #include "opsinputs/VarObsWriterParameters.h" #include "ufo/filters/Variable.h" @@ -69,6 +70,14 @@ VarObsWriter::VarObsWriter(ioda::ObsSpace & obsdb, const Parameters_ & params, const int fallbackChannels = 0; // Avoid passing a null pointer to Fortran. const int *channelsData = channels.empty() ? &fallbackChannels : channels.data(); + +/// std::set chanset = oops::parseIntSet(options_.chanlist.value()); +/// channels_.assign(chanset.begin(), chanset.end()); + + std::set varchanset = oops::parseIntSet(parameters_.varChannels.value()); + varchannels_.assign(varchanset.begin(), varchanset.end()); + conf.set("varChannels", varchannels_); + if (!opsinputs_varobswriter_create_f90(key_, &conf, fortranMpiCommunicatorIsValid, fortranMpiCommunicator, diff --git a/src/opsinputs/VarObsWriter.h b/src/opsinputs/VarObsWriter.h index 80fe226f..ad901cfa 100644 --- a/src/opsinputs/VarObsWriter.h +++ b/src/opsinputs/VarObsWriter.h @@ -87,6 +87,7 @@ class VarObsWriter : public oops::interface::ObsFilterBase, oops::Variables extradiagvars_; std::shared_ptr> flags_; std::shared_ptr> obsErrors_; + std::vector varchannels_; VarObsWriterParameters parameters_; }; diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 4939e9d8..2a34feb0 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -98,7 +98,9 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// oops::Parameter jopaChannels{"jopaChannels", "", this}; - oops::Parameter> varChannels{"varChannels", {}, this}; + /// oops::Parameter> varChannels{"varChannels", {}, this}; + oops::Parameter varChannels{"varChannels", "", this}; + ///oops::RequiredParameter chanlist{"cost channels list", this}; /// If this list of ufo::variable is defined in the yaml a subset of the flags diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index f48538f4..01201e4c 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1034,7 +1034,7 @@ end subroutine opsinputs_fill_fillreal !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield !> are not found. subroutine opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, NumObs, Real2, ObsSpace, Channels, varChannels, JediVarName, & + Hdr, OpsVarName, NumObs, Real2, ObsSpace, Channels, JediVarName, & JediVarGroup, OffsetChans, useActualChans) implicit none @@ -1045,7 +1045,6 @@ subroutine opsinputs_fill_fillreal2d_norecords( & real(real64), pointer, intent(out) :: Real2(:,:) type(c_ptr), value, intent(in) :: ObsSpace integer(c_int), intent(in) :: Channels(:) -integer, intent(in) :: varChannels(:) character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup type(opsinputs_channeloffset), optional, intent(in) :: OffsetChans @@ -1093,7 +1092,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & WRITE(*,*) "JediVarNamesWithChannels(iChannel)=", JediVarNamesWithChannels(iChannel) call obsspace_get_db(ObsSpace, JediVarGroup, JediVarNamesWithChannels(iChannel), VarValue) where (VarValue /= MissingDouble) - Real2(:, varChannels(iChannel)) = VarValue + Real2(:, iChannel) = VarValue end where ! Fill the OPS data structures ! if (localUseActualChans) then @@ -1229,7 +1228,7 @@ end subroutine opsinputs_fill_fillreal2d_records !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield !> are not found. subroutine opsinputs_fill_fillreal2d( & - Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, Channels, varChannels, & + Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, Channels, & VarobsLength, JediVarName, JediVarGroup, OffsetChans, & useActualChans) implicit none @@ -1241,7 +1240,6 @@ subroutine opsinputs_fill_fillreal2d( & real(real64), pointer, intent(out) :: Real2(:,:) type(c_ptr), value, intent(in) :: ObsSpace integer(c_int), intent(in) :: Channels(:) -integer, intent(in) :: varChannels(:) integer(integer64), intent(in) :: VarobsLength character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup @@ -1257,16 +1255,16 @@ subroutine opsinputs_fill_fillreal2d( & if (Present(useActualChans)) then call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - varChannels, JediVarName, JediVarGroup, OffsetChans, useActualChans) + JediVarName, JediVarGroup, OffsetChans, useActualChans) else call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - varChannels, JediVarName, JediVarGroup, OffsetChans) + JediVarName, JediVarGroup, OffsetChans) end if else call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - varChannels, JediVarName, JediVarGroup) + JediVarName, JediVarGroup) end if end if diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 5cb9f672..071f2e0f 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -187,7 +187,7 @@ module opsinputs_varobswriter_mod integer(c_int), allocatable :: channels(:) ! integer(c_int), allocatable :: jopaChannels(:) - integer, allocatable :: varChannels(:) + integer(c_int), allocatable :: varChannels(:) !this stores the atmospheric levels we wish to pass to varobs integer(c_int), allocatable :: modlevs(:) @@ -868,7 +868,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_britemp) call opsinputs_fill_fillreal2d( & Ob % Header % CorBriTemp, "CorBriTemp", JediToOpsLayoutMapping, Ob % CorBriTemp, & - ObsSpace, self % channels, self % varChannels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue", & + ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue", & self % channel_offset, self % useActualChannels) case (VarField_tskin) call opsinputs_fill_fillelementtypefromnormalvariable( & @@ -885,7 +885,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_mwemiss) call opsinputs_fill_fillreal2d( & Ob % Header % MwEmiss, "MwEmiss", JediToOpsLayoutMapping, Ob % MwEmiss, & - ObsSpace, self % channels, self % varChannels, self % VarobsLength, "emissivity", "Emiss") + ObsSpace, self % channels, self % VarobsLength, "emissivity", "Emiss") case (VarField_TCozone) call opsinputs_fill_fillreal( & Ob % Header % TCozone, "TCozone", JediToOpsLayoutMapping, Ob % TCozone, & @@ -998,7 +998,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_Emissivity) call opsinputs_fill_fillreal2d( & Ob % Header % Emissivity, "Emissivity", JediToOpsLayoutMapping, Ob % Emissivity, & - ObsSpace, self % channels, self % varChannels, self % VarobsLength, "emissivity", "OneDVar", self % channel_offset) + ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar", self % channel_offset) case (VarField_QCinfo) ! TODO(someone): This will come from a variable generated by the 1D-Var filter. Its name and ! group are not known yet. Once they are, replace the placeholders in the call below. @@ -1029,7 +1029,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_RadarObAzim) call opsinputs_fill_fillreal2d( & Ob % Header % RadarObAzim, "RadarObAzim", JediToOpsLayoutMapping, Ob % RadarObAzim, & - ObsSpace, self % channels, self % varChannels, self % VarobsLength, "radarAzimuth", "MetaData") + ObsSpace, self % channels, self % VarobsLength, "radarAzimuth", "MetaData") case (VarField_RadIdent) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % RadIdent, "RadIdent", Ob % Header % NumObsLocal, Ob % RadIdent) @@ -1097,7 +1097,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_BriTempVarError) call opsinputs_fill_fillreal2d( & Ob % Header % BriTempVarError, "BriTempVarError", JediToOpsLayoutMapping, Ob % BriTempVarError, & - ObsSpace, self % channels, self % varChannels, self % VarobsLength, "brightnessTemperature",& + ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature",& "ObsErrorData", self % channel_offset) case (VarField_CloudRTError) ! TODO(someone): handle this varfield @@ -1116,11 +1116,11 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_LevelLat) call opsinputs_fill_fillreal2d( & Ob % Header % level_lat, "level_lat", JediToOpsLayoutMapping, Ob % level_lat, & - ObsSpace, self % channels, self % varChannels, self % VarobsLength, "latitude", "MetaData") + ObsSpace, self % channels, self % VarobsLength, "latitude", "MetaData") case (VarField_LevelLon) call opsinputs_fill_fillreal2d( & Ob % Header % level_lon, "level_lon", JediToOpsLayoutMapping, Ob % level_lon, & - ObsSpace, self % channels, self % varChannels, self % VarobsLength, "longitude", "MetaData") + ObsSpace, self % channels, self % VarobsLength, "longitude", "MetaData") case (VarField_RainAccum) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % RainAccum, "RainAccum", Ob % Header % NumObsLocal, Ob % RainAccum) @@ -1170,6 +1170,10 @@ subroutine opsinputs_varobswriter_populateobservations( & end select if (FillChanNum .or. FillNumChans) then + WRITE(*,*) "Call opsinputs_varobswriter_fillchannumandnumchans" + WRITE(*,*) size(self % varChannels) + WRITE(*,*) self % varChannels + WRITE(*,*) size(self % channels) call opsinputs_varobswriter_fillchannumandnumchans( & Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & FillNumChans, self % channel_offset % channel_offset, self % useActualChannels) @@ -1219,14 +1223,14 @@ end subroutine opsinputs_varobswriter_fillreportflags subroutine opsinputs_varobswriter_fillchannumandnumchans( & Ob, ObsSpace, channels, varChannels, Flags, FillChanNum, FillNumChans, & OffsetChans,UseActualChans) + implicit none ! Subroutine arguments: type(OB_type), intent(inout) :: Ob type(c_ptr), value, intent(in) :: ObsSpace integer(c_int), intent(in) :: channels(:) -!integer(c_int), intent(in) :: varChannels(:) -integer, intent(in) :: varChannels(:) +integer(c_int), intent(in) :: varChannels(:) type(c_ptr), value, intent(in) :: Flags logical, intent(in) :: FillChanNum, FillNumChans integer, intent(in) :: OffsetChans @@ -1240,29 +1244,34 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & integer :: iObs logical :: localUseActualChans - +WRITE(*,*) "Enter opsinputs_varobswriter_fillchannumandnumchans" ! Body: NumChannels = size(channels) if (NumChannels == 0) return call opsinputs_varobswriter_findchannelspassingqc( & Ob % Header % NumObsLocal, ObsSpace, channels, Flags, ChannelIndices, ChannelCounts) - +WRITE(*,*) "QC been done" if (FillChanNum) then call Ops_Alloc(Ob % Header % ChanNum, "ChanNum", Ob % Header % NumObsLocal, Ob % ChanNum, & num_levels = NumChannels) WRITE(*,*) "#### In if FillChnaNum" - write(*,*) size(channels) - write(*,*) size(varChannels) - if (size(channels) == size(varChannels)) then - do iChannel=1, NumChannels - WRITE(*,*) "VarChannels=", varChannels(iChannel) - ChannelIndices(:,iChannel) = varChannels(iChannel) - end do - +! if (size(varChannels) > 0) then + if (size(channels) == size(varChannels)) then + do iChannel=1, NumChannels + WRITE(*,*) "VarChannels=", varChannels(iChannel) + ChannelIndices(:,iChannel) = varChannels(iChannel) + end do + end if Ob % ChanNum = ChannelIndices - + ! else + ! Ob % ChanNum = ChannelIndices + !only apply offset to actual channels in list, not missing data + ! where (Ob % ChanNum > 0) + ! Ob % ChanNum = Ob % ChanNum + int(OffsetChans, kind=integer64) + ! end where end if + end if if (FillNumChans) then diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index cc5aa361..e130553d 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -165,7 +165,7 @@ observations: namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp general_mode: debug # size_of_varobs_array: 3 - varChannels: [5,7] + varChannels: 7, 8 output_directory: /home/h01/olewis/ - filter: VarObs Checker expected_main_table_columns: From 9160d8fd9416cab5e87d6593bac9bed1716d9644 Mon Sep 17 00:00:00 2001 From: orlewis Date: Wed, 2 Aug 2023 16:18:33 +0100 Subject: [PATCH 06/44] Strip out the offset channels and use_actual_channels. --- src/opsinputs/VarObsWriterParameters.h | 10 ---- src/opsinputs/opsinputs_fill_mod.F90 | 54 +++----------------- src/opsinputs/opsinputs_varobswriter_mod.F90 | 42 +++------------ 3 files changed, 14 insertions(+), 92 deletions(-) diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 2a34feb0..5a137428 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -79,21 +79,11 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// Update OPS flag to output the varbc predictors oops::Parameter outputVarBCPredictors{"output_varbc_predictors", false, this}; - /// This contains the offset that needs to be added to the channel number in order to - /// index the output arrays correctly. - oops::Parameter channel_offset{"channel_offset", 0, this}; - /// This is the size of the varobs array for output. The default is zero and the size /// of the array will be used. /// For atovs, jedi has 20 brightness temperatures but var expects 40. /// Therefore for atovs brightness_tmperatuere => size_of_varobs_array = 40. oops::Parameter size_of_varobs_array{"size_of_varobs_array", 0, this}; - - /// This matches the channel number with the array index. This is useful when - /// there are skipped channels. e.g. channels 5,6,7,9,10,11 are required from a possible 12 - /// channels. This would fill a size 12 array with the array indexes matching the channel number - /// [NaN,NaN,NaN,Nan,5,6,7,Nan,9,10,11,NaN]. - oops::Parameter use_actual_channels{"use_actual_channels", false, this}; /// oops::Parameter jopaChannels{"jopaChannels", "", this}; diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 01201e4c..14f924a3 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1035,7 +1035,7 @@ end subroutine opsinputs_fill_fillreal !> are not found. subroutine opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, NumObs, Real2, ObsSpace, Channels, JediVarName, & - JediVarGroup, OffsetChans, useActualChans) + JediVarGroup) implicit none ! Subroutine arguments: @@ -1047,8 +1047,6 @@ subroutine opsinputs_fill_fillreal2d_norecords( & integer(c_int), intent(in) :: Channels(:) character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup -type(opsinputs_channeloffset), optional, intent(in) :: OffsetChans -logical, optional, intent(in) :: useActualChans ! Local declarations: real(kind=c_double) :: VarValue(NumObs) @@ -1057,7 +1055,6 @@ subroutine opsinputs_fill_fillreal2d_norecords( & integer :: iChannel integer :: offset integer :: numchans -logical :: localUseActualChans ! Body: @@ -1068,20 +1065,11 @@ subroutine opsinputs_fill_fillreal2d_norecords( & !take into account offsetting of 2nd dimension if required !designed to be used to pack where multiple satellite instruments expected !e.g. HIRS in ATOVS stream -offset = 0 + numchans = size(JediVarNamesWithChannels) WRITE(*,*) "numchans=", numchans -if (present(OffsetChans)) then - offset = OffsetChans % channel_offset - if (OffsetChans % size_of_varobs_array > 0) & - numchans = OffsetChans % size_of_varobs_array -end if -!Setup for channels needing to match array index -localUseActualChans = .false. -if (present(useActualChans)) then - localUseActualChans = useActualChans -end if + if (obsspace_has(ObsSpace, JediVarGroup, JediVarNamesWithChannels(1))) then ! Allocate OPS data structures @@ -1094,18 +1082,6 @@ subroutine opsinputs_fill_fillreal2d_norecords( & where (VarValue /= MissingDouble) Real2(:, iChannel) = VarValue end where - ! Fill the OPS data structures -! if (localUseActualChans) then -! where (VarValue /= MissingDouble) -! Real2(:, Channels(iChannel)) = VarValue -! end where -! else -! ! Fill the OPS data structures -! where (VarValue /= MissingDouble) -! Real2(:, iChannel+offset) = VarValue -! end where -! end if - end do end if ! Data not present? OPS will produce a warning -- we don't need to duplicate it. end subroutine opsinputs_fill_fillreal2d_norecords @@ -1229,8 +1205,7 @@ end subroutine opsinputs_fill_fillreal2d_records !> are not found. subroutine opsinputs_fill_fillreal2d( & Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, Channels, & - VarobsLength, JediVarName, JediVarGroup, OffsetChans, & - useActualChans) + VarobsLength, JediVarName, JediVarGroup) implicit none ! Subroutine arguments: @@ -1243,29 +1218,16 @@ subroutine opsinputs_fill_fillreal2d( & integer(integer64), intent(in) :: VarobsLength character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup -type(opsinputs_channeloffset), optional, intent(in) :: OffsetChans -logical, optional, intent(in) :: useActualChans + ! Body: if (JediToOpsLayoutMapping % ConvertRecordsToMultilevelObs) then call opsinputs_fill_fillreal2d_records( & Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, VarobsLength, JediVarName, JediVarGroup) else - if (Present(OffsetChans)) then - if (Present(useActualChans)) then - call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, OffsetChans, useActualChans) - else - call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, OffsetChans) - end if - else - call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup) - end if + call opsinputs_fill_fillreal2d_norecords( & + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & + JediVarName, JediVarGroup) end if end subroutine opsinputs_fill_fillreal2d diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 071f2e0f..39f83187 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -192,11 +192,6 @@ module opsinputs_varobswriter_mod !this stores the atmospheric levels we wish to pass to varobs integer(c_int), allocatable :: modlevs(:) - !this stores the offset for channels when storing britemp - type(opsinputs_channeloffset) :: channel_offset - - logical :: useActualChannels - type(ufo_geovals), pointer :: GeoVals type(ufo_geovals), pointer :: ObsDiags end type opsinputs_varobswriter @@ -306,31 +301,11 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe call f_conf % get_or_die("varobs_length_is_IC_PLevels", self % VarobsLengthIsIC_PLevels) -! This contains the offset that needs to be added to the channel number in order to -! index the output arrays correctly. -call f_conf % get_or_die("channel_offset", self % channel_offset % channel_offset) - -! This is the size of the varobs array for output. The default is zero and the size of the array will be used. -! For atovs, jedi has 20 brightness temperatures but var expects 40. -! Therefore for atovs brightness_tmperatuere => size_of_varobs_array = 40. -call f_conf % get_or_die("size_of_varobs_array", self % channel_offset % size_of_varobs_array) - -! This defines if to use the channel numbers as the indices to fill an array by -! This prevents channels which are seperated by NaNs from being bunched together in the write -call f_conf % get_or_die("use_actual_channels", self % useActualChannels) - call f_conf % get_or_die("varChannels", self % varChannels) WRITE(*,*) "get or die varChannels", self % varChannels !call f_conf % get_or_die("jopaChannels", self % jopaChannels) -!if ((self % useActualChannels) .and. (self % channel_offset % channel_offset/=0)) then -! write (ErrorMessage, '(A)') "OffsetChans and UseActualChans cannot both be set" -! call gen_warn(RoutineName, ErrorMessage) -! opsinputs_varobswriter_create = .false. -! return -!end if - ! Updates the varbc flag passedaround by a module in OPS call f_conf % get_or_die("output_varbc_predictors", BoolValue) VarBC = BoolValue @@ -868,8 +843,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_britemp) call opsinputs_fill_fillreal2d( & Ob % Header % CorBriTemp, "CorBriTemp", JediToOpsLayoutMapping, Ob % CorBriTemp, & - ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue", & - self % channel_offset, self % useActualChannels) + ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue") case (VarField_tskin) call opsinputs_fill_fillelementtypefromnormalvariable( & Ob % Header % Tskin, "Tskin", Ob % Header % NumObsLocal, Ob % Tskin, & @@ -998,7 +972,7 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_Emissivity) call opsinputs_fill_fillreal2d( & Ob % Header % Emissivity, "Emissivity", JediToOpsLayoutMapping, Ob % Emissivity, & - ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar", self % channel_offset) + ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar") case (VarField_QCinfo) ! TODO(someone): This will come from a variable generated by the 1D-Var filter. Its name and ! group are not known yet. Once they are, replace the placeholders in the call below. @@ -1098,7 +1072,7 @@ subroutine opsinputs_varobswriter_populateobservations( & call opsinputs_fill_fillreal2d( & Ob % Header % BriTempVarError, "BriTempVarError", JediToOpsLayoutMapping, Ob % BriTempVarError, & ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature",& - "ObsErrorData", self % channel_offset) + "ObsErrorData") case (VarField_CloudRTError) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % CloudRTError, "CloudRTError", Ob % Header % NumObsLocal, Ob % CloudRTError) @@ -1176,7 +1150,7 @@ subroutine opsinputs_varobswriter_populateobservations( & WRITE(*,*) size(self % channels) call opsinputs_varobswriter_fillchannumandnumchans( & Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & - FillNumChans, self % channel_offset % channel_offset, self % useActualChannels) + FillNumChans) end if end do @@ -1221,8 +1195,7 @@ end subroutine opsinputs_varobswriter_fillreportflags !> e.g. for HIRS & AMSUA !> the number of these channels is stored in Ob % NumChans. subroutine opsinputs_varobswriter_fillchannumandnumchans( & - Ob, ObsSpace, channels, varChannels, Flags, FillChanNum, FillNumChans, & - OffsetChans,UseActualChans) + Ob, ObsSpace, channels, varChannels, Flags, FillChanNum, FillNumChans) implicit none @@ -1233,8 +1206,6 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & integer(c_int), intent(in) :: varChannels(:) type(c_ptr), value, intent(in) :: Flags logical, intent(in) :: FillChanNum, FillNumChans -integer, intent(in) :: OffsetChans -logical, optional, intent(in) :: UseActualChans ! Local declarations: integer(integer64) :: NumChannels @@ -1242,7 +1213,6 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & integer(integer64) :: ChannelCounts(Ob % Header % NumObsLocal) integer :: iChannel integer :: iObs -logical :: localUseActualChans WRITE(*,*) "Enter opsinputs_varobswriter_fillchannumandnumchans" ! Body: @@ -1270,7 +1240,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & ! where (Ob % ChanNum > 0) ! Ob % ChanNum = Ob % ChanNum + int(OffsetChans, kind=integer64) ! end where - end if + ! end if end if From 7f6eb8b3f962912de930f51e70f56e20f6722cf6 Mon Sep 17 00:00:00 2001 From: orlewis Date: Wed, 2 Aug 2023 17:50:55 +0100 Subject: [PATCH 07/44] This is an example of a setup which works. I think the Ob % ChanNum = ChannelIndices might have been in the wrong place. --- src/opsinputs/VarObsWriter.cc | 8 ++++---- src/opsinputs/VarObsWriter.h | 2 +- src/opsinputs/VarObsWriterParameters.h | 4 ++-- src/opsinputs/opsinputs_varobswriter_mod.F90 | 17 ++++++++++------- test/testinput/010_VarField_britemp.yaml | 2 +- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/opsinputs/VarObsWriter.cc b/src/opsinputs/VarObsWriter.cc index f813fdb9..f34f559e 100644 --- a/src/opsinputs/VarObsWriter.cc +++ b/src/opsinputs/VarObsWriter.cc @@ -18,7 +18,7 @@ #include "oops/base/Variables.h" #include "oops/mpi/mpi.h" #include "oops/util/Logger.h" -#include "oops/util/IntSetParser.h" +///#include "oops/util/IntSetParser.h" #include "opsinputs/LocalEnvironment.h" #include "opsinputs/VarObsWriterParameters.h" #include "ufo/filters/Variable.h" @@ -74,9 +74,9 @@ VarObsWriter::VarObsWriter(ioda::ObsSpace & obsdb, const Parameters_ & params, /// std::set chanset = oops::parseIntSet(options_.chanlist.value()); /// channels_.assign(chanset.begin(), chanset.end()); - std::set varchanset = oops::parseIntSet(parameters_.varChannels.value()); - varchannels_.assign(varchanset.begin(), varchanset.end()); - conf.set("varChannels", varchannels_); +/// std::set varchanset = oops::parseIntSet(parameters_.varChannels.value()); +/// varchannels_.assign(varchanset.begin(), varchanset.end()); +/// conf.set("varChannels", varchannels_); if (!opsinputs_varobswriter_create_f90(key_, &conf, fortranMpiCommunicatorIsValid, diff --git a/src/opsinputs/VarObsWriter.h b/src/opsinputs/VarObsWriter.h index ad901cfa..1eee0449 100644 --- a/src/opsinputs/VarObsWriter.h +++ b/src/opsinputs/VarObsWriter.h @@ -87,7 +87,7 @@ class VarObsWriter : public oops::interface::ObsFilterBase, oops::Variables extradiagvars_; std::shared_ptr> flags_; std::shared_ptr> obsErrors_; - std::vector varchannels_; +/// std::vector varchannels_; VarObsWriterParameters parameters_; }; diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 5a137428..dcdd151b 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -88,8 +88,8 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// oops::Parameter jopaChannels{"jopaChannels", "", this}; - /// oops::Parameter> varChannels{"varChannels", {}, this}; - oops::Parameter varChannels{"varChannels", "", this}; + oops::Parameter> varChannels{"varChannels", {}, this}; + /// oops::Parameter varChannels{"varChannels", "", this}; ///oops::RequiredParameter chanlist{"cost channels list", this}; diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 39f83187..caead76f 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -187,7 +187,8 @@ module opsinputs_varobswriter_mod integer(c_int), allocatable :: channels(:) ! integer(c_int), allocatable :: jopaChannels(:) - integer(c_int), allocatable :: varChannels(:) + ! integer(c_int), allocatable :: varChannels(:) + integer, allocatable :: varChannels(:) !this stores the atmospheric levels we wish to pass to varobs integer(c_int), allocatable :: modlevs(:) @@ -1144,10 +1145,10 @@ subroutine opsinputs_varobswriter_populateobservations( & end select if (FillChanNum .or. FillNumChans) then - WRITE(*,*) "Call opsinputs_varobswriter_fillchannumandnumchans" - WRITE(*,*) size(self % varChannels) - WRITE(*,*) self % varChannels - WRITE(*,*) size(self % channels) +! WRITE(*,*) "Call opsinputs_varobswriter_fillchannumandnumchans" +! WRITE(*,*) size(self % varChannels) +! WRITE(*,*) self % varChannels +! WRITE(*,*) size(self % channels) call opsinputs_varobswriter_fillchannumandnumchans( & Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & FillNumChans) @@ -1203,7 +1204,8 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & type(OB_type), intent(inout) :: Ob type(c_ptr), value, intent(in) :: ObsSpace integer(c_int), intent(in) :: channels(:) -integer(c_int), intent(in) :: varChannels(:) +!integer(c_int), intent(in) :: varChannels(:) +integer, intent(in) :: varChannels(:) type(c_ptr), value, intent(in) :: Flags logical, intent(in) :: FillChanNum, FillNumChans @@ -1232,8 +1234,9 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & WRITE(*,*) "VarChannels=", varChannels(iChannel) ChannelIndices(:,iChannel) = varChannels(iChannel) end do + Ob % ChanNum = ChannelIndices end if - Ob % ChanNum = ChannelIndices + ! else ! Ob % ChanNum = ChannelIndices !only apply offset to actual channels in list, not missing data diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index e130553d..91430df0 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -165,7 +165,7 @@ observations: namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp general_mode: debug # size_of_varobs_array: 3 - varChannels: 7, 8 + varChannels: [5,7] #7, 8 output_directory: /home/h01/olewis/ - filter: VarObs Checker expected_main_table_columns: From 3b70b72a12b565f9d28c84e383474e0f1ee28b9e Mon Sep 17 00:00:00 2001 From: orlewis Date: Wed, 2 Aug 2023 18:19:01 +0100 Subject: [PATCH 08/44] This is the setup that I think is required but this crashes at line 1151 in opsinputs_varobswriter_mod.F90 when going to the opsinputs_varobswriter_fillchannumandandnumchans function. --- src/opsinputs/VarObsWriter.cc | 8 ++++---- src/opsinputs/VarObsWriter.h | 2 +- src/opsinputs/VarObsWriterParameters.h | 4 ++-- src/opsinputs/opsinputs_varobswriter_mod.F90 | 14 ++++++-------- test/testinput/010_VarField_britemp.yaml | 2 +- 5 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/opsinputs/VarObsWriter.cc b/src/opsinputs/VarObsWriter.cc index f34f559e..f813fdb9 100644 --- a/src/opsinputs/VarObsWriter.cc +++ b/src/opsinputs/VarObsWriter.cc @@ -18,7 +18,7 @@ #include "oops/base/Variables.h" #include "oops/mpi/mpi.h" #include "oops/util/Logger.h" -///#include "oops/util/IntSetParser.h" +#include "oops/util/IntSetParser.h" #include "opsinputs/LocalEnvironment.h" #include "opsinputs/VarObsWriterParameters.h" #include "ufo/filters/Variable.h" @@ -74,9 +74,9 @@ VarObsWriter::VarObsWriter(ioda::ObsSpace & obsdb, const Parameters_ & params, /// std::set chanset = oops::parseIntSet(options_.chanlist.value()); /// channels_.assign(chanset.begin(), chanset.end()); -/// std::set varchanset = oops::parseIntSet(parameters_.varChannels.value()); -/// varchannels_.assign(varchanset.begin(), varchanset.end()); -/// conf.set("varChannels", varchannels_); + std::set varchanset = oops::parseIntSet(parameters_.varChannels.value()); + varchannels_.assign(varchanset.begin(), varchanset.end()); + conf.set("varChannels", varchannels_); if (!opsinputs_varobswriter_create_f90(key_, &conf, fortranMpiCommunicatorIsValid, diff --git a/src/opsinputs/VarObsWriter.h b/src/opsinputs/VarObsWriter.h index 1eee0449..ad901cfa 100644 --- a/src/opsinputs/VarObsWriter.h +++ b/src/opsinputs/VarObsWriter.h @@ -87,7 +87,7 @@ class VarObsWriter : public oops::interface::ObsFilterBase, oops::Variables extradiagvars_; std::shared_ptr> flags_; std::shared_ptr> obsErrors_; -/// std::vector varchannels_; + std::vector varchannels_; VarObsWriterParameters parameters_; }; diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index dcdd151b..5a137428 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -88,8 +88,8 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// oops::Parameter jopaChannels{"jopaChannels", "", this}; - oops::Parameter> varChannels{"varChannels", {}, this}; - /// oops::Parameter varChannels{"varChannels", "", this}; + /// oops::Parameter> varChannels{"varChannels", {}, this}; + oops::Parameter varChannels{"varChannels", "", this}; ///oops::RequiredParameter chanlist{"cost channels list", this}; diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index caead76f..5d1bb924 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -187,8 +187,7 @@ module opsinputs_varobswriter_mod integer(c_int), allocatable :: channels(:) ! integer(c_int), allocatable :: jopaChannels(:) - ! integer(c_int), allocatable :: varChannels(:) - integer, allocatable :: varChannels(:) + integer(c_int), allocatable :: varChannels(:) !this stores the atmospheric levels we wish to pass to varobs integer(c_int), allocatable :: modlevs(:) @@ -1145,10 +1144,10 @@ subroutine opsinputs_varobswriter_populateobservations( & end select if (FillChanNum .or. FillNumChans) then -! WRITE(*,*) "Call opsinputs_varobswriter_fillchannumandnumchans" -! WRITE(*,*) size(self % varChannels) -! WRITE(*,*) self % varChannels -! WRITE(*,*) size(self % channels) + WRITE(*,*) "Call opsinputs_varobswriter_fillchannumandnumchans" + WRITE(*,*) size(self % varChannels) + WRITE(*,*) self % varChannels + WRITE(*,*) size(self % channels) call opsinputs_varobswriter_fillchannumandnumchans( & Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & FillNumChans) @@ -1204,8 +1203,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & type(OB_type), intent(inout) :: Ob type(c_ptr), value, intent(in) :: ObsSpace integer(c_int), intent(in) :: channels(:) -!integer(c_int), intent(in) :: varChannels(:) -integer, intent(in) :: varChannels(:) +integer(c_int), intent(in) :: varChannels(:) type(c_ptr), value, intent(in) :: Flags logical, intent(in) :: FillChanNum, FillNumChans diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index 91430df0..5384b02c 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -165,7 +165,7 @@ observations: namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp general_mode: debug # size_of_varobs_array: 3 - varChannels: [5,7] #7, 8 + varChannels: 7, 8 #[5,7] output_directory: /home/h01/olewis/ - filter: VarObs Checker expected_main_table_columns: From d7281c69fa8e28f7d4743ab6d3ab49491e2b329f Mon Sep 17 00:00:00 2001 From: PJLevensMO Date: Thu, 3 Aug 2023 13:53:22 +0100 Subject: [PATCH 09/44] Change output dir to mine (plevens) and remove / at end of output_directory line --- test/testinput/010_VarField_britemp.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index 5384b02c..85a7b9a7 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -166,7 +166,7 @@ observations: general_mode: debug # size_of_varobs_array: 3 varChannels: 7, 8 #[5,7] - output_directory: /home/h01/olewis/ + output_directory: /home/h01/plevens - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. From 6c94212a015c679f46e7d5130af4eaec988a91ab Mon Sep 17 00:00:00 2001 From: PJLevensMO Date: Thu, 3 Aug 2023 15:06:11 +0100 Subject: [PATCH 10/44] Change path of another file to my own directory due to permission problems --- test/testinput/010_VarField_britemp.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index 85a7b9a7..6fdbccf3 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -140,7 +140,7 @@ observations: obsdataout: engine: type: H5File - obsfile: /home/h01/olewis/map_chans.nc4 + obsfile: /home/h01/plevens/map_chans.nc4 simulated variables: [brightnessTemperature] channels: 1,3 obs filters: From 4f4cb4d309e2c3ecb6cc1bb70c5632e36d18c00b Mon Sep 17 00:00:00 2001 From: PJLevensMO Date: Thu, 3 Aug 2023 16:55:10 +0100 Subject: [PATCH 11/44] remove output_directory line --- test/testinput/010_VarField_britemp.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index 6fdbccf3..e340664c 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -166,7 +166,7 @@ observations: general_mode: debug # size_of_varobs_array: 3 varChannels: 7, 8 #[5,7] - output_directory: /home/h01/plevens + #output_directory: /home/h01/plevens - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. From 475552c85a01d89b1b18527ad49c6481d23345cb Mon Sep 17 00:00:00 2001 From: PJLevensMO Date: Fri, 4 Aug 2023 09:49:23 +0100 Subject: [PATCH 12/44] Change to CallSign in yaml --- test/testinput/010_VarField_britemp.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index e340664c..3d4c1784 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -194,10 +194,10 @@ observations: "-3480.00000", "-3480.00000","-3480.00000", "-3420.00000", "-3420.00000","-3420.00000", "-3360.00000", "-3360.00000","-3360.00000"] - Callsign: ["station_1", "station_1","station_1", - "station_2", "station_2","station_2", - "station_3", "station_3","station_3", - "station_4", "station_4","station_4"] + Callsign: ["station_1", "station_1","station_1", "station_1","station_1", + "station_2", "station_2","station_2", "station_2","station_2", + "station_3", "station_3","station_3", "station_3","station_3", + "station_4", "station_4","station_4", "station_4","station_4"] HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 From 76eda7f39929bb55d79b4fe8cbb0ca1fa9760e46 Mon Sep 17 00:00:00 2001 From: PJLevensMO Date: Fri, 4 Aug 2023 11:04:09 +0100 Subject: [PATCH 13/44] Change to fields in yaml --- test/testinput/010_VarField_britemp.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index 3d4c1784..5ddcc431 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -170,10 +170,10 @@ observations: - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. - field: ["10", "10", "10", - "10", "10", "10", - "10", "10", "10", - "10", "10", "10"] + field: ["10", "10", "54", "55", "55", + "10", "10", "54", "55", "55", + "10", "10", "54", "55", "55", + "10", "10", "54", "55", "55"] level: ["5", "6","7", "5", "6","7", "5", "6","7", From d1dcd487b53f68af8496a491d03da585427e7a0e Mon Sep 17 00:00:00 2001 From: PJLevensMO Date: Fri, 4 Aug 2023 12:29:22 +0100 Subject: [PATCH 14/44] Changes to lat, lon, and time for ctest --- test/testinput/010_VarField_britemp.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index 5ddcc431..776c4e0d 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -182,18 +182,18 @@ observations: "-1073741824.00000","-1073741824.00000","3.22000", "1.60000","-1073741824.00000","3.33000", "1.80000","-1073741824.00000","3.44000"] - lat: ["21.00000", "21.00000","21.00000", - "22.00000", "22.00000","22.00000", - "-23.00000", "-23.00000","-23.00000", - "24.00000", "24.00000","24.00000"] - lon: ["31.00000", "31.00000","31.00000", - "32.00000", "32.00000","32.00000", - "33.00000", "33.00000","33.00000", - "34.00000", "34.00000","34.00000"] - time: ["-3540.00000", "-3540.00000","-3540.00000", - "-3480.00000", "-3480.00000","-3480.00000", - "-3420.00000", "-3420.00000","-3420.00000", - "-3360.00000", "-3360.00000","-3360.00000"] + lat: ["21.00000", "21.00000","21.00000", "21.00000","21.00000", + "22.00000", "22.00000","22.00000", "22.00000","22.00000", + "-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", + "24.00000", "24.00000","24.00000", "24.00000","24.00000"] + lon: ["31.00000", "31.00000","31.00000", "31.00000","31.00000", + "32.00000", "32.00000","32.00000", "32.00000","32.00000", + "33.00000", "33.00000","33.00000", "33.00000","33.00000", + "34.00000", "34.00000","34.00000", "34.00000","34.00000"] + time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000","-3540.00000", + "-3480.00000", "-3480.00000","-3480.00000", "-3480.00000","-3480.00000", + "-3420.00000", "-3420.00000","-3420.00000", "-3420.00000","-3420.00000", + "-3360.00000", "-3360.00000","-3360.00000", "-3360.00000","-3360.00000"] Callsign: ["station_1", "station_1","station_1", "station_1","station_1", "station_2", "station_2","station_2", "station_2","station_2", "station_3", "station_3","station_3", "station_3","station_3", From fb14ab90c63462be1114d81b9aaec42e27128d32 Mon Sep 17 00:00:00 2001 From: PJLevensMO Date: Mon, 7 Aug 2023 10:04:20 +0100 Subject: [PATCH 15/44] Change level field in ctest yaml --- test/testinput/010_VarField_britemp.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index 776c4e0d..e0333bfa 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -174,10 +174,10 @@ observations: "10", "10", "54", "55", "55", "10", "10", "54", "55", "55", "10", "10", "54", "55", "55"] - level: ["5", "6","7", - "5", "6","7", - "5", "6","7", - "5", "6","7"] + level: ["1","2","1","1","2", + "1","2","1","1","2", + "1","2","1","1","2", + "1","2","1","1","2"] ob value: ["1.20000","-1073741824.00000","3.11000", "-1073741824.00000","-1073741824.00000","3.22000", "1.60000","-1073741824.00000","3.33000", From 1ec327b84e10c229aefd8d02313286f47159ed68 Mon Sep 17 00:00:00 2001 From: PJLevensMO Date: Mon, 7 Aug 2023 11:35:21 +0100 Subject: [PATCH 16/44] Update ob value field in ctest yaml --- test/testinput/010_VarField_britemp.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index e0333bfa..cc439951 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -178,10 +178,10 @@ observations: "1","2","1","1","2", "1","2","1","1","2", "1","2","1","1","2"] - ob value: ["1.20000","-1073741824.00000","3.11000", - "-1073741824.00000","-1073741824.00000","3.22000", - "1.60000","-1073741824.00000","3.33000", - "1.80000","-1073741824.00000","3.44000"] + ob value: ["1.20000","3.11000","1.00000","7.00000","8.00000", + "-1073741824.00000","3.22000","2.00000","7.00000","8.00000", + "1.60000","3.33000","2.00000","7.00000","8.00000", + "1.80000","3.44000","1.00000","7.00000","8.00000"] lat: ["21.00000", "21.00000","21.00000", "21.00000","21.00000", "22.00000", "22.00000","22.00000", "22.00000","22.00000", "-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", From ff5a6b53ba1c31ad79978c463b99debbce7fffe7 Mon Sep 17 00:00:00 2001 From: PJLevensMO Date: Wed, 9 Aug 2023 14:05:45 +0100 Subject: [PATCH 17/44] Add size_of_varobs_array back to opsinputs_varobswriter_mod.f90 and add it as an explicit entry in opsinputs_varobswriter type instead of being inside the channel_offset type (which was removed) --- src/opsinputs/opsinputs_varobswriter_mod.F90 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 5d1bb924..7f89c2c6 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -188,6 +188,7 @@ module opsinputs_varobswriter_mod integer(c_int), allocatable :: channels(:) ! integer(c_int), allocatable :: jopaChannels(:) integer(c_int), allocatable :: varChannels(:) + integer(integer64) :: size_of_varobs_array !this stores the atmospheric levels we wish to pass to varobs integer(c_int), allocatable :: modlevs(:) @@ -301,6 +302,8 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe call f_conf % get_or_die("varobs_length_is_IC_PLevels", self % VarobsLengthIsIC_PLevels) +call f_conf % get_or_die("size_of_varobs_array", self % size_of_varobs_array) + call f_conf % get_or_die("varChannels", self % varChannels) WRITE(*,*) "get or die varChannels", self % varChannels From 8db38c3771ecbd9c2ceecfbe001dd697bcf3c735 Mon Sep 17 00:00:00 2001 From: orlewis Date: Mon, 21 Aug 2023 14:46:54 +0100 Subject: [PATCH 18/44] This is a bit messy and may need to be reverted but handy to see what changes have been made. --- src/opsinputs/VarObsWriter.cc | 3 +- src/opsinputs/VarObsWriter.h | 2 + src/opsinputs/VarObsWriterParameters.h | 15 +- src/opsinputs/opsinputs_fill_mod.F90 | 113 +++++++++-- src/opsinputs/opsinputs_utils_mod.F90 | 8 + src/opsinputs/opsinputs_varobswriter_mod.F90 | 91 +++++++-- test/testinput/010_VarField_britemp.yaml | 182 +++++++++--------- test/testinput/055_VarField_channum.yaml | 4 +- test/testinput/057_VarField_emissivity.yaml | 3 +- .../076_VarField_britempvarerror.yaml | 3 +- test/testinput/varobs_globalnamelist_atms.nc4 | Bin 16944 -> 16944 bytes test/testinput/varobs_globalnamelist_iasi.nc4 | Bin 20585 -> 20585 bytes .../varobswriter_globalnamelist_atovs.yaml | 3 +- 13 files changed, 287 insertions(+), 140 deletions(-) diff --git a/src/opsinputs/VarObsWriter.cc b/src/opsinputs/VarObsWriter.cc index f813fdb9..56893d29 100644 --- a/src/opsinputs/VarObsWriter.cc +++ b/src/opsinputs/VarObsWriter.cc @@ -17,8 +17,8 @@ #include "ioda/ObsVector.h" #include "oops/base/Variables.h" #include "oops/mpi/mpi.h" -#include "oops/util/Logger.h" #include "oops/util/IntSetParser.h" +#include "oops/util/Logger.h" #include "opsinputs/LocalEnvironment.h" #include "opsinputs/VarObsWriterParameters.h" #include "ufo/filters/Variable.h" @@ -77,6 +77,7 @@ VarObsWriter::VarObsWriter(ioda::ObsSpace & obsdb, const Parameters_ & params, std::set varchanset = oops::parseIntSet(parameters_.varChannels.value()); varchannels_.assign(varchanset.begin(), varchanset.end()); conf.set("varChannels", varchannels_); + conf.set("NumVarChannels", sizeof(varchannels_)); if (!opsinputs_varobswriter_create_f90(key_, &conf, fortranMpiCommunicatorIsValid, diff --git a/src/opsinputs/VarObsWriter.h b/src/opsinputs/VarObsWriter.h index ad901cfa..585dbb42 100644 --- a/src/opsinputs/VarObsWriter.h +++ b/src/opsinputs/VarObsWriter.h @@ -8,7 +8,9 @@ #include #include +#include #include +#include #include "ioda/ObsDataVector.h" #include "oops/base/Variables.h" diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 5a137428..9e2abdc4 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -84,14 +84,15 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// For atovs, jedi has 20 brightness temperatures but var expects 40. /// Therefore for atovs brightness_tmperatuere => size_of_varobs_array = 40. oops::Parameter size_of_varobs_array{"size_of_varobs_array", 0, this}; - - - /// oops::Parameter jopaChannels{"jopaChannels", "", this}; - - /// oops::Parameter> varChannels{"varChannels", {}, this}; + + /// oops::Parameter jopaChannels{"jopaChannels", "", this}; + + /// oops::Parameter> varChannels{"varChannels", {}, this}; oops::Parameter varChannels{"varChannels", "", this}; - ///oops::RequiredParameter chanlist{"cost channels list", this}; - + /// oops::RequiredParameter chanlist{"cost channels list", this}; + + /// Compress the var channels + oops::Parameter compressVarChannels{"compress_var_channels", false, this}; /// If this list of ufo::variable is defined in the yaml a subset of the flags /// will be made with just these variables present. This will allow Fortran calls such-as diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 14f924a3..1e0869c7 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -35,7 +35,8 @@ module opsinputs_fill_mod opsinputs_jeditoopslayoutmapping use opsinputs_utils_mod, only: & max_varname_with_channel_length, & - opsinputs_channeloffset + opsinputs_channeloffset, & + opsinputs_varchannels use GenMod_Core, only: & gen_warn, & @@ -1035,7 +1036,7 @@ end subroutine opsinputs_fill_fillreal !> are not found. subroutine opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, NumObs, Real2, ObsSpace, Channels, JediVarName, & - JediVarGroup) + JediVarGroup, sizeVarObs, varChannels) implicit none ! Subroutine arguments: @@ -1047,6 +1048,9 @@ subroutine opsinputs_fill_fillreal2d_norecords( & integer(c_int), intent(in) :: Channels(:) character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup +integer(integer64), optional, intent(in) :: sizeVarObs +integer(c_int), optional, intent(in) :: varChannels(:) + ! Local declarations: real(kind=c_double) :: VarValue(NumObs) @@ -1057,7 +1061,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & integer :: numchans ! Body: - +WRITE(*,*) "opsinputs_fill_mod - no records" MissingDouble = missing_value(0.0_c_double) JediVarNamesWithChannels = opsinputs_fill_varnames_with_channels(JediVarName, Channels) @@ -1068,8 +1072,24 @@ subroutine opsinputs_fill_fillreal2d_norecords( & numchans = size(JediVarNamesWithChannels) WRITE(*,*) "numchans=", numchans - - +WRITE(*,*) "Present", present(sizeVarObs), sizeVarObs +if ((present(varChannels)) .and. (size(varChannels) > 0)) then + numchans = sizeVarObs +end if +WRITE(*,*) "SizeVarObs=", sizeVarObs +!WRITE(*,*) sizeVarObs == 0 +!WRITE(*,*) sizeVarObs /= 0 +!if (sizeVarObs /= 0) then + ! numchans = sizeVarObs + ! WRITE(*,*) "Numchan set", numchans +!end if +WRITE(*,*) "numchans=", numchans +if (present(sizeVarObs) )then + if (sizeVarObs > 0) then + numchans = sizeVarObs + end if +end if +WRITE(*,*) "numchans=", numchans if (obsspace_has(ObsSpace, JediVarGroup, JediVarNamesWithChannels(1))) then ! Allocate OPS data structures @@ -1077,11 +1097,29 @@ subroutine opsinputs_fill_fillreal2d_norecords( & num_levels = int(numchans, kind=integer64)) do iChannel = 1, size(JediVarNamesWithChannels) ! Retrieve data from JEDI - WRITE(*,*) "JediVarNamesWithChannels(iChannel)=", JediVarNamesWithChannels(iChannel) - call obsspace_get_db(ObsSpace, JediVarGroup, JediVarNamesWithChannels(iChannel), VarValue) - where (VarValue /= MissingDouble) - Real2(:, iChannel) = VarValue - end where + if (iChannel <= size(JediVarNamesWithChannels)) then + WRITE(*,*) "JediVarNamesWithChannels(iChannel)=", JediVarNamesWithChannels(iChannel) + call obsspace_get_db(ObsSpace, JediVarGroup, JediVarNamesWithChannels(iChannel), VarValue) + end if + + + if (present(varChannels)) then + if (size(varChannels) > 0) then + WRITE(*,*) "here" + if (iChannel <= size(varChannels)) then + WRITE(*,*) "here1" + where (VarValue /= MissingDouble) + Real2(:, varChannels(iChannel)) = VarValue + end where + WRITE(*,*) "shape of real2", shape(Real2) + end if + end if + else + WRITE(*,*) "real2 novar" + where (VarValue /= MissingDouble) + Real2(:, iChannel) = VarValue + end where + end if end do end if ! Data not present? OPS will produce a warning -- we don't need to duplicate it. end subroutine opsinputs_fill_fillreal2d_norecords @@ -1205,7 +1243,7 @@ end subroutine opsinputs_fill_fillreal2d_records !> are not found. subroutine opsinputs_fill_fillreal2d( & Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, Channels, & - VarobsLength, JediVarName, JediVarGroup) + VarobsLength, JediVarName, JediVarGroup, sizeVarObs, varChannels) implicit none ! Subroutine arguments: @@ -1218,6 +1256,9 @@ subroutine opsinputs_fill_fillreal2d( & integer(integer64), intent(in) :: VarobsLength character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup +integer(integer64), optional, intent(in) :: sizeVarObs +integer(c_int), optional, intent(in) :: varChannels(:) + ! Body: @@ -1225,9 +1266,18 @@ subroutine opsinputs_fill_fillreal2d( & call opsinputs_fill_fillreal2d_records( & Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, VarobsLength, JediVarName, JediVarGroup) else - call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup) + if (size(varChannels) > 0) then + WRITE(*,*) "opsinputs_fill_mod - varchannel option" + call opsinputs_fill_fillreal2d_norecords( & + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & + JediVarName, JediVarGroup, sizeVarObs, varChannels) + else + WRITE(*,*) "##In no VarChannels" + WRITE(*,*) "sizeVarObs", sizeVarObs + call opsinputs_fill_fillreal2d_norecords( & + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & + JediVarName, JediVarGroup, sizeVarObs) + end if end if end subroutine opsinputs_fill_fillreal2d @@ -1686,6 +1736,7 @@ subroutine opsinputs_fill_fillstring( & ! Fill the OPS data structures call Ops_Alloc(Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, String1) + WRITE(*,*) "loop nubmers", JediToOpsLayoutMapping % NumOpsObs do i = 1, JediToOpsLayoutMapping % NumOpsObs if (JediToOpsLayoutMapping % ConvertRecordsToMultilevelObs) then if (JediToOpsLayoutMapping % RecordStarts(i + 1) > JediToOpsLayoutMapping % RecordStarts(i)) then @@ -1694,6 +1745,7 @@ subroutine opsinputs_fill_fillstring( & JediToOpsLayoutMapping % RecordStarts(i))) end if else + WRITE(*,*) "VarValue(i)=", VarValue(i), i String1(i) = VarValue(i) end if end do @@ -2249,13 +2301,38 @@ function opsinputs_fill_varnames_with_channels(VarName, Channels) result(VarName character(len=max_varname_with_channel_length) :: VarNames(max(size(Channels), 1)) integer :: ichan -if (size(Channels) == 0) then - VarNames(1) = VarName -else +character(len=max_varname_with_channel_length) :: VarNames_emis(max(size(Channels), 1)) + +write(*,*) max_varname_with_channel_length +WRITE(*,*) "VarNames=", VarNames +WRITE(*,*) "opsinputs_fill_mod - opsinputs_fill_varnames_with_channels" +WRITE(*,*) size(Channels), VarName + +if (Varname=="emissivity") then do ichan = 1, size(Channels) - write (VarNames(ichan),'(A,"_",I0)') VarName, Channels(ichan) + write(*,*) "here" + write(*,*) VarName, Channels(ichan) + write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) + ! write(*,'(A,"_",I0)') VarName, Channels(ichan) + ! write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) end do + ! VarNames = VarNames_emis +else + if (size(Channels) == 0) then + VarNames(1) = VarName + else + write(*,*) "start loop" + do ichan = 1, size(Channels) + + WRITE(*,*) VarName, Channels(ichan) + WRITE(*,*) size(VarNames), VarNames + write (VarNames(ichan),'(A,"_",I0)') VarName, Channels(ichan) + write(*,*) VarNames(ichan) + WRITE(*,*) "In loop", VarNames(ichan) + end do + end if end if +WRITE(*,*) "opsinputs_fill_mod - opsinputs_fill_varnames_with_channels finished" end function opsinputs_fill_varnames_with_channels diff --git a/src/opsinputs/opsinputs_utils_mod.F90 b/src/opsinputs/opsinputs_utils_mod.F90 index b05a9b31..20d78739 100644 --- a/src/opsinputs/opsinputs_utils_mod.F90 +++ b/src/opsinputs/opsinputs_utils_mod.F90 @@ -40,6 +40,14 @@ module opsinputs_utils_mod integer :: size_of_varobs_array end type opsinputs_channeloffset + +! Type to contain the channels offset when converting from JEDI to VAR. +type, public :: opsinputs_varchannels + ! This is the size of the varobs array for output. The default is zero and the size of the array will be used. + ! For atovs, jedi has 20 brightness temperatures but var expects 40. + ! Therefore for atovs brightness_tmperatuere => size_of_varobs_array = 40. + integer :: size_of_varobs_array +end type opsinputs_varchannels ! ------------------------------------------------------------------------------ contains diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 7f89c2c6..ab78f38f 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -189,6 +189,7 @@ module opsinputs_varobswriter_mod ! integer(c_int), allocatable :: jopaChannels(:) integer(c_int), allocatable :: varChannels(:) integer(integer64) :: size_of_varobs_array + integer(integer64) :: NumVarChannels !this stores the atmospheric levels we wish to pass to varobs integer(c_int), allocatable :: modlevs(:) @@ -233,6 +234,9 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe allocate(self % channels(size(channels))) self % channels(:) = channels(:) +call f_conf % get_or_die("NumVarChannels", self % NumVarChannels) +allocate(self % varChannels(self % NumVarChannels)) +call f_conf % get_or_die("varChannels", self % varChannels) ! Setup OPS call f_conf % get_or_die("general_mode", StringValue) @@ -280,6 +284,7 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe opsinputs_varobswriter_create = .false. return end if + call datetime_create(StringValue, self % validitytime) call f_conf % get_or_die("reject_obs_with_any_variable_failing_qc", & @@ -304,8 +309,8 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe call f_conf % get_or_die("size_of_varobs_array", self % size_of_varobs_array) -call f_conf % get_or_die("varChannels", self % varChannels) -WRITE(*,*) "get or die varChannels", self % varChannels +!call f_conf % get_or_die("varChannels", self % varChannels) +!WRITE(*,*) "get or die varChannels", self % varChannels !call f_conf % get_or_die("jopaChannels", self % jopaChannels) @@ -563,12 +568,15 @@ subroutine opsinputs_varobswriter_post( & call opsinputs_varobswriter_allocateobservations(self, ObsSpace, JediToOpsLayoutMapping, Ob) call opsinputs_varobswriter_populateobservations(self, ObsSpace, JediToOpsLayoutMapping, & Flags, ObsErrors, Ob) +WRITE(*,*) "Finished populate Ob" call opsinputs_varobswriter_populatecxheader(self, CxHeader) +WRITE(*,*) "Finished populate cx" call Ops_CreateVarobs (Ob, & ! in CxHeader, & ! in AssimDataFormat_VAR, & NumVarobsTotal) +WRITE(*,*) "Finished CreatVarobs" call Ob % deallocate() call CxHeader % dealloc() @@ -731,6 +739,8 @@ subroutine opsinputs_varobswriter_populateobservations( & if (obsspace_has(ObsSpace, "MetaData", "stationIdentification")) then call opsinputs_fill_fillstring(Ob % Header % Callsign, "Callsign", JediToOpsLayoutMapping, & LenCallSign, Ob % Callsign, ObsSpace, "stationIdentification", "MetaData") + WRITE(*,*) "LenCallSign=", LenCallSign + WRITE(*,*) Ob % Callsign else if (obsspace_has(ObsSpace, "MetaData", "satelliteIdentifier")) then call opsinputs_varobswriter_fillsatid(Ob, ObsSpace, JediToOpsLayoutMapping) call Ops_Alloc(Ob % Header % Callsign, "Callsign", JediToOpsLayoutMapping % NumOpsObs, Ob % Callsign) @@ -844,9 +854,11 @@ subroutine opsinputs_varobswriter_populateobservations( & ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % lwp, "LWP", Ob % Header % NumObsLocal, Ob % lwp) case (VarField_britemp) + WRITE(*,*) "At britemp" call opsinputs_fill_fillreal2d( & Ob % Header % CorBriTemp, "CorBriTemp", JediToOpsLayoutMapping, Ob % CorBriTemp, & - ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue") + ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue", & + self % size_of_varobs_array, self % varChannels) case (VarField_tskin) call opsinputs_fill_fillelementtypefromnormalvariable( & Ob % Header % Tskin, "Tskin", Ob % Header % NumObsLocal, Ob % Tskin, & @@ -860,9 +872,10 @@ subroutine opsinputs_varobswriter_populateobservations( & Ob % Header % Zstation, "Zstation", JediToOpsLayoutMapping, Ob % Zstation, & ObsSpace, "stationElevation", "MetaData") case (VarField_mwemiss) + WRITE(*,*) "At emissivity" call opsinputs_fill_fillreal2d( & Ob % Header % MwEmiss, "MwEmiss", JediToOpsLayoutMapping, Ob % MwEmiss, & - ObsSpace, self % channels, self % VarobsLength, "emissivity", "Emiss") + ObsSpace, self % channels, self % VarobsLength, "emissivity", "Emiss", self % size_of_varobs_array, self % varChannels) case (VarField_TCozone) call opsinputs_fill_fillreal( & Ob % Header % TCozone, "TCozone", JediToOpsLayoutMapping, Ob % TCozone, & @@ -975,7 +988,8 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_Emissivity) call opsinputs_fill_fillreal2d( & Ob % Header % Emissivity, "Emissivity", JediToOpsLayoutMapping, Ob % Emissivity, & - ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar") + ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar", self % size_of_varobs_array, & + self % varChannels) case (VarField_QCinfo) ! TODO(someone): This will come from a variable generated by the 1D-Var filter. Its name and ! group are not known yet. Once they are, replace the placeholders in the call below. @@ -1075,7 +1089,7 @@ subroutine opsinputs_varobswriter_populateobservations( & call opsinputs_fill_fillreal2d( & Ob % Header % BriTempVarError, "BriTempVarError", JediToOpsLayoutMapping, Ob % BriTempVarError, & ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature",& - "ObsErrorData") + "ObsErrorData", self % size_of_varobs_array, self % varChannels) case (VarField_CloudRTError) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % CloudRTError, "CloudRTError", Ob % Header % NumObsLocal, Ob % CloudRTError) @@ -1148,12 +1162,13 @@ subroutine opsinputs_varobswriter_populateobservations( & if (FillChanNum .or. FillNumChans) then WRITE(*,*) "Call opsinputs_varobswriter_fillchannumandnumchans" - WRITE(*,*) size(self % varChannels) + WRITE(*,*) "size(self % varChannels) =", size(self % varChannels) WRITE(*,*) self % varChannels - WRITE(*,*) size(self % channels) + WRITE(*,*) "size(self % channels)=", size(self % channels) + WRITE(*,*) self % size_of_varobs_array call opsinputs_varobswriter_fillchannumandnumchans( & Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & - FillNumChans) + FillNumChans, self % size_of_varobs_array) end if end do @@ -1198,7 +1213,7 @@ end subroutine opsinputs_varobswriter_fillreportflags !> e.g. for HIRS & AMSUA !> the number of these channels is stored in Ob % NumChans. subroutine opsinputs_varobswriter_fillchannumandnumchans( & - Ob, ObsSpace, channels, varChannels, Flags, FillChanNum, FillNumChans) + Ob, ObsSpace, channels, varChannels, Flags, FillChanNum, FillNumChans, varObsSize) implicit none @@ -1209,35 +1224,71 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & integer(c_int), intent(in) :: varChannels(:) type(c_ptr), value, intent(in) :: Flags logical, intent(in) :: FillChanNum, FillNumChans +integer(integer64), intent(in) :: varObsSize ! Local declarations: integer(integer64) :: NumChannels +integer(integer64) :: ChannelIndicesVar(Ob % Header % NumObsLocal, size(varChannels)) !varObssize integer(integer64) :: ChannelIndices(Ob % Header % NumObsLocal, size(channels)) integer(integer64) :: ChannelCounts(Ob % Header % NumObsLocal) integer :: iChannel integer :: iObs - +real(kind=c_double) :: MissingDouble WRITE(*,*) "Enter opsinputs_varobswriter_fillchannumandnumchans" ! Body: + +MissingDouble = missing_value(0.0_c_double) + NumChannels = size(channels) if (NumChannels == 0) return call opsinputs_varobswriter_findchannelspassingqc( & Ob % Header % NumObsLocal, ObsSpace, channels, Flags, ChannelIndices, ChannelCounts) WRITE(*,*) "QC been done" +WRITE(*,*) "Initial ChannelIndices=", ChannelIndices(:,1) +WRITE(*,*) "Initial ChannelIndices=", ChannelIndices(:,2) + +ChannelIndicesVar(:,:) = 0 +WRITE(*,*) "orig ChannelIndicesVar", ChannelIndicesVar +WRITE(*,*) "size of varChannels", size(varChannels) +!WRITE(*,*) ChannelIndicesVar(:,1) +WRITE(*,*) "FillChanNum", FillChanNum if (FillChanNum) then call Ops_Alloc(Ob % Header % ChanNum, "ChanNum", Ob % Header % NumObsLocal, Ob % ChanNum, & num_levels = NumChannels) WRITE(*,*) "#### In if FillChnaNum" -! if (size(varChannels) > 0) then + if (size(varChannels) > 0) then if (size(channels) == size(varChannels)) then - do iChannel=1, NumChannels - WRITE(*,*) "VarChannels=", varChannels(iChannel) - ChannelIndices(:,iChannel) = varChannels(iChannel) + WRITE(*,*) "Number of channels=", NumChannels + WRITE(*,*) "#varObsSize", varObsSize + WRITE(*,*) Ob % Header % NumObsLocal + do iChannel=1, NumChannels !varObsSize + WRITE(*,*) "iChannel=", iChannel, size(varChannels) + ! if ((iChannel > size(varChannels))) then + ! WRITE(*,*) "in missing", MissingDouble + ! ChannelIndicesVar(:, iChannel) = MissingDouble + ! WRITE(*,*) "Going to continue" + ! continue + ! end if + + do iObs=1, Ob % Header % NumObsLocal + WRITE(*,*) "VarChannels=", varChannels(iChannel) + if (ChannelIndices(iObs,iChannel) /= 0) then + ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) + end if + + + WRITE(*,*) "ChannelIndicesVar1=", ChannelIndicesVar(:,1) + WRITE(*,*) "ChannelIndicesVar2=", ChannelIndicesVar(:,2) + WRITE(*,*) "ChannelIndicesVar3=", ChannelIndicesVar(:,3) + end do end do - Ob % ChanNum = ChannelIndices + Ob % ChanNum = ChannelIndicesVar end if - + else + WRITE(*,*) "Set ChanNum to ChannelIndices" + Ob % ChanNum = ChannelIndices + end if ! else ! Ob % ChanNum = ChannelIndices !only apply offset to actual channels in list, not missing data @@ -1245,13 +1296,15 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & ! Ob % ChanNum = Ob % ChanNum + int(OffsetChans, kind=integer64) ! end where ! end if - +else + WRITE(*,*) "Not FillChanNum" end if - +!WRITE(*,*) Ob % ChanNum if (FillNumChans) then call Ops_Alloc(Ob % Header % NumChans, "NumChans", Ob % Header % NumObsLocal, Ob % NumChans) Ob % NumChans = ChannelCounts end if +WRITE(*,*) "Finished fillchans" end subroutine opsinputs_varobswriter_fillchannumandnumchans ! ------------------------------------------------------------------------------ diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index cc439951..4c70a920 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -2,63 +2,63 @@ window begin: 2018-01-01T00:00:00Z window end: 2018-01-01T02:00:00Z observations: -# - obs space: -# name: AMSUB -# obsdatain: -# engine: -# type: H5File -# obsfile: Data/010_VarField_britemp.nc4 -# simulated variables: [brightnessTemperature] -# channels: 1, 3 -# obs filters: -# # Set the flag of observations with missing values to "pass": we want to check if these -# # values are encoded correctly in the VarObsFile. -# - filter: Reset Flags to Pass -# flags_to_reset: [10, 15] # missing, Hfailed -# # Reject observation 3: we want to check if it is omitted from the VarObs file, as expected. -# - filter: Domain Check -# where: -# - variable: -# name: MetaData/latitude -# minvalue: 0.0 -# - filter: VarObs Writer -# namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp -# general_mode: debug -# - filter: VarObs Checker -# expected_main_table_columns: -# # Only levels 1 and 3 are simulated and hence written to the VarObs file. -# # In the arrays below, rows denote locations and columns channels. -# field: ["10", "10", -# "10", "10", -# "10", "10", -# "10", "10"] -# level: ["1", "2", -# "1", "2", -# "1", "2", -# "1", "2"] -# ob value: ["1.20000", "3.11000", -# "-1073741824.00000", "3.22000", -# "1.60000", "3.33000", -# "1.80000", "3.44000"] -# lat: ["21.00000", "21.00000", -# "22.00000", "22.00000", -# "-23.00000", "-23.00000", -# "24.00000", "24.00000"] -# lon: ["31.00000", "31.00000", -# "32.00000", "32.00000", -# "33.00000", "33.00000", -# "34.00000", "34.00000"] -# time: ["-3540.00000", "-3540.00000", -# "-3480.00000", "-3480.00000", -# "-3420.00000", "-3420.00000", -# "-3360.00000", "-3360.00000"] -# Callsign: ["station_1", "station_1", -# "station_2", "station_2", -# "station_3", "station_3", -# "station_4", "station_4"] -# HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. -# benchmarkFlag: 1000 # just to keep the ObsFilters test happy -# flaggedBenchmark: 0 + - obs space: + name: AMSUB + obsdatain: + engine: + type: H5File + obsfile: Data/010_VarField_britemp.nc4 + simulated variables: [brightnessTemperature] + channels: 1, 3 + obs filters: + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + # Reject observation 3: we want to check if it is omitted from the VarObs file, as expected. + - filter: Domain Check + where: + - variable: + name: MetaData/latitude + minvalue: 0.0 + - filter: VarObs Writer + namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp + general_mode: debug + - filter: VarObs Checker + expected_main_table_columns: + # Only levels 1 and 3 are simulated and hence written to the VarObs file. + # In the arrays below, rows denote locations and columns channels. + field: ["10", "10","54","55","55", + "10", "10","54","55","55", + "10", "10","54","55","55", + "10", "10","54","55","55"] + level: ["1", "2","1","1","2", + "1", "2","1","1","2", + "1", "2","1","1","2", + "1", "2","1","1","2"] + ob value: ["1.20000", "3.11000", "2.00000","1.00000","2.00000", + "-1073741824.00000", "3.22000","2.00000","1.00000","2.00000", + "1.60000", "3.33000","0.00000","0.00000","0.00000", + "1.80000", "3.44000","2.00000","1.00000","2.00000"] + lat: ["21.00000", "21.00000","21.00000", "21.00000","21.00000", + "22.00000", "22.00000","22.00000", "22.00000","22.00000", + "-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", + "24.00000", "24.00000","24.00000", "24.00000","24.00000"] + lon: ["31.00000", "31.00000","31.00000", "31.00000","31.00000", + "32.00000", "32.00000","32.00000", "32.00000","32.00000", + "33.00000", "33.00000","33.00000", "33.00000","33.00000", + "34.00000", "34.00000","34.00000", "34.00000","34.00000"] + time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000","-3540.00000", + "-3480.00000", "-3480.00000","-3480.00000", "-3480.00000","-3480.00000", + "-3420.00000", "-3420.00000","-3420.00000", "-3420.00000","-3420.00000", + "-3360.00000", "-3360.00000","-3360.00000", "-3360.00000","-3360.00000"] + Callsign: ["station_1", "station_1","station_1", "station_1","station_1", + "station_2", "station_2","station_2", "station_2","station_2", + "station_3", "station_3","station_3", "station_3","station_3", + "station_4", "station_4","station_4", "station_4","station_4"] + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 ## ## Now we test effect of actual channel @@ -94,7 +94,7 @@ observations: # - filter: VarObs Checker # expected_main_table_columns: # # In the arrays below, rows denote locations and columns levels. -# field: ["10", "10", "10", +# field: ["10", "10", "10","54","55","55","55", # "10", "10", "10", # "10", "10", "10", # "10", "10", "10"] @@ -136,11 +136,11 @@ observations: obsdatain: engine: type: H5File - obsfile: Data/010_VarField_britemp.nc4 + obsfile: /data/users/olewis/JEDI_branches/test_geo_opsinputs/build-mo-bundle-jopa_env-RelWithDebInfo/opsinputs/test/Data/010_VarField_britemp.nc4 #Data/010_VarField_britemp.nc4 obsdataout: engine: type: H5File - obsfile: /home/h01/plevens/map_chans.nc4 + obsfile: /home/h01/olewis/map_chans.nc4 simulated variables: [brightnessTemperature] channels: 1,3 obs filters: @@ -164,40 +164,40 @@ observations: channels: 1,3 namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp general_mode: debug - # size_of_varobs_array: 3 - varChannels: 7, 8 #[5,7] - #output_directory: /home/h01/plevens + size_of_varobs_array: 4 + varChannels: 2, 4 #[5,7] + # output_directory: /home/h01/olewis - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. - field: ["10", "10", "54", "55", "55", - "10", "10", "54", "55", "55", - "10", "10", "54", "55", "55", - "10", "10", "54", "55", "55"] - level: ["1","2","1","1","2", - "1","2","1","1","2", - "1","2","1","1","2", - "1","2","1","1","2"] - ob value: ["1.20000","3.11000","1.00000","7.00000","8.00000", - "-1073741824.00000","3.22000","2.00000","7.00000","8.00000", - "1.60000","3.33000","2.00000","7.00000","8.00000", - "1.80000","3.44000","1.00000","7.00000","8.00000"] - lat: ["21.00000", "21.00000","21.00000", "21.00000","21.00000", - "22.00000", "22.00000","22.00000", "22.00000","22.00000", - "-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", - "24.00000", "24.00000","24.00000", "24.00000","24.00000"] - lon: ["31.00000", "31.00000","31.00000", "31.00000","31.00000", - "32.00000", "32.00000","32.00000", "32.00000","32.00000", - "33.00000", "33.00000","33.00000", "33.00000","33.00000", - "34.00000", "34.00000","34.00000", "34.00000","34.00000"] - time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000","-3540.00000", - "-3480.00000", "-3480.00000","-3480.00000", "-3480.00000","-3480.00000", - "-3420.00000", "-3420.00000","-3420.00000", "-3420.00000","-3420.00000", - "-3360.00000", "-3360.00000","-3360.00000", "-3360.00000","-3360.00000"] - Callsign: ["station_1", "station_1","station_1", "station_1","station_1", - "station_2", "station_2","station_2", "station_2","station_2", - "station_3", "station_3","station_3", "station_3","station_3", - "station_4", "station_4","station_4", "station_4","station_4"] + field: ["10", "10", "10", "10", "54", "55", "55", + "10", "10", "10", "10", "54", "55", "55", + "10", "10", "10", "10", "54", "55", "55", + "10", "10", "10", "10", "54", "55", "55"] + level: ["1","2","3","4","1","1","2", + "1","2","3","4","1","1","2", + "1","2","3","4","1","1","2", + "1","2","3","4","1","1","2"] + ob value: ["-1073741824.00000","1.20000","-1073741824.00000","3.11000","1.00000","4.00000","0.00000", + "-1073741824.00000","-1073741824.00000","-1073741824.00000","3.22000","2.00000","2.00000","4.00000", + "-1073741824.00000","1.60000","-1073741824.00000","3.33000","2.00000","2.00000","4.00000", + "-1073741824.00000","1.80000", "-1073741824.00000","3.44000","1.00000","2.00000","0.00000"] + lat: ["21.00000", "21.00000","21.00000", "21.00000","21.00000","21.00000","21.00000", + "22.00000", "22.00000","22.00000", "22.00000","22.00000","22.00000","22.00000", + "-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", + "24.00000", "24.00000","24.00000", "24.00000","24.00000", "24.00000","24.00000"] + lon: ["31.00000", "31.00000","31.00000", "31.00000","31.00000","31.00000","31.00000", + "32.00000", "32.00000","32.00000", "32.00000","32.00000","32.00000","32.00000", + "33.00000", "33.00000","33.00000", "33.00000","33.00000","33.00000","33.00000", + "34.00000", "34.00000","34.00000", "34.00000","34.00000", "34.00000","34.00000"] + time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000","-3540.00000","-3540.00000","-3540.00000", + "-3480.00000", "-3480.00000","-3480.00000", "-3480.00000","-3480.00000","-3480.00000","-3480.00000", + "-3420.00000", "-3420.00000","-3420.00000", "-3420.00000","-3420.00000","-3420.00000","-3420.00000", + "-3360.00000", "-3360.00000","-3360.00000", "-3360.00000","-3360.00000", "-3360.00000","-3360.00000"] + Callsign: ["station_1", "station_1","station_1", "station_1","station_1","station_1", "station_1", + "station_2", "station_2","station_2", "station_2","station_2","station_2","station_2", + "station_3", "station_3","station_3", "station_3","station_3","station_3","station_3", + "station_4", "station_4","station_4", "station_4","station_4", "station_4","station_4"] HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 diff --git a/test/testinput/055_VarField_channum.yaml b/test/testinput/055_VarField_channum.yaml index 544aad15..2b32eee0 100644 --- a/test/testinput/055_VarField_channum.yaml +++ b/test/testinput/055_VarField_channum.yaml @@ -91,7 +91,9 @@ observations: - filter: VarObs Writer namelist_directory: testinput/VarObsWriterNamelists_055_VarField_channum general_mode: debug - channel_offset: 15 + # channel_offset: 15 + size_of_varobs_array: 2 + varChannels: 16, 17 - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. diff --git a/test/testinput/057_VarField_emissivity.yaml b/test/testinput/057_VarField_emissivity.yaml index 6ef78508..791afe9d 100644 --- a/test/testinput/057_VarField_emissivity.yaml +++ b/test/testinput/057_VarField_emissivity.yaml @@ -24,7 +24,8 @@ observations: - filter: VarObs Writer reject_obs_with_all_variables_failing_qc: true namelist_directory: testinput/VarObsWriterNamelists_057_VarField_emissivity - channel_offset: 1 + # channel_offset: 1 + varChannels: 2-4 size_of_varobs_array: 4 general_mode: verbose - filter: VarObs Checker diff --git a/test/testinput/076_VarField_britempvarerror.yaml b/test/testinput/076_VarField_britempvarerror.yaml index 76a6ceab..a6c7443a 100644 --- a/test/testinput/076_VarField_britempvarerror.yaml +++ b/test/testinput/076_VarField_britempvarerror.yaml @@ -24,7 +24,8 @@ observations: - filter: VarObs Writer reject_obs_with_all_variables_failing_qc: true namelist_directory: testinput/VarObsWriterNamelists_076_VarField_britempvarerror - channel_offset: 1 + # channel_offset: 1 + varChannels: 2-4 size_of_varobs_array: 4 general_mode: verbose - filter: VarObs Checker diff --git a/test/testinput/varobs_globalnamelist_atms.nc4 b/test/testinput/varobs_globalnamelist_atms.nc4 index 497450408fc870362c7cbb8e344c1cd4345d4d04..44f916faf2bdee11773dff38c538347ca5a3817c 100644 GIT binary patch delta 30 lcmdnc!nmP@k*Cwc#f_Isii?SZgK;8{0?YAP)hY`OoB)0e2lM~{ delta 30 lcmdnc!nmP@k*Cwc#f_Isii?SZgJB|%0!!s}9q)w(P5^vf2f6?N diff --git a/test/testinput/varobs_globalnamelist_iasi.nc4 b/test/testinput/varobs_globalnamelist_iasi.nc4 index ff0d9017c7f3a3cf4a61b25c98ff9225a817a362..4c78761e64baa5dfc67622de00aac8ce841349f8 100644 GIT binary patch delta 31 ncmaF4fbr!5MxIU&7dKumDJ~`s4#tT*3M@L?rqpdTunYhIm?8-2 delta 31 mcmaF4fbr!5MxIU&7dKumDJ~`s4u*+53M^HgUJe@#ECT?Pm Date: Wed, 23 Aug 2023 11:19:01 +0100 Subject: [PATCH 19/44] A por version which has just reintroduced previous options under new names. Not really what we want but want to save a copy. --- src/opsinputs/VarObsWriterParameters.h | 3 + src/opsinputs/opsinputs_fill_mod.F90 | 64 ++++--- src/opsinputs/opsinputs_varobswriter_mod.F90 | 166 ++++++++++++++---- test/testinput/010_VarField_britemp.yaml | 11 +- test/testinput/055_VarField_channum.yaml | 1 + test/testinput/057_VarField_emissivity.yaml | 3 +- .../varobswriter_globalnamelist_atovs.yaml | 3 +- .../varobswriter_globalnamelist_iasi.yaml | 1 + 8 files changed, 183 insertions(+), 69 deletions(-) diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 9e2abdc4..0e5d36c0 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -94,6 +94,9 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// Compress the var channels oops::Parameter compressVarChannels{"compress_var_channels", false, this}; + /// Where to add the extra varobs channel spaces ahead or behind. + oops::Parameter offsetAhead{"offset_ahead", false, this}; + /// If this list of ufo::variable is defined in the yaml a subset of the flags /// will be made with just these variables present. This will allow Fortran calls such-as /// "reject_obs_with_all_variables_failing_qc" and channel numbering to work correctly. diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 1e0869c7..29073d14 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1036,7 +1036,7 @@ end subroutine opsinputs_fill_fillreal !> are not found. subroutine opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, NumObs, Real2, ObsSpace, Channels, JediVarName, & - JediVarGroup, sizeVarObs, varChannels) + JediVarGroup, compressVarChannels, sizeVarObs, varChannels) implicit none ! Subroutine arguments: @@ -1048,6 +1048,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & integer(c_int), intent(in) :: Channels(:) character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup +logical, optional, intent(in) :: compressVarChannels integer(integer64), optional, intent(in) :: sizeVarObs integer(c_int), optional, intent(in) :: varChannels(:) @@ -1059,11 +1060,14 @@ subroutine opsinputs_fill_fillreal2d_norecords( & integer :: iChannel integer :: offset integer :: numchans +integer :: offsetsize +!compressVarChannels=.TRUE. ! Body: WRITE(*,*) "opsinputs_fill_mod - no records" MissingDouble = missing_value(0.0_c_double) +WRITE(*,*) JediVarName, Channels JediVarNamesWithChannels = opsinputs_fill_varnames_with_channels(JediVarName, Channels) !take into account offsetting of 2nd dimension if required @@ -1097,10 +1101,10 @@ subroutine opsinputs_fill_fillreal2d_norecords( & num_levels = int(numchans, kind=integer64)) do iChannel = 1, size(JediVarNamesWithChannels) ! Retrieve data from JEDI - if (iChannel <= size(JediVarNamesWithChannels)) then - WRITE(*,*) "JediVarNamesWithChannels(iChannel)=", JediVarNamesWithChannels(iChannel) + ! if (iChannel <= size(JediVarNamesWithChannels)) then + ! WRITE(*,*) "JediVarNamesWithChannels(iChannel)=", JediVarNamesWithChannels(iChannel) call obsspace_get_db(ObsSpace, JediVarGroup, JediVarNamesWithChannels(iChannel), VarValue) - end if + ! end if if (present(varChannels)) then @@ -1115,10 +1119,20 @@ subroutine opsinputs_fill_fillreal2d_norecords( & end if end if else - WRITE(*,*) "real2 novar" - where (VarValue /= MissingDouble) - Real2(:, iChannel) = VarValue - end where + if ((present(sizeVarObs)) .and. (sizeVarObs > size(channels))) then + if (compressVarChannels) then + offsetsize = sizeVarObs - size(channels) + WRITE(*,*) "REAL2 with offsetsize" + where (VarValue /= MissingDouble) + Real2(:, iChannel+offsetsize) = VarValue + end where + end if + else + WRITE(*,*) "real2 novar" + where (VarValue /= MissingDouble) + Real2(:, iChannel) = VarValue + end where + end if end if end do end if ! Data not present? OPS will produce a warning -- we don't need to duplicate it. @@ -1243,7 +1257,7 @@ end subroutine opsinputs_fill_fillreal2d_records !> are not found. subroutine opsinputs_fill_fillreal2d( & Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, Channels, & - VarobsLength, JediVarName, JediVarGroup, sizeVarObs, varChannels) + VarobsLength, JediVarName, JediVarGroup, compressVarChannels, sizeVarObs, varChannels) implicit none ! Subroutine arguments: @@ -1256,6 +1270,7 @@ subroutine opsinputs_fill_fillreal2d( & integer(integer64), intent(in) :: VarobsLength character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup +logical, optional, intent(in) :: compressVarChannels integer(integer64), optional, intent(in) :: sizeVarObs integer(c_int), optional, intent(in) :: varChannels(:) @@ -1270,13 +1285,13 @@ subroutine opsinputs_fill_fillreal2d( & WRITE(*,*) "opsinputs_fill_mod - varchannel option" call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, sizeVarObs, varChannels) + JediVarName, JediVarGroup, compressVarChannels, sizeVarObs, varChannels) else WRITE(*,*) "##In no VarChannels" WRITE(*,*) "sizeVarObs", sizeVarObs call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, sizeVarObs) + JediVarName, JediVarGroup, compressVarChannels, sizeVarObs) end if end if @@ -2301,23 +2316,27 @@ function opsinputs_fill_varnames_with_channels(VarName, Channels) result(VarName character(len=max_varname_with_channel_length) :: VarNames(max(size(Channels), 1)) integer :: ichan -character(len=max_varname_with_channel_length) :: VarNames_emis(max(size(Channels), 1)) +!character(len=max_varname_with_channel_length) :: VarNames_emis(max(size(Channels), 1)) write(*,*) max_varname_with_channel_length -WRITE(*,*) "VarNames=", VarNames + WRITE(*,*) "opsinputs_fill_mod - opsinputs_fill_varnames_with_channels" WRITE(*,*) size(Channels), VarName -if (Varname=="emissivity") then - do ichan = 1, size(Channels) - write(*,*) "here" - write(*,*) VarName, Channels(ichan) - write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) + +!allocate(VarNames(max(size(Channels), 1))) +!deallocate(VarNames) +!allocate(VarNames(max(size(Channels), 1))) +!if (Varname=="emissivity") then +! do ichan = 1, size(Channels) +! write(*,*) "here" +! write(*,*) VarName, Channels(ichan) +! write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) ! write(*,'(A,"_",I0)') VarName, Channels(ichan) ! write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) - end do + !end do ! VarNames = VarNames_emis -else +!else if (size(Channels) == 0) then VarNames(1) = VarName else @@ -2325,13 +2344,12 @@ function opsinputs_fill_varnames_with_channels(VarName, Channels) result(VarName do ichan = 1, size(Channels) WRITE(*,*) VarName, Channels(ichan) - WRITE(*,*) size(VarNames), VarNames + WRITE(*,*) size(VarNames) write (VarNames(ichan),'(A,"_",I0)') VarName, Channels(ichan) - write(*,*) VarNames(ichan) WRITE(*,*) "In loop", VarNames(ichan) end do end if -end if +!end if WRITE(*,*) "opsinputs_fill_mod - opsinputs_fill_varnames_with_channels finished" end function opsinputs_fill_varnames_with_channels diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index ab78f38f..12052833 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -190,6 +190,8 @@ module opsinputs_varobswriter_mod integer(c_int), allocatable :: varChannels(:) integer(integer64) :: size_of_varobs_array integer(integer64) :: NumVarChannels + + logical ::compressVarChannels !this stores the atmospheric levels we wish to pass to varobs integer(c_int), allocatable :: modlevs(:) @@ -237,6 +239,8 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe call f_conf % get_or_die("NumVarChannels", self % NumVarChannels) allocate(self % varChannels(self % NumVarChannels)) call f_conf % get_or_die("varChannels", self % varChannels) + +call f_conf % get_or_die("compress_var_channels", self % compressVarChannels) ! Setup OPS call f_conf % get_or_die("general_mode", StringValue) @@ -309,6 +313,8 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe call f_conf % get_or_die("size_of_varobs_array", self % size_of_varobs_array) +call f_conf % get_or_die("compress_var_channels", self % compressVarChannels) + !call f_conf % get_or_die("varChannels", self % varChannels) !WRITE(*,*) "get or die varChannels", self % varChannels @@ -858,7 +864,7 @@ subroutine opsinputs_varobswriter_populateobservations( & call opsinputs_fill_fillreal2d( & Ob % Header % CorBriTemp, "CorBriTemp", JediToOpsLayoutMapping, Ob % CorBriTemp, & ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue", & - self % size_of_varobs_array, self % varChannels) + self % compressVarChannels, self % size_of_varobs_array, self % varChannels) case (VarField_tskin) call opsinputs_fill_fillelementtypefromnormalvariable( & Ob % Header % Tskin, "Tskin", Ob % Header % NumObsLocal, Ob % Tskin, & @@ -875,7 +881,7 @@ subroutine opsinputs_varobswriter_populateobservations( & WRITE(*,*) "At emissivity" call opsinputs_fill_fillreal2d( & Ob % Header % MwEmiss, "MwEmiss", JediToOpsLayoutMapping, Ob % MwEmiss, & - ObsSpace, self % channels, self % VarobsLength, "emissivity", "Emiss", self % size_of_varobs_array, self % varChannels) + ObsSpace, self % channels, self % VarobsLength, "emissivity", "Emiss") case (VarField_TCozone) call opsinputs_fill_fillreal( & Ob % Header % TCozone, "TCozone", JediToOpsLayoutMapping, Ob % TCozone, & @@ -988,8 +994,9 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_Emissivity) call opsinputs_fill_fillreal2d( & Ob % Header % Emissivity, "Emissivity", JediToOpsLayoutMapping, Ob % Emissivity, & - ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar", self % size_of_varobs_array, & - self % varChannels) + ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar", & + self % compressVarChannels, self % size_of_varobs_array) + !self % varChannels) case (VarField_QCinfo) ! TODO(someone): This will come from a variable generated by the 1D-Var filter. Its name and ! group are not known yet. Once they are, replace the placeholders in the call below. @@ -1089,7 +1096,7 @@ subroutine opsinputs_varobswriter_populateobservations( & call opsinputs_fill_fillreal2d( & Ob % Header % BriTempVarError, "BriTempVarError", JediToOpsLayoutMapping, Ob % BriTempVarError, & ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature",& - "ObsErrorData", self % size_of_varobs_array, self % varChannels) + "ObsErrorData", self % compressVarChannels, self % size_of_varobs_array, self % varChannels) case (VarField_CloudRTError) ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % CloudRTError, "CloudRTError", Ob % Header % NumObsLocal, Ob % CloudRTError) @@ -1224,7 +1231,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & integer(c_int), intent(in) :: varChannels(:) type(c_ptr), value, intent(in) :: Flags logical, intent(in) :: FillChanNum, FillNumChans -integer(integer64), intent(in) :: varObsSize +integer(integer64), optional, intent(in) :: varObsSize ! Local declarations: integer(integer64) :: NumChannels @@ -1234,6 +1241,9 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & integer :: iChannel integer :: iObs real(kind=c_double) :: MissingDouble +integer(integer64) ::array_loop +logical :: compressVarChannels +integer :: offsetsize WRITE(*,*) "Enter opsinputs_varobswriter_fillchannumandnumchans" ! Body: @@ -1248,47 +1258,125 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & WRITE(*,*) "Initial ChannelIndices=", ChannelIndices(:,1) WRITE(*,*) "Initial ChannelIndices=", ChannelIndices(:,2) +!Ob % ChanNum = ChannelIndices +!WRITE(*,*) Ob % ChanNum +!Ob % ChanNum = Ob % ChanNum + 1 +!WRITE(*,*) Ob % ChanNum + ChannelIndicesVar(:,:) = 0 WRITE(*,*) "orig ChannelIndicesVar", ChannelIndicesVar WRITE(*,*) "size of varChannels", size(varChannels) !WRITE(*,*) ChannelIndicesVar(:,1) WRITE(*,*) "FillChanNum", FillChanNum + + if (FillChanNum) then call Ops_Alloc(Ob % Header % ChanNum, "ChanNum", Ob % Header % NumObsLocal, Ob % ChanNum, & num_levels = NumChannels) - WRITE(*,*) "#### In if FillChnaNum" - if (size(varChannels) > 0) then - if (size(channels) == size(varChannels)) then - WRITE(*,*) "Number of channels=", NumChannels - WRITE(*,*) "#varObsSize", varObsSize - WRITE(*,*) Ob % Header % NumObsLocal - do iChannel=1, NumChannels !varObsSize - WRITE(*,*) "iChannel=", iChannel, size(varChannels) - ! if ((iChannel > size(varChannels))) then - ! WRITE(*,*) "in missing", MissingDouble - ! ChannelIndicesVar(:, iChannel) = MissingDouble - ! WRITE(*,*) "Going to continue" - ! continue - ! end if - - do iObs=1, Ob % Header % NumObsLocal - WRITE(*,*) "VarChannels=", varChannels(iChannel) - if (ChannelIndices(iObs,iChannel) /= 0) then - ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) - end if - - - WRITE(*,*) "ChannelIndicesVar1=", ChannelIndicesVar(:,1) - WRITE(*,*) "ChannelIndicesVar2=", ChannelIndicesVar(:,2) - WRITE(*,*) "ChannelIndicesVar3=", ChannelIndicesVar(:,3) - end do - end do - Ob % ChanNum = ChannelIndicesVar - end if + WRITE(*,*) "In FillChanNum" + if (varObsSize /= 0) then + if (varObsSize > size(channels)) then + if (compressVarChannels) then + WRITE(*,*) "Doing offset" + offsetsize = varObsSize - size(channels) + WRITE(*,*) "offsetsize=", offsetsize + Ob % ChanNum = ChannelIndices + WRITE(*,*) ChannelIndices(:,1) + WRITE(*,*) ChannelIndices(:,2) + where (Ob % ChanNum > 0) + Ob % ChanNum = Ob % ChanNum + int(offsetsize, kind=integer64) + end where + else + if (size(varChannels) > 0) then + if (size(channels) == size(varChannels)) then + WRITE(*,*) "Number of channels=", NumChannels + WRITE(*,*) "#varObsSize", varObsSize + WRITE(*,*) Ob % Header % NumObsLocal + do iChannel=1, NumChannels !varObsSize + WRITE(*,*) "iChannel=", iChannel, size(varChannels) + if (compressVarChannels) then + do iObs=1, Ob % Header % NumObsLocal + WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) + if (ChannelIndices(iObs,iChannel) /= 0) then + ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) + end if + end do + else + ChannelIndicesVar(:, iChannel) = varChannels(iChannel) + end if + + WRITE(*,*) "ChannelIndicesVar1=", ChannelIndicesVar(:,1) + WRITE(*,*) "ChannelIndicesVar2=", ChannelIndicesVar(:,2) + end do + Ob % ChanNum = ChannelIndicesVar + end if + end if + end if + else if (varObsSize == size(channels)) then + WRITE(*,*) "varObsSize==size(channels)" + WRITE(*,*) compressVarChannels + if (size(varChannels) > 0) then + do iChannel=1, NumChannels !varObsSize + WRITE(*,*) "iChannel=", iChannel, size(varChannels) + if (compressVarChannels) then + WRITE(*,*) "here" + do iObs=1, Ob % Header % NumObsLocal + WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) + if (ChannelIndices(iObs,iChannel) /= 0) then + ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) + end if + end do + end if + end do + Ob % ChanNum = ChannelIndicesVar + else + WRITE(*,*) "Set ChanNum to ChannelIndices" + Ob % ChanNum = ChannelIndices + end if + end if else - WRITE(*,*) "Set ChanNum to ChannelIndices" + WRITE(*,*) "The basic case" Ob % ChanNum = ChannelIndices end if +else + WRITE(*,*) "Not FillChanNum" +end if +! WRITE(*,*) "#### In if FillChnaNum" +! if (size(varChannels) > 0) then +! if (size(channels) == size(varChannels)) then +! WRITE(*,*) "Number of channels=", NumChannels +! WRITE(*,*) "#varObsSize", varObsSize +! WRITE(*,*) Ob % Header % NumObsLocal +! do iChannel=1, NumChannels !varObsSize +! WRITE(*,*) "iChannel=", iChannel, size(varChannels) +! ! if ((iChannel > size(varChannels))) then +! ! WRITE(*,*) "in missing", MissingDouble +! ! ChannelIndicesVar(:, iChannel) = MissingDouble +! ! WRITE(*,*) "Going to continue" +! ! continue +! ! end if +! compressVarChannels=.TRUE. +! if (compressVarChannels) then +! do iObs=1, Ob % Header % NumObsLocal +! WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) +! if (ChannelIndices(iObs,iChannel) /= 0) then +! ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) +! end if +! end do +! else +! ChannelIndicesVar(:, iChannel) = varChannels(iChannel) +! end if + +! WRITE(*,*) "ChannelIndicesVar1=", ChannelIndicesVar(:,1) +! WRITE(*,*) "ChannelIndicesVar2=", ChannelIndicesVar(:,2) + +! end do +! Ob % ChanNum = ChannelIndicesVar +! end if +! else +! WRITE(*,*) "Set ChanNum to ChannelIndices" +! Ob % ChanNum = ChannelIndices +! end if ! else ! Ob % ChanNum = ChannelIndices !only apply offset to actual channels in list, not missing data @@ -1296,9 +1384,9 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & ! Ob % ChanNum = Ob % ChanNum + int(OffsetChans, kind=integer64) ! end where ! end if -else - WRITE(*,*) "Not FillChanNum" -end if +!else +! WRITE(*,*) "Not FillChanNum" +!end if !WRITE(*,*) Ob % ChanNum if (FillNumChans) then call Ops_Alloc(Ob % Header % NumChans, "NumChans", Ob % Header % NumObsLocal, Ob % NumChans) diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index 4c70a920..d3574825 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -164,8 +164,9 @@ observations: channels: 1,3 namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp general_mode: debug + compress_var_channels: true size_of_varobs_array: 4 - varChannels: 2, 4 #[5,7] + # varChannels: 2, 4 #[5,7] # output_directory: /home/h01/olewis - filter: VarObs Checker expected_main_table_columns: @@ -178,10 +179,10 @@ observations: "1","2","3","4","1","1","2", "1","2","3","4","1","1","2", "1","2","3","4","1","1","2"] - ob value: ["-1073741824.00000","1.20000","-1073741824.00000","3.11000","1.00000","4.00000","0.00000", - "-1073741824.00000","-1073741824.00000","-1073741824.00000","3.22000","2.00000","2.00000","4.00000", - "-1073741824.00000","1.60000","-1073741824.00000","3.33000","2.00000","2.00000","4.00000", - "-1073741824.00000","1.80000", "-1073741824.00000","3.44000","1.00000","2.00000","0.00000"] + ob value: ["-1073741824.00000","-1073741824.00000","1.20000","3.11000","1.00000","4.00000","0.00000", + "-1073741824.00000","-1073741824.00000","-1073741824.00000","3.22000","2.00000","3.00000","4.00000", + "-1073741824.00000","-1073741824.00000","1.60000","3.33000","2.00000","3.00000","4.00000", + "-1073741824.00000","-1073741824.00000","1.80000", "3.44000","1.00000","3.00000","0.00000"] lat: ["21.00000", "21.00000","21.00000", "21.00000","21.00000","21.00000","21.00000", "22.00000", "22.00000","22.00000", "22.00000","22.00000","22.00000","22.00000", "-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", diff --git a/test/testinput/055_VarField_channum.yaml b/test/testinput/055_VarField_channum.yaml index 2b32eee0..29f9413d 100644 --- a/test/testinput/055_VarField_channum.yaml +++ b/test/testinput/055_VarField_channum.yaml @@ -92,6 +92,7 @@ observations: namelist_directory: testinput/VarObsWriterNamelists_055_VarField_channum general_mode: debug # channel_offset: 15 + compress_var_channels: true size_of_varobs_array: 2 varChannels: 16, 17 - filter: VarObs Checker diff --git a/test/testinput/057_VarField_emissivity.yaml b/test/testinput/057_VarField_emissivity.yaml index 791afe9d..e03c708d 100644 --- a/test/testinput/057_VarField_emissivity.yaml +++ b/test/testinput/057_VarField_emissivity.yaml @@ -25,7 +25,8 @@ observations: reject_obs_with_all_variables_failing_qc: true namelist_directory: testinput/VarObsWriterNamelists_057_VarField_emissivity # channel_offset: 1 - varChannels: 2-4 + # varChannels: 2-4 + compress_var_channels: true size_of_varobs_array: 4 general_mode: verbose - filter: VarObs Checker diff --git a/test/testinput/varobswriter_globalnamelist_atovs.yaml b/test/testinput/varobswriter_globalnamelist_atovs.yaml index 629c291d..770c90ec 100644 --- a/test/testinput/varobswriter_globalnamelist_atovs.yaml +++ b/test/testinput/varobswriter_globalnamelist_atovs.yaml @@ -26,7 +26,8 @@ observations: namelist_directory: ../etc/global/varobs general_mode: verbose # channel_offset: 1 - varChannels: 2,4 + # varChannels: 2,4 + compress_var_channels: true size_of_varobs_array: 3 - filter: VarObs Checker expected_main_table_columns: diff --git a/test/testinput/varobswriter_globalnamelist_iasi.yaml b/test/testinput/varobswriter_globalnamelist_iasi.yaml index e17e2be6..778e295c 100644 --- a/test/testinput/varobswriter_globalnamelist_iasi.yaml +++ b/test/testinput/varobswriter_globalnamelist_iasi.yaml @@ -41,6 +41,7 @@ observations: reject_obs_with_all_variables_failing_qc: true general_mode: verbose output_varbc_predictors: true + compress_var_channels: true size_of_varobs_array: 4 fill_obstype_from_ops_subtype: true - filter: VarObs Checker From 612f81d1954d624596e414fe4e16f5a42aad7830 Mon Sep 17 00:00:00 2001 From: orlewis Date: Thu, 24 Aug 2023 15:03:28 +0100 Subject: [PATCH 20/44] All the ctests are now passing. --- src/opsinputs/VarObsWriterParameters.h | 3 - src/opsinputs/opsinputs_fill_mod.F90 | 38 ++- src/opsinputs/opsinputs_varobswriter_mod.F90 | 286 +++++++++++------- test/testinput/010_VarField_britemp.yaml | 196 ++++++------ test/testinput/057_VarField_emissivity.yaml | 2 +- .../076_VarField_britempvarerror.yaml | 1 + .../varobswriter_globalnamelist_atovs.yaml | 3 +- 7 files changed, 306 insertions(+), 223 deletions(-) diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 0e5d36c0..9e2abdc4 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -94,9 +94,6 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// Compress the var channels oops::Parameter compressVarChannels{"compress_var_channels", false, this}; - /// Where to add the extra varobs channel spaces ahead or behind. - oops::Parameter offsetAhead{"offset_ahead", false, this}; - /// If this list of ufo::variable is defined in the yaml a subset of the flags /// will be made with just these variables present. This will allow Fortran calls such-as /// "reject_obs_with_all_variables_failing_qc" and channel numbering to work correctly. diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 29073d14..9488d7ef 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1056,7 +1056,8 @@ subroutine opsinputs_fill_fillreal2d_norecords( & ! Local declarations: real(kind=c_double) :: VarValue(NumObs) real(kind=c_double) :: MissingDouble -character(len=max_varname_with_channel_length) :: JediVarNamesWithChannels(max(size(Channels), 1)) +!character(len=max_varname_with_channel_length) :: JediVarNamesWithChannels(max(size(Channels), 1)) +character(len=200) :: JediVarNamesWithChannels(max(size(Channels), 1)) integer :: iChannel integer :: offset integer :: numchans @@ -1067,7 +1068,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & WRITE(*,*) "opsinputs_fill_mod - no records" MissingDouble = missing_value(0.0_c_double) -WRITE(*,*) JediVarName, Channels +WRITE(*,*) "JediVarName=", JediVarName, "Channels", Channels JediVarNamesWithChannels = opsinputs_fill_varnames_with_channels(JediVarName, Channels) !take into account offsetting of 2nd dimension if required @@ -1076,17 +1077,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & numchans = size(JediVarNamesWithChannels) WRITE(*,*) "numchans=", numchans -WRITE(*,*) "Present", present(sizeVarObs), sizeVarObs -if ((present(varChannels)) .and. (size(varChannels) > 0)) then - numchans = sizeVarObs -end if -WRITE(*,*) "SizeVarObs=", sizeVarObs -!WRITE(*,*) sizeVarObs == 0 -!WRITE(*,*) sizeVarObs /= 0 -!if (sizeVarObs /= 0) then - ! numchans = sizeVarObs - ! WRITE(*,*) "Numchan set", numchans -!end if + WRITE(*,*) "numchans=", numchans if (present(sizeVarObs) )then if (sizeVarObs > 0) then @@ -1111,6 +1102,15 @@ subroutine opsinputs_fill_fillreal2d_norecords( & if (size(varChannels) > 0) then WRITE(*,*) "here" if (iChannel <= size(varChannels)) then + if (compressVarChannels) then + offsetsize = abs(varChannels(1) - channels(1)) + write(*,*) "offsetsize=", offsetsize + WRITE(*,*) "REAL2 with offsetsize" + where (VarValue /= MissingDouble) + Real2(:, iChannel+offsetsize) = VarValue + end where + end if + else WRITE(*,*) "here1" where (VarValue /= MissingDouble) Real2(:, varChannels(iChannel)) = VarValue @@ -1122,9 +1122,14 @@ subroutine opsinputs_fill_fillreal2d_norecords( & if ((present(sizeVarObs)) .and. (sizeVarObs > size(channels))) then if (compressVarChannels) then offsetsize = sizeVarObs - size(channels) - WRITE(*,*) "REAL2 with offsetsize" + ! WRITE(*,*) "REAL2 with offsetsize" where (VarValue /= MissingDouble) - Real2(:, iChannel+offsetsize) = VarValue + Real2(:, iChannel) = VarValue + end where + else + WRITE(*,*) "In noncompressed larger size" + where (VarValue /= MissingDouble) + Real2(:, Channels(iChannel)) = VarValue end where end if else @@ -1890,7 +1895,8 @@ subroutine opsinputs_fill_filltimeoffsets2d_norecords( & ! Local declarations: integer(c_int64_t) :: VarValue(NumObs) real(kind=c_double) :: MissingDouble -character(len=max_varname_with_channel_length) :: JediVarNamesWithChannels(max(size(Channels), 1)) +!character(len=max_varname_with_channel_length) :: JediVarNamesWithChannels(max(size(Channels), 1)) +character(len=200) :: JediVarNamesWithChannels(max(size(Channels), 1)) integer :: iChannel ! Body: diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 12052833..d6040b8f 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -878,7 +878,7 @@ subroutine opsinputs_varobswriter_populateobservations( & Ob % Header % Zstation, "Zstation", JediToOpsLayoutMapping, Ob % Zstation, & ObsSpace, "stationElevation", "MetaData") case (VarField_mwemiss) - WRITE(*,*) "At emissivity" + WRITE(*,*) "At mwemissivity" call opsinputs_fill_fillreal2d( & Ob % Header % MwEmiss, "MwEmiss", JediToOpsLayoutMapping, Ob % MwEmiss, & ObsSpace, self % channels, self % VarobsLength, "emissivity", "Emiss") @@ -992,10 +992,11 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_ChanNum) FillChanNum = .true. case (VarField_Emissivity) + WRITE(*,*) "Case is Emissivity" call opsinputs_fill_fillreal2d( & Ob % Header % Emissivity, "Emissivity", JediToOpsLayoutMapping, Ob % Emissivity, & ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar", & - self % compressVarChannels, self % size_of_varobs_array) + self % compressVarChannels, self % size_of_varobs_array, self % varChannels) !self % varChannels) case (VarField_QCinfo) ! TODO(someone): This will come from a variable generated by the 1D-Var filter. Its name and @@ -1173,9 +1174,10 @@ subroutine opsinputs_varobswriter_populateobservations( & WRITE(*,*) self % varChannels WRITE(*,*) "size(self % channels)=", size(self % channels) WRITE(*,*) self % size_of_varobs_array + call opsinputs_varobswriter_fillchannumandnumchans( & - Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & - FillNumChans, self % size_of_varobs_array) + Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & + FillNumChans, self % compressVarChannels, self % size_of_varobs_array) end if end do @@ -1220,7 +1222,7 @@ end subroutine opsinputs_varobswriter_fillreportflags !> e.g. for HIRS & AMSUA !> the number of these channels is stored in Ob % NumChans. subroutine opsinputs_varobswriter_fillchannumandnumchans( & - Ob, ObsSpace, channels, varChannels, Flags, FillChanNum, FillNumChans, varObsSize) + Ob, ObsSpace, channels, varChannels, Flags, FillChanNum, FillNumChans, compressVarChannels, varObsSize) implicit none @@ -1231,8 +1233,10 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & integer(c_int), intent(in) :: varChannels(:) type(c_ptr), value, intent(in) :: Flags logical, intent(in) :: FillChanNum, FillNumChans +logical :: compressVarChannels integer(integer64), optional, intent(in) :: varObsSize + ! Local declarations: integer(integer64) :: NumChannels integer(integer64) :: ChannelIndicesVar(Ob % Header % NumObsLocal, size(varChannels)) !varObssize @@ -1242,7 +1246,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & integer :: iObs real(kind=c_double) :: MissingDouble integer(integer64) ::array_loop -logical :: compressVarChannels + integer :: offsetsize WRITE(*,*) "Enter opsinputs_varobswriter_fillchannumandnumchans" ! Body: @@ -1262,6 +1266,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & !WRITE(*,*) Ob % ChanNum !Ob % ChanNum = Ob % ChanNum + 1 !WRITE(*,*) Ob % ChanNum +WRITE(*,*) "compressVarChannels = ", compressVarChannels ChannelIndicesVar(:,:) = 0 WRITE(*,*) "orig ChannelIndicesVar", ChannelIndicesVar @@ -1274,120 +1279,189 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & call Ops_Alloc(Ob % Header % ChanNum, "ChanNum", Ob % Header % NumObsLocal, Ob % ChanNum, & num_levels = NumChannels) WRITE(*,*) "In FillChanNum" - if (varObsSize /= 0) then + if ((varObsSize /= 0)) then if (varObsSize > size(channels)) then - if (compressVarChannels) then - WRITE(*,*) "Doing offset" - offsetsize = varObsSize - size(channels) - WRITE(*,*) "offsetsize=", offsetsize - Ob % ChanNum = ChannelIndices - WRITE(*,*) ChannelIndices(:,1) - WRITE(*,*) ChannelIndices(:,2) - where (Ob % ChanNum > 0) - Ob % ChanNum = Ob % ChanNum + int(offsetsize, kind=integer64) - end where + if (size(varChannels) > 0 .and. (size(channels) == size(varChannels))) then + if (compressVarChannels) then + offsetsize = abs(varChannels(1) - channels(1)) + write(*,*) "offsetsize=", offsetsize + Ob % ChanNum = ChannelIndices + where (Ob % ChanNum > 0) + Ob % ChanNum = Ob % ChanNum + int(offsetsize, kind=integer64) + end where + WRITE(*,*) "Finished VarChannels with compression" + + else + do iChannel=1, NumChannels + ChannelIndicesVar(:, iChannel) = varChannels(iChannel) + end do + Ob % ChanNum = ChannelIndicesVar + end if + + ! WRITE(*,*) "ChannelIndicesVar1=", ChannelIndicesVar(:,1) + ! WRITE(*,*) "ChannelIndicesVar2=", ChannelIndicesVar(:,2) + !end do + !Ob % ChanNum = ChannelIndicesVar else - if (size(varChannels) > 0) then - if (size(channels) == size(varChannels)) then - WRITE(*,*) "Number of channels=", NumChannels - WRITE(*,*) "#varObsSize", varObsSize - WRITE(*,*) Ob % Header % NumObsLocal - do iChannel=1, NumChannels !varObsSize - WRITE(*,*) "iChannel=", iChannel, size(varChannels) - if (compressVarChannels) then - do iObs=1, Ob % Header % NumObsLocal - WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) - if (ChannelIndices(iObs,iChannel) /= 0) then - ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) - end if - end do - else - ChannelIndicesVar(:, iChannel) = varChannels(iChannel) + WRITE(*,*) "No varChannel, varObsSize is greater" + WRITE(*,*) compressVarChannels + ! compressVarChannels=.false. + if (compressVarChannels) then + write(*,*) "Compressed var channells" + Ob % ChanNum = ChannelIndices + else + WRITE(*,*) "Here now" + do iChannel=1, size(Channels) + ChannelIndices(:,iChannel) = Channels(iChannel) + end do + WRITE(*,*) ChannelIndices(:,1) + WRITE(*,*) ChannelIndices(:,2) + Ob % ChanNum = ChannelIndices + end if + end if + else if (varObsSize == size(channels)) then + WRITE(*,*) "varObsSize==size(channels)" + WRITE(*,*) compressVarChannels + if (size(varChannels) > 0) then + do iChannel=1, NumChannels !varObsSize + WRITE(*,*) "iChannel=", iChannel, size(varChannels) + if (compressVarChannels) then + WRITE(*,*) "here" + do iObs=1, Ob % Header % NumObsLocal + WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) + if (ChannelIndices(iObs,iChannel) /= 0) then + ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) end if - - WRITE(*,*) "ChannelIndicesVar1=", ChannelIndicesVar(:,1) - WRITE(*,*) "ChannelIndicesVar2=", ChannelIndicesVar(:,2) - end do - Ob % ChanNum = ChannelIndicesVar - end if - end if - end if - else if (varObsSize == size(channels)) then - WRITE(*,*) "varObsSize==size(channels)" - WRITE(*,*) compressVarChannels - if (size(varChannels) > 0) then - do iChannel=1, NumChannels !varObsSize - WRITE(*,*) "iChannel=", iChannel, size(varChannels) - if (compressVarChannels) then - WRITE(*,*) "here" - do iObs=1, Ob % Header % NumObsLocal - WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) - if (ChannelIndices(iObs,iChannel) /= 0) then - ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) - end if - end do - end if - end do - Ob % ChanNum = ChannelIndicesVar - else - WRITE(*,*) "Set ChanNum to ChannelIndices" - Ob % ChanNum = ChannelIndices - end if - end if + end do + end if + end do + Ob % ChanNum = ChannelIndicesVar + else + WRITE(*,*) "Set ChanNum to ChannelIndices" + Ob % ChanNum = ChannelIndices + end if + end if else - WRITE(*,*) "The basic case" - Ob % ChanNum = ChannelIndices + WRITE(*,*) "The basic case" + Ob % ChanNum = ChannelIndices end if else WRITE(*,*) "Not FillChanNum" end if -! WRITE(*,*) "#### In if FillChnaNum" -! if (size(varChannels) > 0) then -! if (size(channels) == size(varChannels)) then -! WRITE(*,*) "Number of channels=", NumChannels -! WRITE(*,*) "#varObsSize", varObsSize -! WRITE(*,*) Ob % Header % NumObsLocal -! do iChannel=1, NumChannels !varObsSize -! WRITE(*,*) "iChannel=", iChannel, size(varChannels) -! ! if ((iChannel > size(varChannels))) then -! ! WRITE(*,*) "in missing", MissingDouble -! ! ChannelIndicesVar(:, iChannel) = MissingDouble -! ! WRITE(*,*) "Going to continue" -! ! continue -! ! end if -! compressVarChannels=.TRUE. -! if (compressVarChannels) then -! do iObs=1, Ob % Header % NumObsLocal -! WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) -! if (ChannelIndices(iObs,iChannel) /= 0) then -! ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) -! end if -! end do -! else -! ChannelIndicesVar(:, iChannel) = varChannels(iChannel) -! end if - -! WRITE(*,*) "ChannelIndicesVar1=", ChannelIndicesVar(:,1) -! WRITE(*,*) "ChannelIndicesVar2=", ChannelIndicesVar(:,2) - -! end do -! Ob % ChanNum = ChannelIndicesVar -! end if +! if (varObsSize /= 0) then +! if (varObsSize > size(channels)) then +! if (compressVarChannels) then +! WRITE(*,*) "Doing offset" +! offsetsize = varObsSize - size(channels) +! WRITE(*,*) "offsetsize=", offsetsize +! Ob % ChanNum = ChannelIndices +! WRITE(*,*) ChannelIndices(:,1) +! WRITE(*,*) ChannelIndices(:,2) +! where (Ob % ChanNum > 0) +! Ob % ChanNum = Ob % ChanNum + int(offsetsize, kind=integer64) +! end where +! else +! if (size(varChannels) > 0) then +! if (size(channels) == size(varChannels)) then +! WRITE(*,*) "Number of channels=", NumChannels +! WRITE(*,*) "#varObsSize", varObsSize +! WRITE(*,*) Ob % Header % NumObsLocal +! do iChannel=1, NumChannels !varObsSize +! WRITE(*,*) "iChannel=", iChannel, size(varChannels) +! if (compressVarChannels) then +! do iObs=1, Ob % Header % NumObsLocal +! WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) +! if (ChannelIndices(iObs,iChannel) /= 0) then +! ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) +! end if +! end do +! else +! ChannelIndicesVar(:, iChannel) = varChannels(iChannel) +! end if + +! WRITE(*,*) "ChannelIndicesVar1=", ChannelIndicesVar(:,1) +! WRITE(*,*) "ChannelIndicesVar2=", ChannelIndicesVar(:,2) +! end do +! Ob % ChanNum = ChannelIndicesVar +! end if +! end if +! end if +! else if (varObsSize == size(channels)) then +! WRITE(*,*) "varObsSize==size(channels)" +! WRITE(*,*) compressVarChannels +! if (size(varChannels) > 0) then +! do iChannel=1, NumChannels !varObsSize +! WRITE(*,*) "iChannel=", iChannel, size(varChannels) +! if (compressVarChannels) then +! WRITE(*,*) "here" +! do iObs=1, Ob % Header % NumObsLocal +! WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) +! if (ChannelIndices(iObs,iChannel) /= 0) then +! ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) +! end if +! end do +! end if +! end do +! Ob % ChanNum = ChannelIndicesVar +! else +! WRITE(*,*) "Set ChanNum to ChannelIndices" +! Ob % ChanNum = ChannelIndices +! end if +! end if ! else -! WRITE(*,*) "Set ChanNum to ChannelIndices" +! WRITE(*,*) "The basic case" ! Ob % ChanNum = ChannelIndices ! end if - ! else - ! Ob % ChanNum = ChannelIndices - !only apply offset to actual channels in list, not missing data - ! where (Ob % ChanNum > 0) - ! Ob % ChanNum = Ob % ChanNum + int(OffsetChans, kind=integer64) - ! end where - ! end if !else ! WRITE(*,*) "Not FillChanNum" !end if -!WRITE(*,*) Ob % ChanNum +!! WRITE(*,*) "#### In if FillChnaNum" +!! if (size(varChannels) > 0) then +!! if (size(channels) == size(varChannels)) then +!! WRITE(*,*) "Number of channels=", NumChannels +!! WRITE(*,*) "#varObsSize", varObsSize +!! WRITE(*,*) Ob % Header % NumObsLocal +!! do iChannel=1, NumChannels !varObsSize +!! WRITE(*,*) "iChannel=", iChannel, size(varChannels) +!! ! if ((iChannel > size(varChannels))) then +!! ! WRITE(*,*) "in missing", MissingDouble +!! ! ChannelIndicesVar(:, iChannel) = MissingDouble +!! ! WRITE(*,*) "Going to continue" +!! ! continue +!! ! end if +!! compressVarChannels=.TRUE. +!! if (compressVarChannels) then +!! do iObs=1, Ob % Header % NumObsLocal +!! WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) +!! if (ChannelIndices(iObs,iChannel) /= 0) then +!! ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) +!! end if +!! end do +!! else +!! ChannelIndicesVar(:, iChannel) = varChannels(iChannel) +!! end if + +!! WRITE(*,*) "ChannelIndicesVar1=", ChannelIndicesVar(:,1) +!! WRITE(*,*) "ChannelIndicesVar2=", ChannelIndicesVar(:,2) + +!! end do +!! Ob % ChanNum = ChannelIndicesVar +!! end if +!! else +!! WRITE(*,*) "Set ChanNum to ChannelIndices" +!! Ob % ChanNum = ChannelIndices +!! end if +! ! else +! ! Ob % ChanNum = ChannelIndices +! !only apply offset to actual channels in list, not missing data +! ! where (Ob % ChanNum > 0) +! ! Ob % ChanNum = Ob % ChanNum + int(OffsetChans, kind=integer64) +! ! end where +! ! end if +!!else +!! WRITE(*,*) "Not FillChanNum" +!!end if +!!WRITE(*,*) Ob % ChanNum if (FillNumChans) then call Ops_Alloc(Ob % Header % NumChans, "NumChans", Ob % Header % NumObsLocal, Ob % NumChans) Ob % NumChans = ChannelCounts diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index d3574825..082f4473 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -24,6 +24,7 @@ observations: - filter: VarObs Writer namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp general_mode: debug + # output_directory: /data/users/olewis/ - filter: VarObs Checker expected_main_table_columns: # Only levels 1 and 3 are simulated and hence written to the VarObs file. @@ -59,72 +60,74 @@ observations: HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 - -## -## Now we test effect of actual channel -## -# - obs space: -# name: AMSUB -# obsdatain: -# engine: -# type: H5File -# obsfile: Data/010_VarField_britemp.nc4 -# simulated variables: [brightnessTemperature] -# channels: 1,3 -# obs filters: -# # Double all observation errors: we want to check if error changes made by filters are -# # propagated to VarObs files -# - filter: BlackList -# action: -# name: inflate error -# inflation factor: 2.0 -# # Reject the first level of the first observation and the last level of the last observation -# - filter: Bounds Check -# minvalue: 1.15 -# maxvalue: 3.35 -# # Set the flag of observations with missing values to "pass": we want to check if these -# # values are encoded correctly in the VarObsFile. -# - filter: Reset Flags to Pass -# flags_to_reset: [10, 15] # missing, Hfailed -# - filter: VarObs Writer -# namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp -# general_mode: debug -# size_of_varobs_array: 3 -# use_actual_channels: true -# - filter: VarObs Checker -# expected_main_table_columns: -# # In the arrays below, rows denote locations and columns levels. -# field: ["10", "10", "10","54","55","55","55", -# "10", "10", "10", -# "10", "10", "10", -# "10", "10", "10"] -# level: ["1", "2","3", -# "1", "2","3", -# "1", "2","3", -# "1", "2","3"] -# ob value: ["1.20000","-1073741824.00000","3.11000", -# "-1073741824.00000","-1073741824.00000","3.22000", -# "1.60000","-1073741824.00000","3.33000", -# "1.80000","-1073741824.00000","3.44000"] -# lat: ["21.00000", "21.00000","21.00000", -# "22.00000", "22.00000","22.00000", -# "-23.00000", "-23.00000","-23.00000", -# "24.00000", "24.00000","24.00000"] -# lon: ["31.00000", "31.00000","31.00000", -# "32.00000", "32.00000","32.00000", -# "33.00000", "33.00000","33.00000", -# "34.00000", "34.00000","34.00000"] -# time: ["-3540.00000", "-3540.00000","-3540.00000", -# "-3480.00000", "-3480.00000","-3480.00000", -# "-3420.00000", "-3420.00000","-3420.00000", -# "-3360.00000", "-3360.00000","-3360.00000"] -# Callsign: ["station_1", "station_1","station_1", -# "station_2", "station_2","station_2", -# "station_3", "station_3","station_3", -# "station_4", "station_4","station_4"] -# HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. -# benchmarkFlag: 1000 # just to keep the ObsFilters test happy -# flaggedBenchmark: 0 + +# +# Now we test effect of actual channel +# + - obs space: + name: AMSUB + obsdatain: + engine: + type: H5File + obsfile: Data/010_VarField_britemp.nc4 + simulated variables: [brightnessTemperature] + channels: 1,3 + obs filters: + # Double all observation errors: we want to check if error changes made by filters are + # propagated to VarObs files + - filter: BlackList + action: + name: inflate error + inflation factor: 2.0 + # Reject the first level of the first observation and the last level of the last observation + - filter: Bounds Check + minvalue: 1.15 + maxvalue: 3.35 + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + - filter: VarObs Writer + namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp + # output_directory: /data/users/olewis/ + general_mode: debug + size_of_varobs_array: 3 + # use_actual_channels: true + compress_var_channels: false + - filter: VarObs Checker + expected_main_table_columns: + # In the arrays below, rows denote locations and columns levels. + field: ["10", "10", "10","54","55","55", + "10", "10", "10","54","55","55", + "10", "10", "10","54","55","55", + "10", "10", "10","54","55","55"] + level: ["1", "2","3","1", "1","2", + "1", "2","3","1", "1","2", + "1", "2","3","1", "1","2", + "1", "2","3","1", "1","2"] + ob value: ["1.20000","-1073741824.00000","3.11000", "1.00000","1.00000","3.00000", + "-1073741824.00000","-1073741824.00000","3.22000", "2.00000","1.00000","3.00000", + "1.60000","-1073741824.00000","3.33000", "2.00000","1.00000","3.00000", + "1.80000","-1073741824.00000","3.44000", "1.00000","1.00000","3.00000"] + lat: ["21.00000", "21.00000","21.00000","21.00000", "21.00000","21.00000", + "22.00000", "22.00000","22.00000","22.00000", "22.00000","22.00000", + "-23.00000", "-23.00000","-23.00000","-23.00000", "-23.00000","-23.00000", + "24.00000", "24.00000","24.00000","24.00000", "24.00000","24.00000"] + lon: ["31.00000", "31.00000","31.00000","31.00000", "31.00000","31.00000", + "32.00000", "32.00000","32.00000","32.00000", "32.00000","32.00000", + "33.00000", "33.00000","33.00000","33.00000", "33.00000","33.00000", + "34.00000", "34.00000","34.00000","34.00000", "34.00000","34.00000"] + time: ["-3540.00000", "-3540.00000","-3540.00000","-3540.00000", "-3540.00000","-3540.00000", + "-3480.00000", "-3480.00000","-3480.00000","-3480.00000", "-3480.00000","-3480.00000", + "-3420.00000", "-3420.00000","-3420.00000","-3420.00000", "-3420.00000","-3420.00000", + "-3360.00000", "-3360.00000","-3360.00000","-3360.00000", "-3360.00000","-3360.00000"] + Callsign: ["station_1", "station_1","station_1","station_1", "station_1","station_1", + "station_2", "station_2","station_2","station_2", "station_2","station_2", + "station_3", "station_3","station_3","station_3", "station_3","station_3", + "station_4", "station_4","station_4","station_4", "station_4","station_4"] + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 @@ -165,40 +168,41 @@ observations: namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp general_mode: debug compress_var_channels: true - size_of_varobs_array: 4 - # varChannels: 2, 4 #[5,7] + size_of_varobs_array: 3 + varChannels: 2, 4 #[5,7] # output_directory: /home/h01/olewis + # output_directory: /data/users/olewis/ - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. - field: ["10", "10", "10", "10", "54", "55", "55", - "10", "10", "10", "10", "54", "55", "55", - "10", "10", "10", "10", "54", "55", "55", - "10", "10", "10", "10", "54", "55", "55"] - level: ["1","2","3","4","1","1","2", - "1","2","3","4","1","1","2", - "1","2","3","4","1","1","2", - "1","2","3","4","1","1","2"] - ob value: ["-1073741824.00000","-1073741824.00000","1.20000","3.11000","1.00000","4.00000","0.00000", - "-1073741824.00000","-1073741824.00000","-1073741824.00000","3.22000","2.00000","3.00000","4.00000", - "-1073741824.00000","-1073741824.00000","1.60000","3.33000","2.00000","3.00000","4.00000", - "-1073741824.00000","-1073741824.00000","1.80000", "3.44000","1.00000","3.00000","0.00000"] - lat: ["21.00000", "21.00000","21.00000", "21.00000","21.00000","21.00000","21.00000", - "22.00000", "22.00000","22.00000", "22.00000","22.00000","22.00000","22.00000", - "-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", - "24.00000", "24.00000","24.00000", "24.00000","24.00000", "24.00000","24.00000"] - lon: ["31.00000", "31.00000","31.00000", "31.00000","31.00000","31.00000","31.00000", - "32.00000", "32.00000","32.00000", "32.00000","32.00000","32.00000","32.00000", - "33.00000", "33.00000","33.00000", "33.00000","33.00000","33.00000","33.00000", - "34.00000", "34.00000","34.00000", "34.00000","34.00000", "34.00000","34.00000"] - time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000","-3540.00000","-3540.00000","-3540.00000", - "-3480.00000", "-3480.00000","-3480.00000", "-3480.00000","-3480.00000","-3480.00000","-3480.00000", - "-3420.00000", "-3420.00000","-3420.00000", "-3420.00000","-3420.00000","-3420.00000","-3420.00000", - "-3360.00000", "-3360.00000","-3360.00000", "-3360.00000","-3360.00000", "-3360.00000","-3360.00000"] - Callsign: ["station_1", "station_1","station_1", "station_1","station_1","station_1", "station_1", - "station_2", "station_2","station_2", "station_2","station_2","station_2","station_2", - "station_3", "station_3","station_3", "station_3","station_3","station_3","station_3", - "station_4", "station_4","station_4", "station_4","station_4", "station_4","station_4"] + field: ["10", "10", "10", "54", "55", "55", + "10", "10", "10", "54", "55", "55", + "10", "10", "10", "54", "55", "55", + "10", "10", "10", "54", "55", "55"] + level: ["1","2","3","1","1","2", + "1","2","3","1","1","2", + "1","2","3","1","1","2", + "1","2","3","1","1","2"] + ob value: ["-1073741824.00000","1.20000","3.11000","1.00000","3.00000","0.00000", + "-1073741824.00000","-1073741824.00000","3.22000","2.00000","2.00000","3.00000", + "-1073741824.00000","1.60000","3.33000","2.00000","2.00000","3.00000", + "-1073741824.00000","1.80000", "3.44000","1.00000","2.00000","0.00000"] + lat: ["21.00000", "21.00000", "21.00000","21.00000","21.00000","21.00000", + "22.00000", "22.00000", "22.00000","22.00000","22.00000","22.00000", + "-23.00000", "-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", + "24.00000", "24.00000", "24.00000","24.00000", "24.00000","24.00000"] + lon: ["31.00000", "31.00000", "31.00000","31.00000","31.00000","31.00000", + "32.00000", "32.00000", "32.00000","32.00000","32.00000","32.00000", + "33.00000", "33.00000", "33.00000","33.00000","33.00000","33.00000", + "34.00000", "34.00000", "34.00000","34.00000", "34.00000","34.00000"] + time: ["-3540.00000", "-3540.00000", "-3540.00000","-3540.00000","-3540.00000","-3540.00000", + "-3480.00000", "-3480.00000", "-3480.00000","-3480.00000","-3480.00000","-3480.00000", + "-3420.00000", "-3420.00000", "-3420.00000","-3420.00000","-3420.00000","-3420.00000", + "-3360.00000", "-3360.00000", "-3360.00000","-3360.00000", "-3360.00000","-3360.00000"] + Callsign: ["station_1", "station_1", "station_1","station_1","station_1", "station_1", + "station_2", "station_2", "station_2","station_2","station_2","station_2", + "station_3", "station_3", "station_3","station_3","station_3","station_3", + "station_4", "station_4", "station_4","station_4", "station_4","station_4"] HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 diff --git a/test/testinput/057_VarField_emissivity.yaml b/test/testinput/057_VarField_emissivity.yaml index e03c708d..fe1bfba6 100644 --- a/test/testinput/057_VarField_emissivity.yaml +++ b/test/testinput/057_VarField_emissivity.yaml @@ -25,7 +25,7 @@ observations: reject_obs_with_all_variables_failing_qc: true namelist_directory: testinput/VarObsWriterNamelists_057_VarField_emissivity # channel_offset: 1 - # varChannels: 2-4 + varChannels: 2-4 compress_var_channels: true size_of_varobs_array: 4 general_mode: verbose diff --git a/test/testinput/076_VarField_britempvarerror.yaml b/test/testinput/076_VarField_britempvarerror.yaml index a6c7443a..66fda4c7 100644 --- a/test/testinput/076_VarField_britempvarerror.yaml +++ b/test/testinput/076_VarField_britempvarerror.yaml @@ -26,6 +26,7 @@ observations: namelist_directory: testinput/VarObsWriterNamelists_076_VarField_britempvarerror # channel_offset: 1 varChannels: 2-4 + compress_var_channels: true size_of_varobs_array: 4 general_mode: verbose - filter: VarObs Checker diff --git a/test/testinput/varobswriter_globalnamelist_atovs.yaml b/test/testinput/varobswriter_globalnamelist_atovs.yaml index 770c90ec..ac30b2ff 100644 --- a/test/testinput/varobswriter_globalnamelist_atovs.yaml +++ b/test/testinput/varobswriter_globalnamelist_atovs.yaml @@ -26,9 +26,10 @@ observations: namelist_directory: ../etc/global/varobs general_mode: verbose # channel_offset: 1 - # varChannels: 2,4 + varChannels: 2,4 compress_var_channels: true size_of_varobs_array: 3 + # output_directory: /data/users/olewis/ - filter: VarObs Checker expected_main_table_columns: # Only channels 1 and 3 are simulated and hence written to the VarObs file. From 6cf78b2ff58a8da15ba1e0ff2c5419d0314074a8 Mon Sep 17 00:00:00 2001 From: orlewis Date: Fri, 25 Aug 2023 12:11:04 +0100 Subject: [PATCH 21/44] Change to the logic to work for var channels not compressed. --- src/opsinputs/opsinputs_fill_mod.F90 | 14 ++-- src/opsinputs/opsinputs_varobswriter_mod.F90 | 1 - test/testinput/010_VarField_britemp.yaml | 77 ++++++++++++++++++++ 3 files changed, 85 insertions(+), 7 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 9488d7ef..b762f20c 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1109,13 +1109,14 @@ subroutine opsinputs_fill_fillreal2d_norecords( & where (VarValue /= MissingDouble) Real2(:, iChannel+offsetsize) = VarValue end where + + else + WRITE(*,*) "here1" + where (VarValue /= MissingDouble) + Real2(:, varChannels(iChannel)) = VarValue + end where + WRITE(*,*) "shape of real2", shape(Real2) end if - else - WRITE(*,*) "here1" - where (VarValue /= MissingDouble) - Real2(:, varChannels(iChannel)) = VarValue - end where - WRITE(*,*) "shape of real2", shape(Real2) end if end if else @@ -1141,6 +1142,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & end if end do end if ! Data not present? OPS will produce a warning -- we don't need to duplicate it. +WRITE(*,*) "Finished opsinputs_fill_fillreal2d_norecord" end subroutine opsinputs_fill_fillreal2d_norecords ! ------------------------------------------------------------------------------ diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index d6040b8f..17f801cd 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -1326,7 +1326,6 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & do iChannel=1, NumChannels !varObsSize WRITE(*,*) "iChannel=", iChannel, size(varChannels) if (compressVarChannels) then - WRITE(*,*) "here" do iObs=1, Ob % Header % NumObsLocal WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) if (ChannelIndices(iObs,iChannel) /= 0) then diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index 082f4473..173c4057 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -206,3 +206,80 @@ observations: HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 + + +# +# Now we test effect of actual channel +# + - obs space: + name: AMSUB + obsdatain: + engine: + type: H5File + obsfile: /data/users/olewis/JEDI_branches/test_geo_opsinputs/build-mo-bundle-jopa_env-RelWithDebInfo/opsinputs/test/Data/010_VarField_britemp.nc4 #Data/010_VarField_britemp.nc4 + obsdataout: + engine: + type: H5File + obsfile: /home/h01/olewis/map_chans.nc4 + simulated variables: [brightnessTemperature] + channels: 1,3 + obs filters: + # Double all observation errors: we want to check if error changes made by filters are + # propagated to VarObs files + - filter: BlackList + action: + name: inflate error + inflation factor: 2.0 + # Reject the first level of the first observation and the last level of the last observation + - filter: Bounds Check + minvalue: 1.15 + maxvalue: 3.35 + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + - filter: VarObs Writer + variables_for_quality_control: + - name: brightnessTemperature + channels: 1,3 + namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp + general_mode: debug + compress_var_channels: false + size_of_varobs_array: 4 + varChannels: 2, 4 #[5,7] + # output_directory: /home/h01/olewis + # output_directory: /data/users/olewis/ + - filter: VarObs Checker + expected_main_table_columns: + # In the arrays below, rows denote locations and columns levels. + field: ["10", "10", "10", "10", "54", "55", "55", + "10", "10", "10", "10", "54", "55", "55", + "10", "10", "10", "10", "54", "55", "55", + "10", "10", "10", "10", "54", "55", "55"] + level: ["1","2","3","4","1","1","2", + "1","2","3","4","1","1","2", + "1","2","3","4","1","1","2", + "1","2","3","4","1","1","2"] + ob value: ["-1073741824.00000","1.20000","-1073741824.00000","3.11000","1.00000","2.00000","4.00000", + "-1073741824.00000","-1073741824.00000","-1073741824.00000","3.22000","2.00000","2.00000","4.00000", + "-1073741824.00000","1.60000","-1073741824.00000","3.33000","2.00000","2.00000","4.00000", + "-1073741824.00000","1.80000", "-1073741824.00000","3.44000","1.00000","2.00000","4.00000"] + lat: ["21.00000", "21.00000", "21.00000", "21.00000","21.00000","21.00000","21.00000", + "22.00000", "22.00000", "22.00000", "22.00000", "22.00000","22.00000","22.00000", + "-23.00000", "-23.00000", "-23.00000","-23.00000","-23.00000", "-23.00000","-23.00000", + "24.00000", "24.00000", "24.00000","24.00000", "24.00000","24.00000","24.00000"] + lon: ["31.00000","31.00000", "31.00000", "31.00000","31.00000","31.00000","31.00000", + "32.00000", "32.00000", "32.00000","32.00000","32.00000","32.00000","32.00000", + "33.00000", "33.00000","33.00000", "33.00000","33.00000","33.00000","33.00000", + "34.00000", "34.00000", "34.00000","34.00000","34.00000", "34.00000","34.00000"] + time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000","-3540.00000","-3540.00000","-3540.00000", + "-3480.00000", "-3480.00000", "-3480.00000","-3480.00000","-3480.00000","-3480.00000","-3480.00000", + "-3420.00000", "-3420.00000", "-3420.00000","-3420.00000","-3420.00000","-3420.00000","-3420.00000", + "-3360.00000", "-3360.00000", "-3360.00000","-3360.00000","-3360.00000", "-3360.00000","-3360.00000"] + Callsign: ["station_1", "station_1","station_1", "station_1","station_1","station_1", "station_1", + "station_2", "station_2", "station_2","station_2","station_2","station_2","station_2", + "station_3", "station_3", "station_3","station_3","station_3","station_3","station_3", + "station_4", "station_4", "station_4","station_4", "station_4","station_4","station_4"] + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 From 585cdf54299fa10592ea06dfe8c7cfae366047d7 Mon Sep 17 00:00:00 2001 From: orlewis Date: Tue, 29 Aug 2023 11:28:02 +0100 Subject: [PATCH 22/44] Creating an option to increase the size of the channel array to the size of the varobs array. --- src/opsinputs/VarObsWriterParameters.h | 3 + src/opsinputs/opsinputs_varobswriter_mod.F90 | 29 +++++++- test/testinput/010_VarField_britemp.yaml | 78 ++++++++++++++++++++ 3 files changed, 106 insertions(+), 4 deletions(-) diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 9e2abdc4..929ea35a 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -94,6 +94,9 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// Compress the var channels oops::Parameter compressVarChannels{"compress_var_channels", false, this}; + /// Compress the var channels + oops::Parameter increaseChanArray{"increase_chan_array", false, this}; + /// If this list of ufo::variable is defined in the yaml a subset of the flags /// will be made with just these variables present. This will allow Fortran calls such-as /// "reject_obs_with_all_variables_failing_qc" and channel numbering to work correctly. diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 17f801cd..a3aad7af 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -192,6 +192,7 @@ module opsinputs_varobswriter_mod integer(integer64) :: NumVarChannels logical ::compressVarChannels + logical :: increaseChanArray !this stores the atmospheric levels we wish to pass to varobs integer(c_int), allocatable :: modlevs(:) @@ -241,6 +242,8 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe call f_conf % get_or_die("varChannels", self % varChannels) call f_conf % get_or_die("compress_var_channels", self % compressVarChannels) + +call f_conf % get_or_die("increase_chan_array", self % increaseChanArray) ! Setup OPS call f_conf % get_or_die("general_mode", StringValue) @@ -1177,7 +1180,8 @@ subroutine opsinputs_varobswriter_populateobservations( & call opsinputs_varobswriter_fillchannumandnumchans( & Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & - FillNumChans, self % compressVarChannels, self % size_of_varobs_array) + FillNumChans, self % compressVarChannels, self % size_of_varobs_array, & + self % increaseChanArray) end if end do @@ -1222,7 +1226,8 @@ end subroutine opsinputs_varobswriter_fillreportflags !> e.g. for HIRS & AMSUA !> the number of these channels is stored in Ob % NumChans. subroutine opsinputs_varobswriter_fillchannumandnumchans( & - Ob, ObsSpace, channels, varChannels, Flags, FillChanNum, FillNumChans, compressVarChannels, varObsSize) + Ob, ObsSpace, channels, varChannels, Flags, FillChanNum, FillNumChans, compressVarChannels, & + varObsSize, increaseChanArray) implicit none @@ -1235,11 +1240,13 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & logical, intent(in) :: FillChanNum, FillNumChans logical :: compressVarChannels integer(integer64), optional, intent(in) :: varObsSize +logical, optional, intent(in) :: increaseChanArray ! Local declarations: integer(integer64) :: NumChannels -integer(integer64) :: ChannelIndicesVar(Ob % Header % NumObsLocal, size(varChannels)) !varObssize +!integer(integer64) :: ChannelIndicesVar(Ob % Header % NumObsLocal, size(varChannels)) !varObssize +integer(integer64), allocatable :: ChannelIndicesVar(:,:) !varObssize integer(integer64) :: ChannelIndices(Ob % Header % NumObsLocal, size(channels)) integer(integer64) :: ChannelCounts(Ob % Header % NumObsLocal) integer :: iChannel @@ -1256,6 +1263,15 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & NumChannels = size(channels) if (NumChannels == 0) return +WRITE(*,*) "increaseChanArray", increaseChanArray +if (increaseChanArray) then + NumChannels = varObsSize + allocate(ChannelIndicesVar(Ob % Header % NumObsLocal, varObsSize)) +else + allocate(ChannelIndicesVar(Ob % Header % NumObsLocal, size(varChannels))) +end if +WRITE(*,*) varObsSize, NumChannels + call opsinputs_varobswriter_findchannelspassingqc( & Ob % Header % NumObsLocal, ObsSpace, channels, Flags, ChannelIndices, ChannelCounts) WRITE(*,*) "QC been done" @@ -1292,8 +1308,13 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & WRITE(*,*) "Finished VarChannels with compression" else + WRITE(*,*) "NumChannels=", NumChannels do iChannel=1, NumChannels - ChannelIndicesVar(:, iChannel) = varChannels(iChannel) + if (iChannel > size(varChannels)) then + ChannelIndicesVar(:, iChannel) = -32768 + else + ChannelIndicesVar(:, iChannel) = varChannels(iChannel) + end if end do Ob % ChanNum = ChannelIndicesVar end if diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index 173c4057..f946985d 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -283,3 +283,81 @@ observations: HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 + + +# +# Now we test effect of actual channel +# + - obs space: + name: AMSUB + obsdatain: + engine: + type: H5File + obsfile: /data/users/olewis/JEDI_branches/test_geo_opsinputs/build-mo-bundle-jopa_env-RelWithDebInfo/opsinputs/test/Data/010_VarField_britemp.nc4 #Data/010_VarField_britemp.nc4 + obsdataout: + engine: + type: H5File + obsfile: /home/h01/olewis/map_chans.nc4 + simulated variables: [brightnessTemperature] + channels: 1,3 + obs filters: + # Double all observation errors: we want to check if error changes made by filters are + # propagated to VarObs files + - filter: BlackList + action: + name: inflate error + inflation factor: 2.0 + # Reject the first level of the first observation and the last level of the last observation + - filter: Bounds Check + minvalue: 1.15 + maxvalue: 3.35 + # Set the flag of observations with missing values to "pass": we want to check if these + # values are encoded correctly in the VarObsFile. + - filter: Reset Flags to Pass + flags_to_reset: [10, 15] # missing, Hfailed + - filter: VarObs Writer + variables_for_quality_control: + - name: brightnessTemperature + channels: 1,3 + namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp + general_mode: debug + compress_var_channels: false + size_of_varobs_array: 4 + varChannels: 2, 4 #[5,7] + increase_chan_array: true + # output_directory: /home/h01/olewis + # output_directory: /data/users/olewis/ + - filter: VarObs Checker + expected_main_table_columns: + # In the arrays below, rows denote locations and columns levels. + field: ["10", "10", "10", "10", "54", "55", "55", "55", "55", + "10", "10", "10", "10", "54", "55", "55", "55", "55", + "10", "10", "10", "10", "54", "55", "55", "55", "55", + "10", "10", "10", "10", "54", "55", "55", "55", "55"] + level: ["1","2","3","4","1","1","2","3","4", + "1","2","3","4","1","1","2","3","4", + "1","2","3","4","1","1","2","3","4", + "1","2","3","4","1","1","2","3","4"] + ob value: ["-1073741824.00000","1.20000","-1073741824.00000","3.11000","1.00000","2.00000","4.00000","-32768.00000","-32768.00000", + "-1073741824.00000","-1073741824.00000","-1073741824.00000","3.22000","2.00000","2.00000","4.00000","-32768.00000","-32768.00000", + "-1073741824.00000","1.60000","-1073741824.00000","3.33000","2.00000","2.00000","4.00000","-32768.00000","-32768.00000", + "-1073741824.00000","1.80000", "-1073741824.00000","3.44000","1.00000","2.00000","4.00000","-32768.00000","-32768.00000"] + lat: ["21.00000", "21.00000", "21.00000", "21.00000","21.00000","21.00000","21.00000","21.00000","21.00000", + "22.00000", "22.00000", "22.00000", "22.00000", "22.00000","22.00000","22.00000","22.00000","22.00000", + "-23.00000", "-23.00000", "-23.00000","-23.00000","-23.00000", "-23.00000","-23.00000","-23.00000","-23.00000", + "24.00000", "24.00000", "24.00000","24.00000", "24.00000","24.00000","24.00000","24.00000","24.00000"] + lon: ["31.00000","31.00000", "31.00000", "31.00000","31.00000","31.00000","31.00000","31.00000","31.00000", + "32.00000", "32.00000", "32.00000","32.00000","32.00000","32.00000","32.00000","32.00000","32.00000", + "33.00000", "33.00000","33.00000", "33.00000","33.00000","33.00000","33.00000","33.00000","33.00000", + "34.00000", "34.00000", "34.00000","34.00000","34.00000", "34.00000","34.00000","34.00000","34.00000"] + time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000","-3540.00000","-3540.00000","-3540.00000","-3540.00000","-3540.00000", + "-3480.00000", "-3480.00000", "-3480.00000","-3480.00000","-3480.00000","-3480.00000","-3480.00000","-3480.00000","-3480.00000", + "-3420.00000", "-3420.00000", "-3420.00000","-3420.00000","-3420.00000","-3420.00000","-3420.00000","-3420.00000","-3420.00000", + "-3360.00000", "-3360.00000", "-3360.00000","-3360.00000","-3360.00000", "-3360.00000","-3360.00000","-3360.00000","-3360.00000"] + Callsign: ["station_1", "station_1","station_1", "station_1","station_1","station_1", "station_1","station_1", "station_1", + "station_2", "station_2", "station_2","station_2","station_2","station_2","station_2","station_2", "station_2", + "station_3", "station_3", "station_3","station_3","station_3","station_3","station_3","station_3", "station_3", + "station_4", "station_4", "station_4","station_4", "station_4","station_4","station_4","station_4", "station_4"] + HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. + benchmarkFlag: 1000 # just to keep the ObsFilters test happy + flaggedBenchmark: 0 From 604247d47bdccd7a76735296f35e97bb7c9a12d4 Mon Sep 17 00:00:00 2001 From: orlewis Date: Fri, 15 Sep 2023 16:46:08 +0100 Subject: [PATCH 23/44] remove some of the write statements --- src/opsinputs/opsinputs_varobswriter_mod.F90 | 157 +------------------ 1 file changed, 2 insertions(+), 155 deletions(-) diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index a204761d..18a92c20 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -577,15 +577,13 @@ subroutine opsinputs_varobswriter_post( & call opsinputs_varobswriter_allocateobservations(self, ObsSpace, JediToOpsLayoutMapping, Ob) call opsinputs_varobswriter_populateobservations(self, ObsSpace, JediToOpsLayoutMapping, & Flags, ObsErrors, Ob) -WRITE(*,*) "Finished populate Ob" + call opsinputs_varobswriter_populatecxheader(self, CxHeader) -WRITE(*,*) "Finished populate cx" call Ops_CreateVarobs (Ob, & ! in CxHeader, & ! in AssimDataFormat_VAR, & NumVarobsTotal) -WRITE(*,*) "Finished CreatVarobs" call Ob % deallocate() call CxHeader % dealloc() @@ -748,8 +746,6 @@ subroutine opsinputs_varobswriter_populateobservations( & if (obsspace_has(ObsSpace, "MetaData", "stationIdentification")) then call opsinputs_fill_fillstring(Ob % Header % Callsign, "Callsign", JediToOpsLayoutMapping, & LenCallSign, Ob % Callsign, ObsSpace, "stationIdentification", "MetaData") - WRITE(*,*) "LenCallSign=", LenCallSign - WRITE(*,*) Ob % Callsign else if (obsspace_has(ObsSpace, "MetaData", "satelliteIdentifier")) then call opsinputs_varobswriter_fillsatid(Ob, ObsSpace, JediToOpsLayoutMapping) call Ops_Alloc(Ob % Header % Callsign, "Callsign", JediToOpsLayoutMapping % NumOpsObs, Ob % Callsign) @@ -863,7 +859,6 @@ subroutine opsinputs_varobswriter_populateobservations( & ! TODO(someone): handle this varfield ! call Ops_Alloc(Ob % Header % lwp, "LWP", Ob % Header % NumObsLocal, Ob % lwp) case (VarField_britemp) - WRITE(*,*) "At britemp" call opsinputs_fill_fillreal2d( & Ob % Header % CorBriTemp, "CorBriTemp", JediToOpsLayoutMapping, Ob % CorBriTemp, & ObsSpace, self % channels, self % VarobsLength, "brightnessTemperature", "BiasCorrObsValue", & @@ -881,7 +876,6 @@ subroutine opsinputs_varobswriter_populateobservations( & Ob % Header % Zstation, "Zstation", JediToOpsLayoutMapping, Ob % Zstation, & ObsSpace, "stationElevation", "MetaData") case (VarField_mwemiss) - WRITE(*,*) "At mwemissivity" call opsinputs_fill_fillreal2d( & Ob % Header % MwEmiss, "MwEmiss", JediToOpsLayoutMapping, Ob % MwEmiss, & ObsSpace, self % channels, self % VarobsLength, "emissivity", "Emiss") @@ -999,7 +993,6 @@ subroutine opsinputs_varobswriter_populateobservations( & case (VarField_ChanNum) FillChanNum = .true. case (VarField_Emissivity) - WRITE(*,*) "Case is Emissivity" call opsinputs_fill_fillreal2d( & Ob % Header % Emissivity, "Emissivity", JediToOpsLayoutMapping, Ob % Emissivity, & ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar", & @@ -1177,10 +1170,6 @@ subroutine opsinputs_varobswriter_populateobservations( & if (FillChanNum .or. FillNumChans) then WRITE(*,*) "Call opsinputs_varobswriter_fillchannumandnumchans" - WRITE(*,*) "size(self % varChannels) =", size(self % varChannels) - WRITE(*,*) self % varChannels - WRITE(*,*) "size(self % channels)=", size(self % channels) - WRITE(*,*) self % size_of_varobs_array call opsinputs_varobswriter_fillchannumandnumchans( & Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & @@ -1267,44 +1256,26 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & NumChannels = size(channels) if (NumChannels == 0) return -WRITE(*,*) "increaseChanArray", increaseChanArray if (increaseChanArray) then NumChannels = varObsSize allocate(ChannelIndicesVar(Ob % Header % NumObsLocal, varObsSize)) else allocate(ChannelIndicesVar(Ob % Header % NumObsLocal, size(varChannels))) end if -WRITE(*,*) varObsSize, NumChannels call opsinputs_varobswriter_findchannelspassingqc( & Ob % Header % NumObsLocal, ObsSpace, channels, Flags, ChannelIndices, ChannelCounts) -WRITE(*,*) "QC been done" -WRITE(*,*) "Initial ChannelIndices=", ChannelIndices(:,1) -WRITE(*,*) "Initial ChannelIndices=", ChannelIndices(:,2) - -!Ob % ChanNum = ChannelIndices -!WRITE(*,*) Ob % ChanNum -!Ob % ChanNum = Ob % ChanNum + 1 -!WRITE(*,*) Ob % ChanNum -WRITE(*,*) "compressVarChannels = ", compressVarChannels ChannelIndicesVar(:,:) = 0 -WRITE(*,*) "orig ChannelIndicesVar", ChannelIndicesVar -WRITE(*,*) "size of varChannels", size(varChannels) -!WRITE(*,*) ChannelIndicesVar(:,1) -WRITE(*,*) "FillChanNum", FillChanNum - if (FillChanNum) then call Ops_Alloc(Ob % Header % ChanNum, "ChanNum", Ob % Header % NumObsLocal, Ob % ChanNum, & num_levels = NumChannels) - WRITE(*,*) "In FillChanNum" if ((varObsSize /= 0)) then if (varObsSize > size(channels)) then if (size(varChannels) > 0 .and. (size(channels) == size(varChannels))) then if (compressVarChannels) then offsetsize = abs(varChannels(1) - channels(1)) - write(*,*) "offsetsize=", offsetsize Ob % ChanNum = ChannelIndices where (Ob % ChanNum > 0) Ob % ChanNum = Ob % ChanNum + int(offsetsize, kind=integer64) @@ -1312,7 +1283,6 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & WRITE(*,*) "Finished VarChannels with compression" else - WRITE(*,*) "NumChannels=", NumChannels do iChannel=1, NumChannels if (iChannel > size(varChannels)) then ChannelIndicesVar(:, iChannel) = -32768 @@ -1323,10 +1293,6 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & Ob % ChanNum = ChannelIndicesVar end if - ! WRITE(*,*) "ChannelIndicesVar1=", ChannelIndicesVar(:,1) - ! WRITE(*,*) "ChannelIndicesVar2=", ChannelIndicesVar(:,2) - !end do - !Ob % ChanNum = ChannelIndicesVar else WRITE(*,*) "No varChannel, varObsSize is greater" WRITE(*,*) compressVarChannels @@ -1335,12 +1301,9 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & write(*,*) "Compressed var channells" Ob % ChanNum = ChannelIndices else - WRITE(*,*) "Here now" do iChannel=1, size(Channels) ChannelIndices(:,iChannel) = Channels(iChannel) end do - WRITE(*,*) ChannelIndices(:,1) - WRITE(*,*) ChannelIndices(:,2) Ob % ChanNum = ChannelIndices end if end if @@ -1349,10 +1312,8 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & WRITE(*,*) compressVarChannels if (size(varChannels) > 0) then do iChannel=1, NumChannels !varObsSize - WRITE(*,*) "iChannel=", iChannel, size(varChannels) if (compressVarChannels) then do iObs=1, Ob % Header % NumObsLocal - WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) if (ChannelIndices(iObs,iChannel) /= 0) then ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) end if @@ -1372,125 +1333,11 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & else WRITE(*,*) "Not FillChanNum" end if -! if (varObsSize /= 0) then -! if (varObsSize > size(channels)) then -! if (compressVarChannels) then -! WRITE(*,*) "Doing offset" -! offsetsize = varObsSize - size(channels) -! WRITE(*,*) "offsetsize=", offsetsize -! Ob % ChanNum = ChannelIndices -! WRITE(*,*) ChannelIndices(:,1) -! WRITE(*,*) ChannelIndices(:,2) -! where (Ob % ChanNum > 0) -! Ob % ChanNum = Ob % ChanNum + int(offsetsize, kind=integer64) -! end where -! else -! if (size(varChannels) > 0) then -! if (size(channels) == size(varChannels)) then -! WRITE(*,*) "Number of channels=", NumChannels -! WRITE(*,*) "#varObsSize", varObsSize -! WRITE(*,*) Ob % Header % NumObsLocal -! do iChannel=1, NumChannels !varObsSize -! WRITE(*,*) "iChannel=", iChannel, size(varChannels) -! if (compressVarChannels) then -! do iObs=1, Ob % Header % NumObsLocal -! WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) -! if (ChannelIndices(iObs,iChannel) /= 0) then -! ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) -! end if -! end do -! else -! ChannelIndicesVar(:, iChannel) = varChannels(iChannel) -! end if - -! WRITE(*,*) "ChannelIndicesVar1=", ChannelIndicesVar(:,1) -! WRITE(*,*) "ChannelIndicesVar2=", ChannelIndicesVar(:,2) -! end do -! Ob % ChanNum = ChannelIndicesVar -! end if -! end if -! end if -! else if (varObsSize == size(channels)) then -! WRITE(*,*) "varObsSize==size(channels)" -! WRITE(*,*) compressVarChannels -! if (size(varChannels) > 0) then -! do iChannel=1, NumChannels !varObsSize -! WRITE(*,*) "iChannel=", iChannel, size(varChannels) -! if (compressVarChannels) then -! WRITE(*,*) "here" -! do iObs=1, Ob % Header % NumObsLocal -! WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) -! if (ChannelIndices(iObs,iChannel) /= 0) then -! ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) -! end if -! end do -! end if -! end do -! Ob % ChanNum = ChannelIndicesVar -! else -! WRITE(*,*) "Set ChanNum to ChannelIndices" -! Ob % ChanNum = ChannelIndices -! end if -! end if -! else -! WRITE(*,*) "The basic case" -! Ob % ChanNum = ChannelIndices -! end if -!else -! WRITE(*,*) "Not FillChanNum" -!end if -!! WRITE(*,*) "#### In if FillChnaNum" -!! if (size(varChannels) > 0) then -!! if (size(channels) == size(varChannels)) then -!! WRITE(*,*) "Number of channels=", NumChannels -!! WRITE(*,*) "#varObsSize", varObsSize -!! WRITE(*,*) Ob % Header % NumObsLocal -!! do iChannel=1, NumChannels !varObsSize -!! WRITE(*,*) "iChannel=", iChannel, size(varChannels) -!! ! if ((iChannel > size(varChannels))) then -!! ! WRITE(*,*) "in missing", MissingDouble -!! ! ChannelIndicesVar(:, iChannel) = MissingDouble -!! ! WRITE(*,*) "Going to continue" -!! ! continue -!! ! end if -!! compressVarChannels=.TRUE. -!! if (compressVarChannels) then -!! do iObs=1, Ob % Header % NumObsLocal -!! WRITE(*,*) "VarChannels=", varChannels(ChannelIndices(iObs,iChannel)) -!! if (ChannelIndices(iObs,iChannel) /= 0) then -!! ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) -!! end if -!! end do -!! else -!! ChannelIndicesVar(:, iChannel) = varChannels(iChannel) -!! end if - -!! WRITE(*,*) "ChannelIndicesVar1=", ChannelIndicesVar(:,1) -!! WRITE(*,*) "ChannelIndicesVar2=", ChannelIndicesVar(:,2) - -!! end do -!! Ob % ChanNum = ChannelIndicesVar -!! end if -!! else -!! WRITE(*,*) "Set ChanNum to ChannelIndices" -!! Ob % ChanNum = ChannelIndices -!! end if -! ! else -! ! Ob % ChanNum = ChannelIndices -! !only apply offset to actual channels in list, not missing data -! ! where (Ob % ChanNum > 0) -! ! Ob % ChanNum = Ob % ChanNum + int(OffsetChans, kind=integer64) -! ! end where -! ! end if -!!else -!! WRITE(*,*) "Not FillChanNum" -!!end if -!!WRITE(*,*) Ob % ChanNum + if (FillNumChans) then call Ops_Alloc(Ob % Header % NumChans, "NumChans", Ob % Header % NumObsLocal, Ob % NumChans) Ob % NumChans = ChannelCounts end if -WRITE(*,*) "Finished fillchans" end subroutine opsinputs_varobswriter_fillchannumandnumchans ! ------------------------------------------------------------------------------ From 6de28dbc69484258a58cd8562040ea2aa3f500bb Mon Sep 17 00:00:00 2001 From: orlewis Date: Fri, 22 Sep 2023 13:53:55 +0100 Subject: [PATCH 24/44] Some tidy up, removing write statements etc. --- src/opsinputs/VarObsWriterParameters.h | 7 +--- src/opsinputs/opsinputs_fill_mod.F90 | 40 ++----------------- src/opsinputs/opsinputs_varobswriter_mod.F90 | 27 +------------ test/testinput/010_VarField_britemp.yaml | 19 ++------- test/testinput/055_VarField_channum.yaml | 1 - test/testinput/057_VarField_emissivity.yaml | 1 - .../076_VarField_britempvarerror.yaml | 1 - .../varobswriter_globalnamelist_atovs.yaml | 2 - 8 files changed, 10 insertions(+), 88 deletions(-) diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 929ea35a..4b6cb449 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -85,16 +85,13 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// Therefore for atovs brightness_tmperatuere => size_of_varobs_array = 40. oops::Parameter size_of_varobs_array{"size_of_varobs_array", 0, this}; - /// oops::Parameter jopaChannels{"jopaChannels", "", this}; - - /// oops::Parameter> varChannels{"varChannels", {}, this}; + /// List of channels which are expected for Var. oops::Parameter varChannels{"varChannels", "", this}; - /// oops::RequiredParameter chanlist{"cost channels list", this}; /// Compress the var channels oops::Parameter compressVarChannels{"compress_var_channels", false, this}; - /// Compress the var channels + /// Increase the channel array size to the same size as the varobs array oops::Parameter increaseChanArray{"increase_chan_array", false, this}; /// If this list of ufo::variable is defined in the yaml a subset of the flags diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index b762f20c..b5dd45d6 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1065,10 +1065,9 @@ subroutine opsinputs_fill_fillreal2d_norecords( & !compressVarChannels=.TRUE. ! Body: -WRITE(*,*) "opsinputs_fill_mod - no records" + MissingDouble = missing_value(0.0_c_double) -WRITE(*,*) "JediVarName=", JediVarName, "Channels", Channels JediVarNamesWithChannels = opsinputs_fill_varnames_with_channels(JediVarName, Channels) !take into account offsetting of 2nd dimension if required @@ -1076,46 +1075,32 @@ subroutine opsinputs_fill_fillreal2d_norecords( & !e.g. HIRS in ATOVS stream numchans = size(JediVarNamesWithChannels) -WRITE(*,*) "numchans=", numchans -WRITE(*,*) "numchans=", numchans if (present(sizeVarObs) )then if (sizeVarObs > 0) then numchans = sizeVarObs end if end if -WRITE(*,*) "numchans=", numchans if (obsspace_has(ObsSpace, JediVarGroup, JediVarNamesWithChannels(1))) then ! Allocate OPS data structures call Ops_Alloc(Hdr, OpsVarName, NumObs, Real2, & num_levels = int(numchans, kind=integer64)) do iChannel = 1, size(JediVarNamesWithChannels) - ! Retrieve data from JEDI - ! if (iChannel <= size(JediVarNamesWithChannels)) then - ! WRITE(*,*) "JediVarNamesWithChannels(iChannel)=", JediVarNamesWithChannels(iChannel) call obsspace_get_db(ObsSpace, JediVarGroup, JediVarNamesWithChannels(iChannel), VarValue) - ! end if - if (present(varChannels)) then if (size(varChannels) > 0) then - WRITE(*,*) "here" if (iChannel <= size(varChannels)) then if (compressVarChannels) then offsetsize = abs(varChannels(1) - channels(1)) - write(*,*) "offsetsize=", offsetsize - WRITE(*,*) "REAL2 with offsetsize" where (VarValue /= MissingDouble) Real2(:, iChannel+offsetsize) = VarValue end where - else - WRITE(*,*) "here1" where (VarValue /= MissingDouble) Real2(:, varChannels(iChannel)) = VarValue end where - WRITE(*,*) "shape of real2", shape(Real2) end if end if end if @@ -1123,18 +1108,15 @@ subroutine opsinputs_fill_fillreal2d_norecords( & if ((present(sizeVarObs)) .and. (sizeVarObs > size(channels))) then if (compressVarChannels) then offsetsize = sizeVarObs - size(channels) - ! WRITE(*,*) "REAL2 with offsetsize" where (VarValue /= MissingDouble) Real2(:, iChannel) = VarValue end where else - WRITE(*,*) "In noncompressed larger size" where (VarValue /= MissingDouble) Real2(:, Channels(iChannel)) = VarValue end where end if else - WRITE(*,*) "real2 novar" where (VarValue /= MissingDouble) Real2(:, iChannel) = VarValue end where @@ -1142,7 +1124,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & end if end do end if ! Data not present? OPS will produce a warning -- we don't need to duplicate it. -WRITE(*,*) "Finished opsinputs_fill_fillreal2d_norecord" + end subroutine opsinputs_fill_fillreal2d_norecords ! ------------------------------------------------------------------------------ @@ -1289,13 +1271,10 @@ subroutine opsinputs_fill_fillreal2d( & Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, VarobsLength, JediVarName, JediVarGroup) else if (size(varChannels) > 0) then - WRITE(*,*) "opsinputs_fill_mod - varchannel option" call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & JediVarName, JediVarGroup, compressVarChannels, sizeVarObs, varChannels) else - WRITE(*,*) "##In no VarChannels" - WRITE(*,*) "sizeVarObs", sizeVarObs call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & JediVarName, JediVarGroup, compressVarChannels, sizeVarObs) @@ -1758,7 +1737,6 @@ subroutine opsinputs_fill_fillstring( & ! Fill the OPS data structures call Ops_Alloc(Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, String1) - WRITE(*,*) "loop nubmers", JediToOpsLayoutMapping % NumOpsObs do i = 1, JediToOpsLayoutMapping % NumOpsObs if (JediToOpsLayoutMapping % ConvertRecordsToMultilevelObs) then if (JediToOpsLayoutMapping % RecordStarts(i + 1) > JediToOpsLayoutMapping % RecordStarts(i)) then @@ -1767,7 +1745,6 @@ subroutine opsinputs_fill_fillstring( & JediToOpsLayoutMapping % RecordStarts(i))) end if else - WRITE(*,*) "VarValue(i)=", VarValue(i), i String1(i) = VarValue(i) end if end do @@ -2326,15 +2303,8 @@ function opsinputs_fill_varnames_with_channels(VarName, Channels) result(VarName !character(len=max_varname_with_channel_length) :: VarNames_emis(max(size(Channels), 1)) -write(*,*) max_varname_with_channel_length - -WRITE(*,*) "opsinputs_fill_mod - opsinputs_fill_varnames_with_channels" -WRITE(*,*) size(Channels), VarName -!allocate(VarNames(max(size(Channels), 1))) -!deallocate(VarNames) -!allocate(VarNames(max(size(Channels), 1))) !if (Varname=="emissivity") then ! do ichan = 1, size(Channels) ! write(*,*) "here" @@ -2348,17 +2318,13 @@ function opsinputs_fill_varnames_with_channels(VarName, Channels) result(VarName if (size(Channels) == 0) then VarNames(1) = VarName else - write(*,*) "start loop" do ichan = 1, size(Channels) - WRITE(*,*) VarName, Channels(ichan) - WRITE(*,*) size(VarNames) write (VarNames(ichan),'(A,"_",I0)') VarName, Channels(ichan) - WRITE(*,*) "In loop", VarNames(ichan) end do end if !end if -WRITE(*,*) "opsinputs_fill_mod - opsinputs_fill_varnames_with_channels finished" + end function opsinputs_fill_varnames_with_channels diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 18a92c20..636d019a 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -291,7 +291,6 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe opsinputs_varobswriter_create = .false. return end if - call datetime_create(StringValue, self % validitytime) call f_conf % get_or_die("reject_obs_with_any_variable_failing_qc", & @@ -318,11 +317,6 @@ function opsinputs_varobswriter_create(self, f_conf, comm_is_valid, comm, channe call f_conf % get_or_die("compress_var_channels", self % compressVarChannels) -!call f_conf % get_or_die("varChannels", self % varChannels) -!WRITE(*,*) "get or die varChannels", self % varChannels - -!call f_conf % get_or_die("jopaChannels", self % jopaChannels) - ! Updates the varbc flag passedaround by a module in OPS call f_conf % get_or_die("output_varbc_predictors", BoolValue) VarBC = BoolValue @@ -577,7 +571,6 @@ subroutine opsinputs_varobswriter_post( & call opsinputs_varobswriter_allocateobservations(self, ObsSpace, JediToOpsLayoutMapping, Ob) call opsinputs_varobswriter_populateobservations(self, ObsSpace, JediToOpsLayoutMapping, & Flags, ObsErrors, Ob) - call opsinputs_varobswriter_populatecxheader(self, CxHeader) call Ops_CreateVarobs (Ob, & ! in @@ -997,7 +990,6 @@ subroutine opsinputs_varobswriter_populateobservations( & Ob % Header % Emissivity, "Emissivity", JediToOpsLayoutMapping, Ob % Emissivity, & ObsSpace, self % channels, self % VarobsLength, "emissivity", "OneDVar", & self % compressVarChannels, self % size_of_varobs_array, self % varChannels) - !self % varChannels) case (VarField_QCinfo) ! TODO(someone): This will come from a variable generated by the 1D-Var filter. Its name and ! group are not known yet. Once they are, replace the placeholders in the call below. @@ -1169,7 +1161,6 @@ subroutine opsinputs_varobswriter_populateobservations( & end select if (FillChanNum .or. FillNumChans) then - WRITE(*,*) "Call opsinputs_varobswriter_fillchannumandnumchans" call opsinputs_varobswriter_fillchannumandnumchans( & Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & @@ -1238,8 +1229,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & ! Local declarations: integer(integer64) :: NumChannels -!integer(integer64) :: ChannelIndicesVar(Ob % Header % NumObsLocal, size(varChannels)) !varObssize -integer(integer64), allocatable :: ChannelIndicesVar(:,:) !varObssize +integer(integer64), allocatable :: ChannelIndicesVar(:,:) integer(integer64) :: ChannelIndices(Ob % Header % NumObsLocal, size(channels)) integer(integer64) :: ChannelCounts(Ob % Header % NumObsLocal) integer :: iChannel @@ -1248,7 +1238,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & integer(integer64) ::array_loop integer :: offsetsize -WRITE(*,*) "Enter opsinputs_varobswriter_fillchannumandnumchans" + ! Body: MissingDouble = missing_value(0.0_c_double) @@ -1280,8 +1270,6 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & where (Ob % ChanNum > 0) Ob % ChanNum = Ob % ChanNum + int(offsetsize, kind=integer64) end where - WRITE(*,*) "Finished VarChannels with compression" - else do iChannel=1, NumChannels if (iChannel > size(varChannels)) then @@ -1292,13 +1280,8 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & end do Ob % ChanNum = ChannelIndicesVar end if - else - WRITE(*,*) "No varChannel, varObsSize is greater" - WRITE(*,*) compressVarChannels - ! compressVarChannels=.false. if (compressVarChannels) then - write(*,*) "Compressed var channells" Ob % ChanNum = ChannelIndices else do iChannel=1, size(Channels) @@ -1308,8 +1291,6 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & end if end if else if (varObsSize == size(channels)) then - WRITE(*,*) "varObsSize==size(channels)" - WRITE(*,*) compressVarChannels if (size(varChannels) > 0) then do iChannel=1, NumChannels !varObsSize if (compressVarChannels) then @@ -1322,16 +1303,12 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & end do Ob % ChanNum = ChannelIndicesVar else - WRITE(*,*) "Set ChanNum to ChannelIndices" Ob % ChanNum = ChannelIndices end if end if else - WRITE(*,*) "The basic case" Ob % ChanNum = ChannelIndices end if -else - WRITE(*,*) "Not FillChanNum" end if if (FillNumChans) then diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index f946985d..34aead52 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -24,7 +24,6 @@ observations: - filter: VarObs Writer namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp general_mode: debug - # output_directory: /data/users/olewis/ - filter: VarObs Checker expected_main_table_columns: # Only levels 1 and 3 are simulated and hence written to the VarObs file. @@ -60,7 +59,6 @@ observations: HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 - # # Now we test effect of actual channel # @@ -89,10 +87,8 @@ observations: flags_to_reset: [10, 15] # missing, Hfailed - filter: VarObs Writer namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp - # output_directory: /data/users/olewis/ general_mode: debug size_of_varobs_array: 3 - # use_actual_channels: true compress_var_channels: false - filter: VarObs Checker expected_main_table_columns: @@ -128,11 +124,8 @@ observations: HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 - - - # -# Now we test effect of actual channel +# Now we test effect varChannels and compress var channels # - obs space: name: AMSUB @@ -170,8 +163,6 @@ observations: compress_var_channels: true size_of_varobs_array: 3 varChannels: 2, 4 #[5,7] - # output_directory: /home/h01/olewis - # output_directory: /data/users/olewis/ - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. @@ -209,7 +200,7 @@ observations: # -# Now we test effect of actual channel +# Now we test effect of var channels # - obs space: name: AMSUB @@ -247,8 +238,6 @@ observations: compress_var_channels: false size_of_varobs_array: 4 varChannels: 2, 4 #[5,7] - # output_directory: /home/h01/olewis - # output_directory: /data/users/olewis/ - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. @@ -286,7 +275,7 @@ observations: # -# Now we test effect of actual channel +# Now we test effect of varchannels, compress var cahnnels and increase channel array size. # - obs space: name: AMSUB @@ -325,8 +314,6 @@ observations: size_of_varobs_array: 4 varChannels: 2, 4 #[5,7] increase_chan_array: true - # output_directory: /home/h01/olewis - # output_directory: /data/users/olewis/ - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. diff --git a/test/testinput/055_VarField_channum.yaml b/test/testinput/055_VarField_channum.yaml index 29f9413d..0d89b45d 100644 --- a/test/testinput/055_VarField_channum.yaml +++ b/test/testinput/055_VarField_channum.yaml @@ -91,7 +91,6 @@ observations: - filter: VarObs Writer namelist_directory: testinput/VarObsWriterNamelists_055_VarField_channum general_mode: debug - # channel_offset: 15 compress_var_channels: true size_of_varobs_array: 2 varChannels: 16, 17 diff --git a/test/testinput/057_VarField_emissivity.yaml b/test/testinput/057_VarField_emissivity.yaml index fe1bfba6..1a4f9177 100644 --- a/test/testinput/057_VarField_emissivity.yaml +++ b/test/testinput/057_VarField_emissivity.yaml @@ -24,7 +24,6 @@ observations: - filter: VarObs Writer reject_obs_with_all_variables_failing_qc: true namelist_directory: testinput/VarObsWriterNamelists_057_VarField_emissivity - # channel_offset: 1 varChannels: 2-4 compress_var_channels: true size_of_varobs_array: 4 diff --git a/test/testinput/076_VarField_britempvarerror.yaml b/test/testinput/076_VarField_britempvarerror.yaml index 66fda4c7..62bce9d3 100644 --- a/test/testinput/076_VarField_britempvarerror.yaml +++ b/test/testinput/076_VarField_britempvarerror.yaml @@ -24,7 +24,6 @@ observations: - filter: VarObs Writer reject_obs_with_all_variables_failing_qc: true namelist_directory: testinput/VarObsWriterNamelists_076_VarField_britempvarerror - # channel_offset: 1 varChannels: 2-4 compress_var_channels: true size_of_varobs_array: 4 diff --git a/test/testinput/varobswriter_globalnamelist_atovs.yaml b/test/testinput/varobswriter_globalnamelist_atovs.yaml index ac30b2ff..c2ef2dab 100644 --- a/test/testinput/varobswriter_globalnamelist_atovs.yaml +++ b/test/testinput/varobswriter_globalnamelist_atovs.yaml @@ -25,11 +25,9 @@ observations: - filter: VarObs Writer namelist_directory: ../etc/global/varobs general_mode: verbose - # channel_offset: 1 varChannels: 2,4 compress_var_channels: true size_of_varobs_array: 3 - # output_directory: /data/users/olewis/ - filter: VarObs Checker expected_main_table_columns: # Only channels 1 and 3 are simulated and hence written to the VarObs file. From c1d6ab6a4e866fdaaef3e05822c20d57ba881a3e Mon Sep 17 00:00:00 2001 From: orlewis Date: Fri, 22 Sep 2023 14:07:19 +0100 Subject: [PATCH 25/44] Removing outdated comments --- src/opsinputs/VarObsWriter.cc | 3 --- src/opsinputs/opsinputs_fill_mod.F90 | 1 - 2 files changed, 4 deletions(-) diff --git a/src/opsinputs/VarObsWriter.cc b/src/opsinputs/VarObsWriter.cc index 6c3e66e0..e2fc164a 100644 --- a/src/opsinputs/VarObsWriter.cc +++ b/src/opsinputs/VarObsWriter.cc @@ -72,9 +72,6 @@ VarObsWriter::VarObsWriter(ioda::ObsSpace & obsdb, const Parameters_ & params, // Avoid passing a null pointer to Fortran. const int *channelsData = channels.empty() ? &fallbackChannels : channels.data(); -/// std::set chanset = oops::parseIntSet(options_.chanlist.value()); -/// channels_.assign(chanset.begin(), chanset.end()); - std::set varchanset = oops::parseIntSet(parameters_.varChannels.value()); varchannels_.assign(varchanset.begin(), varchanset.end()); conf.set("varChannels", varchannels_); diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 01707e0e..a883d0b9 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1056,7 +1056,6 @@ subroutine opsinputs_fill_fillreal2d_norecords( & ! Local declarations: real(kind=c_double) :: VarValue(NumObs) real(kind=c_double) :: MissingDouble -!character(len=max_varname_with_channel_length) :: JediVarNamesWithChannels(max(size(Channels), 1)) character(len=200) :: JediVarNamesWithChannels(max(size(Channels), 1)) integer :: iChannel integer :: offset From 61daafd33a79a5397a2f64023f3a1ef3e696eb97 Mon Sep 17 00:00:00 2001 From: orlewis Date: Mon, 25 Sep 2023 14:27:52 +0100 Subject: [PATCH 26/44] Restore develop versions, nothing should have changed with these files. --- test/testinput/varobs_globalnamelist_atms.nc4 | Bin 16944 -> 16944 bytes test/testinput/varobs_globalnamelist_iasi.nc4 | Bin 20585 -> 20585 bytes 2 files changed, 0 insertions(+), 0 deletions(-) diff --git a/test/testinput/varobs_globalnamelist_atms.nc4 b/test/testinput/varobs_globalnamelist_atms.nc4 index 44f916faf2bdee11773dff38c538347ca5a3817c..497450408fc870362c7cbb8e344c1cd4345d4d04 100644 GIT binary patch delta 30 lcmdnc!nmP@k*Cwc#f_Isii?SZgJB|%0!!s}9q)w(P5^vf2f6?N delta 30 lcmdnc!nmP@k*Cwc#f_Isii?SZgK;8{0?YAP)hY`OoB)0e2lM~{ diff --git a/test/testinput/varobs_globalnamelist_iasi.nc4 b/test/testinput/varobs_globalnamelist_iasi.nc4 index 4c78761e64baa5dfc67622de00aac8ce841349f8..ff0d9017c7f3a3cf4a61b25c98ff9225a817a362 100644 GIT binary patch delta 31 mcmaF4fbr!5MxIU&7dKumDJ~`s4u*+53M^HgUJe@#ECT?Pm Date: Tue, 26 Sep 2023 10:55:50 +0100 Subject: [PATCH 27/44] Adding local variables to handle optional better. Dealing with review comments. --- src/opsinputs/VarObsWriter.cc | 1 + src/opsinputs/opsinputs_fill_mod.F90 | 78 +++++++++++++------- src/opsinputs/opsinputs_utils_mod.F90 | 8 +- src/opsinputs/opsinputs_varobswriter_mod.F90 | 64 ++++++++++------ 4 files changed, 93 insertions(+), 58 deletions(-) diff --git a/src/opsinputs/VarObsWriter.cc b/src/opsinputs/VarObsWriter.cc index e2fc164a..69e350e2 100644 --- a/src/opsinputs/VarObsWriter.cc +++ b/src/opsinputs/VarObsWriter.cc @@ -72,6 +72,7 @@ VarObsWriter::VarObsWriter(ioda::ObsSpace & obsdb, const Parameters_ & params, // Avoid passing a null pointer to Fortran. const int *channelsData = channels.empty() ? &fallbackChannels : channels.data(); + // Want to also set up an ordered list of channels numbered for VAR. std::set varchanset = oops::parseIntSet(parameters_.varChannels.value()); varchannels_.assign(varchanset.begin(), varchanset.end()); conf.set("varChannels", varchannels_); diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index a883d0b9..a3e8fd85 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -35,8 +35,7 @@ module opsinputs_fill_mod opsinputs_jeditoopslayoutmapping use opsinputs_utils_mod, only: & max_varname_with_channel_length, & - opsinputs_channeloffset, & - opsinputs_varchannels + opsinputs_channeloffset use GenMod_Core, only: & gen_warn, & @@ -1056,17 +1055,22 @@ subroutine opsinputs_fill_fillreal2d_norecords( & ! Local declarations: real(kind=c_double) :: VarValue(NumObs) real(kind=c_double) :: MissingDouble -character(len=200) :: JediVarNamesWithChannels(max(size(Channels), 1)) +character(len=max_varname_with_channel_lengt) :: JediVarNamesWithChannels(max(size(Channels), 1)) integer :: iChannel integer :: offset integer :: numchans -integer :: offsetsize +integer :: offsetsize +logical :: compressChannels -!compressVarChannels=.TRUE. ! Body: MissingDouble = missing_value(0.0_c_double) +compressChannels = .false. +if present(compressVarChannels) then + compressChannels = compressVarChannels +end if + JediVarNamesWithChannels = opsinputs_fill_varnames_with_channels(JediVarName, Channels) !take into account offsetting of 2nd dimension if required @@ -1086,12 +1090,17 @@ subroutine opsinputs_fill_fillreal2d_norecords( & call Ops_Alloc(Hdr, OpsVarName, NumObs, Real2, & num_levels = int(numchans, kind=integer64)) do iChannel = 1, size(JediVarNamesWithChannels) - call obsspace_get_db(ObsSpace, JediVarGroup, JediVarNamesWithChannels(iChannel), VarValue) + call obsspace_get_db(ObsSpace, JediVarGroup, JediVarNamesWithChannels(iChannel), VarValue) + + ! if VAR channels have been assigned then jopa channels will be mapped to these var channels + ! Set up the size of the array, if channels are being pushed together an offset between + ! the var and jopa channel numbers is added onto the size of the array. + ! If not compressed the positions in the array are based on the actual channel number. if (present(varChannels)) then if (size(varChannels) > 0) then if (iChannel <= size(varChannels)) then - if (compressVarChannels) then + if (compressChannels) then offsetsize = abs(varChannels(1) - channels(1)) where (VarValue /= MissingDouble) Real2(:, iChannel+offsetsize) = VarValue @@ -1104,21 +1113,23 @@ subroutine opsinputs_fill_fillreal2d_norecords( & end if end if else - if ((present(sizeVarObs)) .and. (sizeVarObs > size(channels))) then - if (compressVarChannels) then - offsetsize = sizeVarObs - size(channels) - where (VarValue /= MissingDouble) - Real2(:, iChannel) = VarValue - end where + if ((present(sizeVarObs)) then + if (sizeVarObs > size(channels))) then + if (compressChannels) then + offsetsize = sizeVarObs - size(channels) + where (VarValue /= MissingDouble) + Real2(:, iChannel) = VarValue + end where + else + where (VarValue /= MissingDouble) + Real2(:, Channels(iChannel)) = VarValue + end where + end if else where (VarValue /= MissingDouble) - Real2(:, Channels(iChannel)) = VarValue + Real2(:, iChannel) = VarValue end where - end if - else - where (VarValue /= MissingDouble) - Real2(:, iChannel) = VarValue - end where + end if ! the end end if end if end do @@ -1262,21 +1273,35 @@ subroutine opsinputs_fill_fillreal2d( & integer(integer64), optional, intent(in) :: sizeVarObs integer(c_int), optional, intent(in) :: varChannels(:) - +! local variables +logical :: compressChannels +integer(integer64) :: sizeVarObs_local ! Body: + +compressChannels = .false. +if present(compressVarChannels) then + compressChannels = compressVarChannels +end if + +sizeVarobs_local = 0 +if present(sizeVarObs) then + sizeVarObs_local = sizeVarObs +end if + if (JediToOpsLayoutMapping % ConvertRecordsToMultilevelObs) then call opsinputs_fill_fillreal2d_records( & Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, VarobsLength, JediVarName, JediVarGroup) else - if (size(varChannels) > 0) then - call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, compressVarChannels, sizeVarObs, varChannels) + if (present(varChannels)) then + if (size(varChannels) > 0) then + call opsinputs_fill_fillreal2d_norecords( & + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & + JediVarName, JediVarGroup, compressChannels, sizeVarObs_lcoal, varChannels) else call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, compressVarChannels, sizeVarObs) + JediVarName, JediVarGroup, compressChannels, sizeVarObs_local) end if end if @@ -1872,8 +1897,7 @@ subroutine opsinputs_fill_filltimeoffsets2d_norecords( & ! Local declarations: integer(c_int64_t) :: VarValue(NumObs) real(kind=c_double) :: MissingDouble -!character(len=max_varname_with_channel_length) :: JediVarNamesWithChannels(max(size(Channels), 1)) -character(len=200) :: JediVarNamesWithChannels(max(size(Channels), 1)) +character(len=max_varname_with_channel_length) :: JediVarNamesWithChannels(max(size(Channels), 1)) integer :: iChannel ! Body: diff --git a/src/opsinputs/opsinputs_utils_mod.F90 b/src/opsinputs/opsinputs_utils_mod.F90 index 20d78739..bae5440d 100644 --- a/src/opsinputs/opsinputs_utils_mod.F90 +++ b/src/opsinputs/opsinputs_utils_mod.F90 @@ -41,13 +41,7 @@ module opsinputs_utils_mod end type opsinputs_channeloffset -! Type to contain the channels offset when converting from JEDI to VAR. -type, public :: opsinputs_varchannels - ! This is the size of the varobs array for output. The default is zero and the size of the array will be used. - ! For atovs, jedi has 20 brightness temperatures but var expects 40. - ! Therefore for atovs brightness_tmperatuere => size_of_varobs_array = 40. - integer :: size_of_varobs_array -end type opsinputs_varchannels + ! ------------------------------------------------------------------------------ contains diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 849a0e34..162f2afb 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -186,12 +186,11 @@ module opsinputs_varobswriter_mod integer(integer64) :: VarobsLength integer(c_int), allocatable :: channels(:) - ! integer(c_int), allocatable :: jopaChannels(:) integer(c_int), allocatable :: varChannels(:) integer(integer64) :: size_of_varobs_array integer(integer64) :: NumVarChannels - logical ::compressVarChannels + logical :: compressVarChannels logical :: increaseChanArray !this stores the atmospheric levels we wish to pass to varobs @@ -1161,7 +1160,6 @@ subroutine opsinputs_varobswriter_populateobservations( & end select if (FillChanNum .or. FillNumChans) then - call opsinputs_varobswriter_fillchannumandnumchans( & Ob, ObsSpace, self % channels, self % varChannels, Flags, FillChanNum, & FillNumChans, self % compressVarChannels, self % size_of_varobs_array, & @@ -1222,33 +1220,51 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & integer(c_int), intent(in) :: varChannels(:) type(c_ptr), value, intent(in) :: Flags logical, intent(in) :: FillChanNum, FillNumChans -logical :: compressVarChannels +logical, optional, intent(in) :: compressVarChannels integer(integer64), optional, intent(in) :: varObsSize logical, optional, intent(in) :: increaseChanArray ! Local declarations: -integer(integer64) :: NumChannels -integer(integer64), allocatable :: ChannelIndicesVar(:,:) -integer(integer64) :: ChannelIndices(Ob % Header % NumObsLocal, size(channels)) -integer(integer64) :: ChannelCounts(Ob % Header % NumObsLocal) -integer :: iChannel -integer :: iObs -real(kind=c_double) :: MissingDouble -integer(integer64) ::array_loop - -integer :: offsetsize +integer(integer64) :: NumChannels +integer(integer64), allocatable :: ChannelIndicesVar(:,:) +integer(integer64) :: ChannelIndices(Ob % Header % NumObsLocal, size(channels)) +integer(integer64) :: ChannelCounts(Ob % Header % NumObsLocal) +integer :: iChannel +integer :: iObs +real(kind=c_double) :: MissingDouble +integer(integer64) :: array_loop +integer :: offsetsize +logical :: compressChannels +integer(integer64) :: varObsSize_loc +logical :: increaseChanArray_loc ! Body: +compressChannels = .false. +if present(compressVarChannels) then + compressChannels = compressVarChannels +end if + +varObsSize_loc = 0 +if present(varObsSize) then + varObsSize_loc = varObsSize +end if + +increaseChanArray_loc = .false. +if present(increaseChanArray) then + increaseChanArray_loc = increaseChanArray +end if + + MissingDouble = missing_value(0.0_c_double) NumChannels = size(channels) if (NumChannels == 0) return -if (increaseChanArray) then - NumChannels = varObsSize - allocate(ChannelIndicesVar(Ob % Header % NumObsLocal, varObsSize)) +if (increaseChanArray_loc) then + NumChannels = varObsSize_loc + allocate(ChannelIndicesVar(Ob % Header % NumObsLocal, varObsSize_loc)) else allocate(ChannelIndicesVar(Ob % Header % NumObsLocal, size(varChannels))) end if @@ -1261,10 +1277,10 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & if (FillChanNum) then call Ops_Alloc(Ob % Header % ChanNum, "ChanNum", Ob % Header % NumObsLocal, Ob % ChanNum, & num_levels = NumChannels) - if ((varObsSize /= 0)) then - if (varObsSize > size(channels)) then + if ((varObsSize_loc /= 0)) then + if (varObsSize_loc > size(channels)) then if (size(varChannels) > 0 .and. (size(channels) == size(varChannels))) then - if (compressVarChannels) then + if (compressChannels) then offsetsize = abs(varChannels(1) - channels(1)) Ob % ChanNum = ChannelIndices where (Ob % ChanNum > 0) @@ -1281,7 +1297,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & Ob % ChanNum = ChannelIndicesVar end if else - if (compressVarChannels) then + if (compressChannels) then Ob % ChanNum = ChannelIndices else do iChannel=1, size(Channels) @@ -1290,10 +1306,10 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & Ob % ChanNum = ChannelIndices end if end if - else if (varObsSize == size(channels)) then + else if (varObsSize_loc == size(channels)) then if (size(varChannels) > 0) then - do iChannel=1, NumChannels !varObsSize - if (compressVarChannels) then + do iChannel=1, NumChannels + if (compressChannels) then do iObs=1, Ob % Header % NumObsLocal if (ChannelIndices(iObs,iChannel) /= 0) then ChannelIndicesVar(iObs,iChannel) = varChannels(ChannelIndices(iObs,iChannel)) From e69c9e547c89a9b8a828ffe2842652e6ccfb0d1a Mon Sep 17 00:00:00 2001 From: orlewis Date: Tue, 26 Sep 2023 11:03:47 +0100 Subject: [PATCH 28/44] Remove own drietories and correct typo --- src/opsinputs/opsinputs_fill_mod.F90 | 2 +- test/testinput/010_VarField_britemp.yaml | 18 +++--------------- 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index a3e8fd85..a4d22a1a 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1055,7 +1055,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & ! Local declarations: real(kind=c_double) :: VarValue(NumObs) real(kind=c_double) :: MissingDouble -character(len=max_varname_with_channel_lengt) :: JediVarNamesWithChannels(max(size(Channels), 1)) +character(len=max_varname_with_channel_length) :: JediVarNamesWithChannels(max(size(Channels), 1)) integer :: iChannel integer :: offset integer :: numchans diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index 34aead52..2c94095a 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -132,11 +132,7 @@ observations: obsdatain: engine: type: H5File - obsfile: /data/users/olewis/JEDI_branches/test_geo_opsinputs/build-mo-bundle-jopa_env-RelWithDebInfo/opsinputs/test/Data/010_VarField_britemp.nc4 #Data/010_VarField_britemp.nc4 - obsdataout: - engine: - type: H5File - obsfile: /home/h01/olewis/map_chans.nc4 + obsfile: Data/010_VarField_britemp.nc4 simulated variables: [brightnessTemperature] channels: 1,3 obs filters: @@ -207,11 +203,7 @@ observations: obsdatain: engine: type: H5File - obsfile: /data/users/olewis/JEDI_branches/test_geo_opsinputs/build-mo-bundle-jopa_env-RelWithDebInfo/opsinputs/test/Data/010_VarField_britemp.nc4 #Data/010_VarField_britemp.nc4 - obsdataout: - engine: - type: H5File - obsfile: /home/h01/olewis/map_chans.nc4 + obsfile: Data/010_VarField_britemp.nc4 simulated variables: [brightnessTemperature] channels: 1,3 obs filters: @@ -282,11 +274,7 @@ observations: obsdatain: engine: type: H5File - obsfile: /data/users/olewis/JEDI_branches/test_geo_opsinputs/build-mo-bundle-jopa_env-RelWithDebInfo/opsinputs/test/Data/010_VarField_britemp.nc4 #Data/010_VarField_britemp.nc4 - obsdataout: - engine: - type: H5File - obsfile: /home/h01/olewis/map_chans.nc4 + obsfile: Data/010_VarField_britemp.nc4 simulated variables: [brightnessTemperature] channels: 1,3 obs filters: From 61c2116df1d68183cbeba5b23d6fa44e304c8706 Mon Sep 17 00:00:00 2001 From: orlewis Date: Tue, 26 Sep 2023 15:05:50 +0100 Subject: [PATCH 29/44] re include emissivity --- src/opsinputs/opsinputs_fill_mod.F90 | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index a4d22a1a..2e4a9dfd 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -2323,29 +2323,28 @@ function opsinputs_fill_varnames_with_channels(VarName, Channels) result(VarName character(len=max_varname_with_channel_length) :: VarNames(max(size(Channels), 1)) integer :: ichan -!character(len=max_varname_with_channel_length) :: VarNames_emis(max(size(Channels), 1)) +character(len=max_varname_with_channel_length) :: VarNames_emis(max(size(Channels), 1)) -!if (Varname=="emissivity") then -! do ichan = 1, size(Channels) -! write(*,*) "here" -! write(*,*) VarName, Channels(ichan) -! write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) - ! write(*,'(A,"_",I0)') VarName, Channels(ichan) - ! write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) - !end do - ! VarNames = VarNames_emis -!else +if (Varname=="emissivity") then + do ichan = 1, size(Channels) + write(*,*) "here" + write(*,*) VarName, Channels(ichan) + write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) + write(*,'(A,"_",I0)') VarName, Channels(ichan) + write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) + end do + VarNames = VarNames_emis +else if (size(Channels) == 0) then VarNames(1) = VarName else do ichan = 1, size(Channels) - write (VarNames(ichan),'(A,"_",I0)') VarName, Channels(ichan) end do end if -!end if +end if end function opsinputs_fill_varnames_with_channels From 4a7ff4529d5ebf922a6e0e013fc156cb5748e3eb Mon Sep 17 00:00:00 2001 From: orlewis Date: Tue, 26 Sep 2023 16:09:37 +0100 Subject: [PATCH 30/44] Missed brackets off if statement --- src/opsinputs/opsinputs_fill_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 2e4a9dfd..b316d365 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1067,7 +1067,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & MissingDouble = missing_value(0.0_c_double) compressChannels = .false. -if present(compressVarChannels) then +if (present(compressVarChannels)) then compressChannels = compressVarChannels end if From 178c036195d1638d1edaa8ec35ccb68ab7f1a6bc Mon Sep 17 00:00:00 2001 From: orlewis Date: Wed, 27 Sep 2023 17:05:26 +0100 Subject: [PATCH 31/44] Fix missing brackets for if statements, remove extra variables from the brightness temp ctest. --- src/opsinputs/opsinputs_fill_mod.F90 | 25 +- src/opsinputs/opsinputs_varobswriter_mod.F90 | 6 +- test/testinput/010_VarField_britemp.yaml | 282 +++++++++--------- .../AMSUB.nl | 2 +- 4 files changed, 160 insertions(+), 155 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index b316d365..d7a7695f 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1079,7 +1079,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & numchans = size(JediVarNamesWithChannels) -if (present(sizeVarObs) )then +if (present(sizeVarObs)) then if (sizeVarObs > 0) then numchans = sizeVarObs end if @@ -1113,8 +1113,8 @@ subroutine opsinputs_fill_fillreal2d_norecords( & end if end if else - if ((present(sizeVarObs)) then - if (sizeVarObs > size(channels))) then + if (present(sizeVarObs)) then + if (sizeVarObs > size(channels)) then if (compressChannels) then offsetsize = sizeVarObs - size(channels) where (VarValue /= MissingDouble) @@ -1276,16 +1276,22 @@ subroutine opsinputs_fill_fillreal2d( & ! local variables logical :: compressChannels integer(integer64) :: sizeVarObs_local +integer(c_int), allocatable :: localvarChannels(:) ! Body: compressChannels = .false. -if present(compressVarChannels) then +if (present(compressVarChannels)) then compressChannels = compressVarChannels end if +if (present(varChannels)) then + allocate(localvarChannels(size(varChannels))) + localvarChannels = varChannels +end if + sizeVarobs_local = 0 -if present(sizeVarObs) then +if (present(sizeVarObs)) then sizeVarObs_local = sizeVarObs end if @@ -1293,11 +1299,10 @@ subroutine opsinputs_fill_fillreal2d( & call opsinputs_fill_fillreal2d_records( & Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, VarobsLength, JediVarName, JediVarGroup) else - if (present(varChannels)) then - if (size(varChannels) > 0) then - call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, compressChannels, sizeVarObs_lcoal, varChannels) + if (allocated(localvarChannels)) then + call opsinputs_fill_fillreal2d_norecords( & + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & + JediVarName, JediVarGroup, compressChannels, sizeVarObs_local, varChannels) else call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 162f2afb..d2fca3f1 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -1242,17 +1242,17 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & ! Body: compressChannels = .false. -if present(compressVarChannels) then +if (present(compressVarChannels)) then compressChannels = compressVarChannels end if varObsSize_loc = 0 -if present(varObsSize) then +if (present(varObsSize)) then varObsSize_loc = varObsSize end if increaseChanArray_loc = .false. -if present(increaseChanArray) then +if (present(increaseChanArray)) then increaseChanArray_loc = increaseChanArray end if diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index 2c94095a..d534ee88 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -28,34 +28,34 @@ observations: expected_main_table_columns: # Only levels 1 and 3 are simulated and hence written to the VarObs file. # In the arrays below, rows denote locations and columns channels. - field: ["10", "10","54","55","55", - "10", "10","54","55","55", - "10", "10","54","55","55", - "10", "10","54","55","55"] - level: ["1", "2","1","1","2", - "1", "2","1","1","2", - "1", "2","1","1","2", - "1", "2","1","1","2"] - ob value: ["1.20000", "3.11000", "2.00000","1.00000","2.00000", - "-1073741824.00000", "3.22000","2.00000","1.00000","2.00000", - "1.60000", "3.33000","0.00000","0.00000","0.00000", - "1.80000", "3.44000","2.00000","1.00000","2.00000"] - lat: ["21.00000", "21.00000","21.00000", "21.00000","21.00000", - "22.00000", "22.00000","22.00000", "22.00000","22.00000", - "-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", - "24.00000", "24.00000","24.00000", "24.00000","24.00000"] - lon: ["31.00000", "31.00000","31.00000", "31.00000","31.00000", - "32.00000", "32.00000","32.00000", "32.00000","32.00000", - "33.00000", "33.00000","33.00000", "33.00000","33.00000", - "34.00000", "34.00000","34.00000", "34.00000","34.00000"] - time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000","-3540.00000", - "-3480.00000", "-3480.00000","-3480.00000", "-3480.00000","-3480.00000", - "-3420.00000", "-3420.00000","-3420.00000", "-3420.00000","-3420.00000", - "-3360.00000", "-3360.00000","-3360.00000", "-3360.00000","-3360.00000"] - Callsign: ["station_1", "station_1","station_1", "station_1","station_1", - "station_2", "station_2","station_2", "station_2","station_2", - "station_3", "station_3","station_3", "station_3","station_3", - "station_4", "station_4","station_4", "station_4","station_4"] + field: ["10", "10", + "10", "10", + "10", "10", + "10", "10"] + level: ["1", "2", + "1", "2", + "1", "2", + "1", "2"] + ob value: ["1.20000", "3.11000", + "-1073741824.00000", "3.22000", + "1.60000", "3.33000", + "1.80000", "3.44000"] + lat: ["21.00000", "21.00000", + "22.00000", "22.00000", + "-23.00000", "-23.00000", + "24.00000", "24.00000"] + lon: ["31.00000", "31.00000", + "32.00000", "32.00000", + "33.00000", "33.00000", + "34.00000", "34.00000"] + time: ["-3540.00000", "-3540.00000", + "-3480.00000", "-3480.00000", + "-3420.00000", "-3420.00000", + "-3360.00000", "-3360.00000"] + Callsign: ["station_1", "station_1", + "station_2", "station_2", + "station_3", "station_3", + "station_4", "station_4"] HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 @@ -93,34 +93,34 @@ observations: - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. - field: ["10", "10", "10","54","55","55", - "10", "10", "10","54","55","55", - "10", "10", "10","54","55","55", - "10", "10", "10","54","55","55"] - level: ["1", "2","3","1", "1","2", - "1", "2","3","1", "1","2", - "1", "2","3","1", "1","2", - "1", "2","3","1", "1","2"] - ob value: ["1.20000","-1073741824.00000","3.11000", "1.00000","1.00000","3.00000", - "-1073741824.00000","-1073741824.00000","3.22000", "2.00000","1.00000","3.00000", - "1.60000","-1073741824.00000","3.33000", "2.00000","1.00000","3.00000", - "1.80000","-1073741824.00000","3.44000", "1.00000","1.00000","3.00000"] - lat: ["21.00000", "21.00000","21.00000","21.00000", "21.00000","21.00000", - "22.00000", "22.00000","22.00000","22.00000", "22.00000","22.00000", - "-23.00000", "-23.00000","-23.00000","-23.00000", "-23.00000","-23.00000", - "24.00000", "24.00000","24.00000","24.00000", "24.00000","24.00000"] - lon: ["31.00000", "31.00000","31.00000","31.00000", "31.00000","31.00000", - "32.00000", "32.00000","32.00000","32.00000", "32.00000","32.00000", - "33.00000", "33.00000","33.00000","33.00000", "33.00000","33.00000", - "34.00000", "34.00000","34.00000","34.00000", "34.00000","34.00000"] - time: ["-3540.00000", "-3540.00000","-3540.00000","-3540.00000", "-3540.00000","-3540.00000", - "-3480.00000", "-3480.00000","-3480.00000","-3480.00000", "-3480.00000","-3480.00000", - "-3420.00000", "-3420.00000","-3420.00000","-3420.00000", "-3420.00000","-3420.00000", - "-3360.00000", "-3360.00000","-3360.00000","-3360.00000", "-3360.00000","-3360.00000"] - Callsign: ["station_1", "station_1","station_1","station_1", "station_1","station_1", - "station_2", "station_2","station_2","station_2", "station_2","station_2", - "station_3", "station_3","station_3","station_3", "station_3","station_3", - "station_4", "station_4","station_4","station_4", "station_4","station_4"] + field: ["10", "10", "10", + "10", "10", "10", + "10", "10", "10", + "10", "10", "10"] + level: ["1", "2","3", + "1", "2","3", + "1", "2","3", + "1", "2","3"] + ob value: ["1.20000","-1073741824.00000","3.11000", + "-1073741824.00000","-1073741824.00000","3.22000", + "1.60000","-1073741824.00000","3.33000", + "1.80000","-1073741824.00000","3.44000"] + lat: ["21.00000", "21.00000","21.00000", + "22.00000", "22.00000","22.00000", + "-23.00000", "-23.00000","-23.00000", + "24.00000", "24.00000","24.00000"] + lon: ["31.00000", "31.00000","31.00000", + "32.00000", "32.00000","32.00000", + "33.00000", "33.00000","33.00000", + "34.00000", "34.00000","34.00000"] + time: ["-3540.00000", "-3540.00000","-3540.00000", + "-3480.00000", "-3480.00000","-3480.00000", + "-3420.00000", "-3420.00000","-3420.00000", + "-3360.00000", "-3360.00000","-3360.00000"] + Callsign: ["station_1", "station_1","station_1", + "station_2", "station_2","station_2", + "station_3", "station_3","station_3", + "station_4", "station_4","station_4"] HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 @@ -162,34 +162,34 @@ observations: - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. - field: ["10", "10", "10", "54", "55", "55", - "10", "10", "10", "54", "55", "55", - "10", "10", "10", "54", "55", "55", - "10", "10", "10", "54", "55", "55"] - level: ["1","2","3","1","1","2", - "1","2","3","1","1","2", - "1","2","3","1","1","2", - "1","2","3","1","1","2"] - ob value: ["-1073741824.00000","1.20000","3.11000","1.00000","3.00000","0.00000", - "-1073741824.00000","-1073741824.00000","3.22000","2.00000","2.00000","3.00000", - "-1073741824.00000","1.60000","3.33000","2.00000","2.00000","3.00000", - "-1073741824.00000","1.80000", "3.44000","1.00000","2.00000","0.00000"] - lat: ["21.00000", "21.00000", "21.00000","21.00000","21.00000","21.00000", - "22.00000", "22.00000", "22.00000","22.00000","22.00000","22.00000", - "-23.00000", "-23.00000", "-23.00000","-23.00000", "-23.00000","-23.00000", - "24.00000", "24.00000", "24.00000","24.00000", "24.00000","24.00000"] - lon: ["31.00000", "31.00000", "31.00000","31.00000","31.00000","31.00000", - "32.00000", "32.00000", "32.00000","32.00000","32.00000","32.00000", - "33.00000", "33.00000", "33.00000","33.00000","33.00000","33.00000", - "34.00000", "34.00000", "34.00000","34.00000", "34.00000","34.00000"] - time: ["-3540.00000", "-3540.00000", "-3540.00000","-3540.00000","-3540.00000","-3540.00000", - "-3480.00000", "-3480.00000", "-3480.00000","-3480.00000","-3480.00000","-3480.00000", - "-3420.00000", "-3420.00000", "-3420.00000","-3420.00000","-3420.00000","-3420.00000", - "-3360.00000", "-3360.00000", "-3360.00000","-3360.00000", "-3360.00000","-3360.00000"] - Callsign: ["station_1", "station_1", "station_1","station_1","station_1", "station_1", - "station_2", "station_2", "station_2","station_2","station_2","station_2", - "station_3", "station_3", "station_3","station_3","station_3","station_3", - "station_4", "station_4", "station_4","station_4", "station_4","station_4"] + field: ["10", "10", "10", + "10", "10", "10", + "10", "10", "10", + "10", "10", "10"] + level: ["1","2","3", + "1","2","3", + "1","2","3", + "1","2","3"] + ob value: ["-1073741824.00000","1.20000","3.11000", + "-1073741824.00000","-1073741824.00000","3.22000", + "-1073741824.00000","1.60000","3.33000", + "-1073741824.00000","1.80000", "3.44000"] + lat: ["21.00000", "21.00000", "21.00000", + "22.00000", "22.00000", "22.00000", + "-23.00000", "-23.00000", "-23.00000", + "24.00000", "24.00000", "24.00000"] + lon: ["31.00000", "31.00000", "31.00000", + "32.00000", "32.00000", "32.00000", + "33.00000", "33.00000", "33.00000", + "34.00000", "34.00000", "34.00000"] + time: ["-3540.00000", "-3540.00000", "-3540.00000", + "-3480.00000", "-3480.00000", "-3480.00000", + "-3420.00000", "-3420.00000", "-3420.00000", + "-3360.00000", "-3360.00000", "-3360.00000"] + Callsign: ["station_1", "station_1", "station_1", + "station_2", "station_2", "station_2", + "station_3", "station_3", "station_3", + "station_4", "station_4", "station_4"] HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 @@ -233,41 +233,41 @@ observations: - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. - field: ["10", "10", "10", "10", "54", "55", "55", - "10", "10", "10", "10", "54", "55", "55", - "10", "10", "10", "10", "54", "55", "55", - "10", "10", "10", "10", "54", "55", "55"] - level: ["1","2","3","4","1","1","2", - "1","2","3","4","1","1","2", - "1","2","3","4","1","1","2", - "1","2","3","4","1","1","2"] - ob value: ["-1073741824.00000","1.20000","-1073741824.00000","3.11000","1.00000","2.00000","4.00000", - "-1073741824.00000","-1073741824.00000","-1073741824.00000","3.22000","2.00000","2.00000","4.00000", - "-1073741824.00000","1.60000","-1073741824.00000","3.33000","2.00000","2.00000","4.00000", - "-1073741824.00000","1.80000", "-1073741824.00000","3.44000","1.00000","2.00000","4.00000"] - lat: ["21.00000", "21.00000", "21.00000", "21.00000","21.00000","21.00000","21.00000", - "22.00000", "22.00000", "22.00000", "22.00000", "22.00000","22.00000","22.00000", - "-23.00000", "-23.00000", "-23.00000","-23.00000","-23.00000", "-23.00000","-23.00000", - "24.00000", "24.00000", "24.00000","24.00000", "24.00000","24.00000","24.00000"] - lon: ["31.00000","31.00000", "31.00000", "31.00000","31.00000","31.00000","31.00000", - "32.00000", "32.00000", "32.00000","32.00000","32.00000","32.00000","32.00000", - "33.00000", "33.00000","33.00000", "33.00000","33.00000","33.00000","33.00000", - "34.00000", "34.00000", "34.00000","34.00000","34.00000", "34.00000","34.00000"] - time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000","-3540.00000","-3540.00000","-3540.00000", - "-3480.00000", "-3480.00000", "-3480.00000","-3480.00000","-3480.00000","-3480.00000","-3480.00000", - "-3420.00000", "-3420.00000", "-3420.00000","-3420.00000","-3420.00000","-3420.00000","-3420.00000", - "-3360.00000", "-3360.00000", "-3360.00000","-3360.00000","-3360.00000", "-3360.00000","-3360.00000"] - Callsign: ["station_1", "station_1","station_1", "station_1","station_1","station_1", "station_1", - "station_2", "station_2", "station_2","station_2","station_2","station_2","station_2", - "station_3", "station_3", "station_3","station_3","station_3","station_3","station_3", - "station_4", "station_4", "station_4","station_4", "station_4","station_4","station_4"] + field: ["10", "10", "10", "10", + "10", "10", "10", "10", + "10", "10", "10", "10", + "10", "10", "10", "10"] + level: ["1","2","3","4", + "1","2","3","4", + "1","2","3","4", + "1","2","3","4"] + ob value: ["-1073741824.00000","1.20000","-1073741824.00000","3.11000", + "-1073741824.00000","-1073741824.00000","-1073741824.00000","3.22000", + "-1073741824.00000","1.60000","-1073741824.00000","3.33000", + "-1073741824.00000","1.80000", "-1073741824.00000","3.44000"] + lat: ["21.00000", "21.00000", "21.00000", "21.00000", + "22.00000", "22.00000", "22.00000", "22.00000", + "-23.00000", "-23.00000", "-23.00000","-23.00000", + "24.00000", "24.00000", "24.00000","24.00000"] + lon: ["31.00000","31.00000", "31.00000", "31.00000", + "32.00000", "32.00000", "32.00000","32.00000", + "33.00000", "33.00000","33.00000", "33.00000", + "34.00000", "34.00000", "34.00000","34.00000"] + time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000", + "-3480.00000", "-3480.00000", "-3480.00000","-3480.00000", + "-3420.00000", "-3420.00000", "-3420.00000","-3420.00000", + "-3360.00000", "-3360.00000", "-3360.00000","-3360.00000"] + Callsign: ["station_1", "station_1","station_1", "station_1", + "station_2", "station_2", "station_2","station_2", + "station_3", "station_3", "station_3","station_3", + "station_4", "station_4", "station_4","station_4"] HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 # -# Now we test effect of varchannels, compress var cahnnels and increase channel array size. +# Now we test effect of varchannels, compress var channels and increase channel array size. # - obs space: name: AMSUB @@ -305,34 +305,34 @@ observations: - filter: VarObs Checker expected_main_table_columns: # In the arrays below, rows denote locations and columns levels. - field: ["10", "10", "10", "10", "54", "55", "55", "55", "55", - "10", "10", "10", "10", "54", "55", "55", "55", "55", - "10", "10", "10", "10", "54", "55", "55", "55", "55", - "10", "10", "10", "10", "54", "55", "55", "55", "55"] - level: ["1","2","3","4","1","1","2","3","4", - "1","2","3","4","1","1","2","3","4", - "1","2","3","4","1","1","2","3","4", - "1","2","3","4","1","1","2","3","4"] - ob value: ["-1073741824.00000","1.20000","-1073741824.00000","3.11000","1.00000","2.00000","4.00000","-32768.00000","-32768.00000", - "-1073741824.00000","-1073741824.00000","-1073741824.00000","3.22000","2.00000","2.00000","4.00000","-32768.00000","-32768.00000", - "-1073741824.00000","1.60000","-1073741824.00000","3.33000","2.00000","2.00000","4.00000","-32768.00000","-32768.00000", - "-1073741824.00000","1.80000", "-1073741824.00000","3.44000","1.00000","2.00000","4.00000","-32768.00000","-32768.00000"] - lat: ["21.00000", "21.00000", "21.00000", "21.00000","21.00000","21.00000","21.00000","21.00000","21.00000", - "22.00000", "22.00000", "22.00000", "22.00000", "22.00000","22.00000","22.00000","22.00000","22.00000", - "-23.00000", "-23.00000", "-23.00000","-23.00000","-23.00000", "-23.00000","-23.00000","-23.00000","-23.00000", - "24.00000", "24.00000", "24.00000","24.00000", "24.00000","24.00000","24.00000","24.00000","24.00000"] - lon: ["31.00000","31.00000", "31.00000", "31.00000","31.00000","31.00000","31.00000","31.00000","31.00000", - "32.00000", "32.00000", "32.00000","32.00000","32.00000","32.00000","32.00000","32.00000","32.00000", - "33.00000", "33.00000","33.00000", "33.00000","33.00000","33.00000","33.00000","33.00000","33.00000", - "34.00000", "34.00000", "34.00000","34.00000","34.00000", "34.00000","34.00000","34.00000","34.00000"] - time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000","-3540.00000","-3540.00000","-3540.00000","-3540.00000","-3540.00000", - "-3480.00000", "-3480.00000", "-3480.00000","-3480.00000","-3480.00000","-3480.00000","-3480.00000","-3480.00000","-3480.00000", - "-3420.00000", "-3420.00000", "-3420.00000","-3420.00000","-3420.00000","-3420.00000","-3420.00000","-3420.00000","-3420.00000", - "-3360.00000", "-3360.00000", "-3360.00000","-3360.00000","-3360.00000", "-3360.00000","-3360.00000","-3360.00000","-3360.00000"] - Callsign: ["station_1", "station_1","station_1", "station_1","station_1","station_1", "station_1","station_1", "station_1", - "station_2", "station_2", "station_2","station_2","station_2","station_2","station_2","station_2", "station_2", - "station_3", "station_3", "station_3","station_3","station_3","station_3","station_3","station_3", "station_3", - "station_4", "station_4", "station_4","station_4", "station_4","station_4","station_4","station_4", "station_4"] + field: ["10", "10", "10", "10", + "10", "10", "10", "10", + "10", "10", "10", "10", + "10", "10", "10", "10", ] + level: ["1","2","3","4", + "1","2","3","4", + "1","2","3","4", + "1","2","3","4"] + ob value: ["-1073741824.00000","1.20000","-1073741824.00000","3.11000", + "-1073741824.00000","-1073741824.00000","-1073741824.00000","3.22000", + "-1073741824.00000","1.60000","-1073741824.00000","3.33000", + "-1073741824.00000","1.80000", "-1073741824.00000","3.44000"] + lat: ["21.00000", "21.00000", "21.00000", "21.00000", + "22.00000", "22.00000", "22.00000", "22.00000", + "-23.00000", "-23.00000", "-23.00000","-23.00000", + "24.00000", "24.00000", "24.00000","24.00000"] + lon: ["31.00000","31.00000", "31.00000", "31.00000", + "32.00000", "32.00000", "32.00000","32.00000", + "33.00000", "33.00000","33.00000", "33.00000", + "34.00000", "34.00000", "34.00000","34.00000"] + time: ["-3540.00000", "-3540.00000","-3540.00000", "-3540.00000", + "-3480.00000", "-3480.00000", "-3480.00000","-3480.00000", + "-3420.00000", "-3420.00000", "-3420.00000","-3420.00000", + "-3360.00000", "-3360.00000", "-3360.00000","-3360.00000"] + Callsign: ["station_1", "station_1","station_1", "station_1", + "station_2", "station_2", "station_2","station_2", + "station_3", "station_3", "station_3","station_3", + "station_4", "station_4", "station_4","station_4"] HofX: ObsValue # just a placeholder -- not used, but needed to force calls to postFilter. benchmarkFlag: 1000 # just to keep the ObsFilters test happy flaggedBenchmark: 0 diff --git a/test/testinput/VarObsWriterNamelists_010_VarField_britemp/AMSUB.nl b/test/testinput/VarObsWriterNamelists_010_VarField_britemp/AMSUB.nl index 3eb07827..15d2b7a2 100644 --- a/test/testinput/VarObsWriterNamelists_010_VarField_britemp/AMSUB.nl +++ b/test/testinput/VarObsWriterNamelists_010_VarField_britemp/AMSUB.nl @@ -1,3 +1,3 @@ &VarobsControlNL -Varfields=10,54,55 +Varfields=10 / From 5ed1762f97c6099255397d19a09e3de51242e891 Mon Sep 17 00:00:00 2001 From: mikecooke77 Date: Thu, 28 Sep 2023 11:38:18 +0100 Subject: [PATCH 32/44] MCC changes --- src/opsinputs/opsinputs_fill_mod.F90 | 25 ++++++++++++-------- src/opsinputs/opsinputs_varobswriter_mod.F90 | 6 ++--- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index b316d365..d7a7695f 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1079,7 +1079,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & numchans = size(JediVarNamesWithChannels) -if (present(sizeVarObs) )then +if (present(sizeVarObs)) then if (sizeVarObs > 0) then numchans = sizeVarObs end if @@ -1113,8 +1113,8 @@ subroutine opsinputs_fill_fillreal2d_norecords( & end if end if else - if ((present(sizeVarObs)) then - if (sizeVarObs > size(channels))) then + if (present(sizeVarObs)) then + if (sizeVarObs > size(channels)) then if (compressChannels) then offsetsize = sizeVarObs - size(channels) where (VarValue /= MissingDouble) @@ -1276,16 +1276,22 @@ subroutine opsinputs_fill_fillreal2d( & ! local variables logical :: compressChannels integer(integer64) :: sizeVarObs_local +integer(c_int), allocatable :: localvarChannels(:) ! Body: compressChannels = .false. -if present(compressVarChannels) then +if (present(compressVarChannels)) then compressChannels = compressVarChannels end if +if (present(varChannels)) then + allocate(localvarChannels(size(varChannels))) + localvarChannels = varChannels +end if + sizeVarobs_local = 0 -if present(sizeVarObs) then +if (present(sizeVarObs)) then sizeVarObs_local = sizeVarObs end if @@ -1293,11 +1299,10 @@ subroutine opsinputs_fill_fillreal2d( & call opsinputs_fill_fillreal2d_records( & Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, VarobsLength, JediVarName, JediVarGroup) else - if (present(varChannels)) then - if (size(varChannels) > 0) then - call opsinputs_fill_fillreal2d_norecords( & - Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, compressChannels, sizeVarObs_lcoal, varChannels) + if (allocated(localvarChannels)) then + call opsinputs_fill_fillreal2d_norecords( & + Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & + JediVarName, JediVarGroup, compressChannels, sizeVarObs_local, varChannels) else call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 162f2afb..d2fca3f1 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -1242,17 +1242,17 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & ! Body: compressChannels = .false. -if present(compressVarChannels) then +if (present(compressVarChannels)) then compressChannels = compressVarChannels end if varObsSize_loc = 0 -if present(varObsSize) then +if (present(varObsSize)) then varObsSize_loc = varObsSize end if increaseChanArray_loc = .false. -if present(increaseChanArray) then +if (present(increaseChanArray)) then increaseChanArray_loc = increaseChanArray end if From c6c03ecd5362f7335fad8a211bca454ab2bca393 Mon Sep 17 00:00:00 2001 From: orlewis Date: Thu, 28 Sep 2023 12:23:47 +0100 Subject: [PATCH 33/44] Switch compress_var_channels to be default true --- src/opsinputs/VarObsWriterParameters.h | 2 +- src/opsinputs/opsinputs_fill_mod.F90 | 4 ++-- src/opsinputs/opsinputs_varobswriter_mod.F90 | 2 +- test/testinput/010_VarField_britemp.yaml | 1 - test/testinput/055_VarField_channum.yaml | 1 - test/testinput/057_VarField_emissivity.yaml | 1 - test/testinput/076_VarField_britempvarerror.yaml | 1 - test/testinput/varobswriter_globalnamelist_atovs.yaml | 1 - test/testinput/varobswriter_globalnamelist_iasi.yaml | 1 - 9 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 4b6cb449..10b1567c 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -89,7 +89,7 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { oops::Parameter varChannels{"varChannels", "", this}; /// Compress the var channels - oops::Parameter compressVarChannels{"compress_var_channels", false, this}; + oops::Parameter compressVarChannels{"compress_var_channels", true, this}; /// Increase the channel array size to the same size as the varobs array oops::Parameter increaseChanArray{"increase_chan_array", false, this}; diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index d7a7695f..6146ab05 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1066,7 +1066,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & MissingDouble = missing_value(0.0_c_double) -compressChannels = .false. +compressChannels = .true. if (present(compressVarChannels)) then compressChannels = compressVarChannels end if @@ -1280,7 +1280,7 @@ subroutine opsinputs_fill_fillreal2d( & ! Body: -compressChannels = .false. +compressChannels = .true. if (present(compressVarChannels)) then compressChannels = compressVarChannels end if diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index d2fca3f1..94e65b55 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -1241,7 +1241,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & ! Body: -compressChannels = .false. +compressChannels = .true. if (present(compressVarChannels)) then compressChannels = compressVarChannels end if diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index d534ee88..baec26f5 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -156,7 +156,6 @@ observations: channels: 1,3 namelist_directory: testinput/VarObsWriterNamelists_010_VarField_britemp general_mode: debug - compress_var_channels: true size_of_varobs_array: 3 varChannels: 2, 4 #[5,7] - filter: VarObs Checker diff --git a/test/testinput/055_VarField_channum.yaml b/test/testinput/055_VarField_channum.yaml index 0d89b45d..5ed585df 100644 --- a/test/testinput/055_VarField_channum.yaml +++ b/test/testinput/055_VarField_channum.yaml @@ -91,7 +91,6 @@ observations: - filter: VarObs Writer namelist_directory: testinput/VarObsWriterNamelists_055_VarField_channum general_mode: debug - compress_var_channels: true size_of_varobs_array: 2 varChannels: 16, 17 - filter: VarObs Checker diff --git a/test/testinput/057_VarField_emissivity.yaml b/test/testinput/057_VarField_emissivity.yaml index 1a4f9177..f4338dbd 100644 --- a/test/testinput/057_VarField_emissivity.yaml +++ b/test/testinput/057_VarField_emissivity.yaml @@ -25,7 +25,6 @@ observations: reject_obs_with_all_variables_failing_qc: true namelist_directory: testinput/VarObsWriterNamelists_057_VarField_emissivity varChannels: 2-4 - compress_var_channels: true size_of_varobs_array: 4 general_mode: verbose - filter: VarObs Checker diff --git a/test/testinput/076_VarField_britempvarerror.yaml b/test/testinput/076_VarField_britempvarerror.yaml index 62bce9d3..2f1ce339 100644 --- a/test/testinput/076_VarField_britempvarerror.yaml +++ b/test/testinput/076_VarField_britempvarerror.yaml @@ -25,7 +25,6 @@ observations: reject_obs_with_all_variables_failing_qc: true namelist_directory: testinput/VarObsWriterNamelists_076_VarField_britempvarerror varChannels: 2-4 - compress_var_channels: true size_of_varobs_array: 4 general_mode: verbose - filter: VarObs Checker diff --git a/test/testinput/varobswriter_globalnamelist_atovs.yaml b/test/testinput/varobswriter_globalnamelist_atovs.yaml index c2ef2dab..402ad92b 100644 --- a/test/testinput/varobswriter_globalnamelist_atovs.yaml +++ b/test/testinput/varobswriter_globalnamelist_atovs.yaml @@ -26,7 +26,6 @@ observations: namelist_directory: ../etc/global/varobs general_mode: verbose varChannels: 2,4 - compress_var_channels: true size_of_varobs_array: 3 - filter: VarObs Checker expected_main_table_columns: diff --git a/test/testinput/varobswriter_globalnamelist_iasi.yaml b/test/testinput/varobswriter_globalnamelist_iasi.yaml index 778e295c..e17e2be6 100644 --- a/test/testinput/varobswriter_globalnamelist_iasi.yaml +++ b/test/testinput/varobswriter_globalnamelist_iasi.yaml @@ -41,7 +41,6 @@ observations: reject_obs_with_all_variables_failing_qc: true general_mode: verbose output_varbc_predictors: true - compress_var_channels: true size_of_varobs_array: 4 fill_obstype_from_ops_subtype: true - filter: VarObs Checker From f3f53ea0ba0b487e89e4fc31b6683f5f57e421ab Mon Sep 17 00:00:00 2001 From: orlewis Date: Thu, 28 Sep 2023 17:24:31 +0100 Subject: [PATCH 34/44] Fix if statement logic --- src/opsinputs/opsinputs_fill_mod.F90 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 6146ab05..2b55804c 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1125,12 +1125,12 @@ subroutine opsinputs_fill_fillreal2d_norecords( & Real2(:, Channels(iChannel)) = VarValue end where end if - else - where (VarValue /= MissingDouble) - Real2(:, iChannel) = VarValue - end where - end if ! the end - end if + end if + else + where (VarValue /= MissingDouble) + Real2(:, iChannel) = VarValue + end where + end if ! the end end if end do end if ! Data not present? OPS will produce a warning -- we don't need to duplicate it. From be8274c4f4e8fe2416498b372e4abafbade3b3bd Mon Sep 17 00:00:00 2001 From: orlewis Date: Fri, 29 Sep 2023 17:02:45 +0100 Subject: [PATCH 35/44] More changes to the if logic so that the ctests pass. --- src/opsinputs/opsinputs_fill_mod.F90 | 24 ++++++++++++++++++------ test/testinput/010_VarField_britemp.yaml | 2 +- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 6146ab05..1230f43f 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1111,6 +1111,16 @@ subroutine opsinputs_fill_fillreal2d_norecords( & end where end if end if + else + if (compressChannels) then + where (VarValue /= MissingDouble) + Real2(:, iChannel) = VarValue + end where + else + where (VarValue /= MissingDouble) + Real2(:, Channels(iChannel)) = VarValue + end where + end if end if else if (present(sizeVarObs)) then @@ -1125,12 +1135,16 @@ subroutine opsinputs_fill_fillreal2d_norecords( & Real2(:, Channels(iChannel)) = VarValue end where end if - else - where (VarValue /= MissingDouble) + else + where (VarValue /= MissingDouble) Real2(:, iChannel) = VarValue end where - end if ! the end - end if + end if + else + where (VarValue /= MissingDouble) + Real2(:, iChannel) = VarValue + end where + end if ! the end end if end do end if ! Data not present? OPS will produce a warning -- we don't need to duplicate it. @@ -2334,8 +2348,6 @@ function opsinputs_fill_varnames_with_channels(VarName, Channels) result(VarName if (Varname=="emissivity") then do ichan = 1, size(Channels) - write(*,*) "here" - write(*,*) VarName, Channels(ichan) write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) write(*,'(A,"_",I0)') VarName, Channels(ichan) write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) diff --git a/test/testinput/010_VarField_britemp.yaml b/test/testinput/010_VarField_britemp.yaml index baec26f5..205506ca 100644 --- a/test/testinput/010_VarField_britemp.yaml +++ b/test/testinput/010_VarField_britemp.yaml @@ -307,7 +307,7 @@ observations: field: ["10", "10", "10", "10", "10", "10", "10", "10", "10", "10", "10", "10", - "10", "10", "10", "10", ] + "10", "10", "10", "10"] level: ["1","2","3","4", "1","2","3","4", "1","2","3","4", From ca33a65a7cf3913c0667afdbf6b71a64d3f3d7cd Mon Sep 17 00:00:00 2001 From: orlewis Date: Fri, 29 Sep 2023 17:08:37 +0100 Subject: [PATCH 36/44] Remove merge issues --- src/opsinputs/opsinputs_fill_mod.F90 | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 1a0eb10a..8f38bd88 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1135,13 +1135,6 @@ subroutine opsinputs_fill_fillreal2d_norecords( & Real2(:, Channels(iChannel)) = VarValue end where end if -<<<<<<< HEAD - else - where (VarValue /= MissingDouble) - Real2(:, iChannel) = VarValue - end where -======= ->>>>>>> f3f53ea0ba0b487e89e4fc31b6683f5f57e421ab end if else where (VarValue /= MissingDouble) From 7dd1b1a09d69fff4242316dc7ba580dded25bd25 Mon Sep 17 00:00:00 2001 From: orlewis Date: Fri, 29 Sep 2023 17:09:40 +0100 Subject: [PATCH 37/44] Return emissivity setting to original. --- src/opsinputs/opsinputs_fill_mod.F90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 8f38bd88..b3f1808e 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -2345,8 +2345,6 @@ function opsinputs_fill_varnames_with_channels(VarName, Channels) result(VarName if (Varname=="emissivity") then do ichan = 1, size(Channels) write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) - write(*,'(A,"_",I0)') VarName, Channels(ichan) - write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) end do VarNames = VarNames_emis else From 2a88f138a3da84921c9c0ded4f23354e4a832e8b Mon Sep 17 00:00:00 2001 From: orlewis Date: Mon, 2 Oct 2023 12:12:10 +0100 Subject: [PATCH 38/44] empty commit From 0578f7373d50100b7f5be6f18f6189814c8ad1d1 Mon Sep 17 00:00:00 2001 From: orlewis Date: Wed, 4 Oct 2023 17:35:19 +0100 Subject: [PATCH 39/44] Another edit to the if statement logic. --- src/opsinputs/opsinputs_fill_mod.F90 | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index b3f1808e..22d48ee1 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1123,7 +1123,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & end if end if else - if (present(sizeVarObs)) then + if (present(sizeVarObs)) then if (sizeVarObs > size(channels)) then if (compressChannels) then offsetsize = sizeVarObs - size(channels) @@ -1135,12 +1135,12 @@ subroutine opsinputs_fill_fillreal2d_norecords( & Real2(:, Channels(iChannel)) = VarValue end where end if + else + where (VarValue /= MissingDouble) + Real2(:, iChannel) = VarValue + end where end if - else - where (VarValue /= MissingDouble) - Real2(:, iChannel) = VarValue - end where - end if ! the end + end if ! the end end if end do end if ! Data not present? OPS will produce a warning -- we don't need to duplicate it. From 4f60601331ce3848547163a150d34b2949aa8350 Mon Sep 17 00:00:00 2001 From: mikecooke77 Date: Tue, 10 Oct 2023 17:23:30 +0100 Subject: [PATCH 40/44] Improve logic --- src/opsinputs/opsinputs_fill_mod.F90 | 46 +++++++++------------------- 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index 22d48ee1..2561ea9c 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1060,6 +1060,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & integer :: offset integer :: numchans integer :: offsetsize +integer :: arrayindex logical :: compressChannels ! Body: @@ -1091,6 +1092,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & num_levels = int(numchans, kind=integer64)) do iChannel = 1, size(JediVarNamesWithChannels) call obsspace_get_db(ObsSpace, JediVarGroup, JediVarNamesWithChannels(iChannel), VarValue) + arrayindex = iChannel ! if VAR channels have been assigned then jopa channels will be mapped to these var channels ! Set up the size of the array, if channels are being pushed together an offset between @@ -1102,46 +1104,28 @@ subroutine opsinputs_fill_fillreal2d_norecords( & if (iChannel <= size(varChannels)) then if (compressChannels) then offsetsize = abs(varChannels(1) - channels(1)) - where (VarValue /= MissingDouble) - Real2(:, iChannel+offsetsize) = VarValue - end where + arrayindex = arrayindex + offsetsize else - where (VarValue /= MissingDouble) - Real2(:, varChannels(iChannel)) = VarValue - end where + arrayindex = varChannels(iChannel) end if end if else - if (compressChannels) then - where (VarValue /= MissingDouble) - Real2(:, iChannel) = VarValue - end where - else - where (VarValue /= MissingDouble) - Real2(:, Channels(iChannel)) = VarValue - end where - end if + if (.not. compressChannels) then + arrayindex = Channels(iChannel) + end if end if else - if (present(sizeVarObs)) then + if (present(sizeVarObs)) then if (sizeVarObs > size(channels)) then - if (compressChannels) then - offsetsize = sizeVarObs - size(channels) - where (VarValue /= MissingDouble) - Real2(:, iChannel) = VarValue - end where - else - where (VarValue /= MissingDouble) - Real2(:, Channels(iChannel)) = VarValue - end where + if (.not. compressChannels) then + arrayindex = Channels(iChannel) end if - else - where (VarValue /= MissingDouble) - Real2(:, iChannel) = VarValue - end where - end if - end if ! the end + end if + end if ! the end end if + where (VarValue /= MissingDouble) + Real2(:, arrayindex) = VarValue + end where end do end if ! Data not present? OPS will produce a warning -- we don't need to duplicate it. From 197a18a37a3bbf34e04b8ced6fe32354c59593c3 Mon Sep 17 00:00:00 2001 From: orlewis Date: Fri, 20 Oct 2023 12:55:28 +0100 Subject: [PATCH 41/44] Update descriptions for new options. --- src/opsinputs/VarObsWriterParameters.h | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 10b1567c..00a41e56 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -85,10 +85,15 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// Therefore for atovs brightness_tmperatuere => size_of_varobs_array = 40. oops::Parameter size_of_varobs_array{"size_of_varobs_array", 0, this}; - /// List of channels which are expected for Var. + /// List of channels which are expected for Var. This enables mapping of JOPA + /// channel numbers to VAR numbers. Default is empty and the JOPA channels will be used. oops::Parameter varChannels{"varChannels", "", this}; - /// Compress the var channels + /// If set to false (default true) this matches the channel number with the array index. + /// This is useful when there are skipped channels. e.g. channels 5,6,7,9,10,11 are + /// required from a possible 12 channels. This would fill a size 12 array with the array + /// indexes matching the channel number [NaN,NaN,NaN,Nan,5,6,7,Nan,9,10,11,NaN]. + /// Compressed example could be [2,8,50,100], no missin values between channel numbers. oops::Parameter compressVarChannels{"compress_var_channels", true, this}; /// Increase the channel array size to the same size as the varobs array From 334b6963d5100bf26982f745f9e02aedf83822a1 Mon Sep 17 00:00:00 2001 From: orlewis Date: Mon, 30 Oct 2023 16:51:23 +0000 Subject: [PATCH 42/44] Change sizeVarObs to sizeOfVarobsArray --- src/opsinputs/VarObsWriterParameters.h | 2 +- src/opsinputs/opsinputs_fill_mod.F90 | 35 +++++++++++++------------- src/opsinputs/opsinputs_utils_mod.F90 | 2 -- 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/src/opsinputs/VarObsWriterParameters.h b/src/opsinputs/VarObsWriterParameters.h index 00a41e56..0e6b9926 100644 --- a/src/opsinputs/VarObsWriterParameters.h +++ b/src/opsinputs/VarObsWriterParameters.h @@ -93,7 +93,7 @@ class VarObsWriterParameters : public oops::ObsFilterParametersBase { /// This is useful when there are skipped channels. e.g. channels 5,6,7,9,10,11 are /// required from a possible 12 channels. This would fill a size 12 array with the array /// indexes matching the channel number [NaN,NaN,NaN,Nan,5,6,7,Nan,9,10,11,NaN]. - /// Compressed example could be [2,8,50,100], no missin values between channel numbers. + /// Compressed example could be [2,8,50,100], no missing values between channel numbers. oops::Parameter compressVarChannels{"compress_var_channels", true, this}; /// Increase the channel array size to the same size as the varobs array diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index eea3f0c9..e9e34ebc 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1036,7 +1036,7 @@ end subroutine opsinputs_fill_fillreal !> are not found. subroutine opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, NumObs, Real2, ObsSpace, Channels, JediVarName, & - JediVarGroup, compressVarChannels, sizeVarObs, varChannels) + JediVarGroup, compressVarChannels, sizeOfVarobsArray, varChannels) implicit none ! Subroutine arguments: @@ -1049,7 +1049,7 @@ subroutine opsinputs_fill_fillreal2d_norecords( & character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup logical, optional, intent(in) :: compressVarChannels -integer(integer64), optional, intent(in) :: sizeVarObs +integer(integer64), optional, intent(in) :: sizeOfVarobsArray integer(c_int), optional, intent(in) :: varChannels(:) @@ -1080,10 +1080,11 @@ subroutine opsinputs_fill_fillreal2d_norecords( & !e.g. HIRS in ATOVS stream numchans = size(JediVarNamesWithChannels) - -if (present(sizeVarObs)) then - if (sizeVarObs > 0) then - numchans = sizeVarObs +!sizeOfVarobsArray comes from intitial setting of size_of_varobs_array +! used to define the size of the channel array to fill. +if (present(sizeOfVarobsArray)) then + if (sizeOfVarobsArray > 0) then + numchans = sizeOfVarobsArray end if end if @@ -1116,8 +1117,8 @@ subroutine opsinputs_fill_fillreal2d_norecords( & end if end if else - if (present(sizeVarObs)) then - if (sizeVarObs > size(channels)) then + if (present(sizeOfVarobsArray)) then + if (sizeOfVarobsArray > size(channels)) then if (.not. compressChannels) then arrayindex = Channels(iChannel) end if @@ -1251,7 +1252,7 @@ end subroutine opsinputs_fill_fillreal2d_records !> are not found. subroutine opsinputs_fill_fillreal2d( & Hdr, OpsVarName, JediToOpsLayoutMapping, Real2, ObsSpace, Channels, & - VarobsLength, JediVarName, JediVarGroup, compressVarChannels, sizeVarObs, varChannels) + VarobsLength, JediVarName, JediVarGroup, compressVarChannels, sizeOfVarobsArray, varChannels) implicit none ! Subroutine arguments: @@ -1265,12 +1266,12 @@ subroutine opsinputs_fill_fillreal2d( & character(len=*), intent(in) :: JediVarName character(len=*), intent(in) :: JediVarGroup logical, optional, intent(in) :: compressVarChannels -integer(integer64), optional, intent(in) :: sizeVarObs +integer(integer64), optional, intent(in) :: sizeOfVarobsArray integer(c_int), optional, intent(in) :: varChannels(:) ! local variables logical :: compressChannels -integer(integer64) :: sizeVarObs_local +integer(integer64) :: sizeOfVarobsArray_local integer(c_int), allocatable :: localvarChannels(:) ! Body: @@ -1285,9 +1286,9 @@ subroutine opsinputs_fill_fillreal2d( & localvarChannels = varChannels end if -sizeVarobs_local = 0 -if (present(sizeVarObs)) then - sizeVarObs_local = sizeVarObs +sizeOfVarobsArray_local = 0 +if (present(sizeOfVarobsArray)) then + sizeOfVarobsArray_local = sizeOfVarobsArray end if if (JediToOpsLayoutMapping % ConvertRecordsToMultilevelObs) then @@ -1297,11 +1298,11 @@ subroutine opsinputs_fill_fillreal2d( & if (allocated(localvarChannels)) then call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, compressChannels, sizeVarObs_local, varChannels) + JediVarName, JediVarGroup, compressChannels, sizeOfVarobsArray_local, varChannels) else call opsinputs_fill_fillreal2d_norecords( & Hdr, OpsVarName, JediToOpsLayoutMapping % NumOpsObs, Real2, ObsSpace, Channels, & - JediVarName, JediVarGroup, compressChannels, sizeVarObs_local) + JediVarName, JediVarGroup, compressChannels, sizeOfVarobsArray_local) end if end if @@ -2424,8 +2425,6 @@ function opsinputs_fill_varnames_with_channels(VarName, Channels) result(VarName character(len=max_varname_with_channel_length) :: VarNames_emis(max(size(Channels), 1)) - - if (Varname=="emissivity") then do ichan = 1, size(Channels) write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) diff --git a/src/opsinputs/opsinputs_utils_mod.F90 b/src/opsinputs/opsinputs_utils_mod.F90 index bae5440d..b05a9b31 100644 --- a/src/opsinputs/opsinputs_utils_mod.F90 +++ b/src/opsinputs/opsinputs_utils_mod.F90 @@ -40,8 +40,6 @@ module opsinputs_utils_mod integer :: size_of_varobs_array end type opsinputs_channeloffset - - ! ------------------------------------------------------------------------------ contains From c6f17ed7204a06f211ab3526aef33305a8b3222f Mon Sep 17 00:00:00 2001 From: orlewis Date: Wed, 1 Nov 2023 11:10:09 +0000 Subject: [PATCH 43/44] Remove the emissivity part from https://rom-saf.eumetsat.int/romsaf_ropp_ug_1dvar.pdf, change name of sizeVarObs and add comments. --- src/opsinputs/opsinputs_fill_mod.F90 | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/opsinputs/opsinputs_fill_mod.F90 b/src/opsinputs/opsinputs_fill_mod.F90 index e9e34ebc..2899218d 100644 --- a/src/opsinputs/opsinputs_fill_mod.F90 +++ b/src/opsinputs/opsinputs_fill_mod.F90 @@ -1030,6 +1030,12 @@ end subroutine opsinputs_fill_fillreal !> of variables with suffixes corresponding to the indices specified in \p Channels. !> \param[in] JediGroup !> Group of the JEDI variable used to populate \p Real2. +!> \param[in] compressVarChannels +!> Whether to apply var channel compression (No NaN spaces between channels) +!> \param[in] sizeOfVarobsArray +!> The size of the varobs array which the output data will be stored in. +!> \param[in] varChannels +!> A list of the var channel numbers which the channels will be mapped to. !> !> \note This function returns early (without a warning) if the specified JEDI variable is not found. !> We rely on warnings printed by the OPS code whenever data needed to output a requested varfield @@ -2423,21 +2429,12 @@ function opsinputs_fill_varnames_with_channels(VarName, Channels) result(VarName character(len=max_varname_with_channel_length) :: VarNames(max(size(Channels), 1)) integer :: ichan -character(len=max_varname_with_channel_length) :: VarNames_emis(max(size(Channels), 1)) - -if (Varname=="emissivity") then +if (size(Channels) == 0) then + VarNames(1) = VarName +else do ichan = 1, size(Channels) - write (VarNames_emis(ichan),'(A,"_",I0)') VarName, Channels(ichan) + write (VarNames(ichan),'(A,"_",I0)') VarName, Channels(ichan) end do - VarNames = VarNames_emis -else - if (size(Channels) == 0) then - VarNames(1) = VarName - else - do ichan = 1, size(Channels) - write (VarNames(ichan),'(A,"_",I0)') VarName, Channels(ichan) - end do - end if end if end function opsinputs_fill_varnames_with_channels From fbeb883896abb8396d342348a6b1715ab4c20288 Mon Sep 17 00:00:00 2001 From: orlewis Date: Thu, 30 Nov 2023 18:07:03 +0000 Subject: [PATCH 44/44] Update to use IMDI --- src/opsinputs/opsinputs_varobswriter_mod.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/opsinputs/opsinputs_varobswriter_mod.F90 b/src/opsinputs/opsinputs_varobswriter_mod.F90 index 94e65b55..ba82f315 100644 --- a/src/opsinputs/opsinputs_varobswriter_mod.F90 +++ b/src/opsinputs/opsinputs_varobswriter_mod.F90 @@ -1289,7 +1289,7 @@ subroutine opsinputs_varobswriter_fillchannumandnumchans( & else do iChannel=1, NumChannels if (iChannel > size(varChannels)) then - ChannelIndicesVar(:, iChannel) = -32768 + ChannelIndicesVar(:, iChannel) = IMDI else ChannelIndicesVar(:, iChannel) = varChannels(iChannel) end if