edit_data.m

function d = edit_data(d,p)
% function d = edit_data(d,p)
%
% perform data editing (e.g. sidelobes, previous-ping interference, &c)
%
% HISTORY:
%  Jul  3, 2004: - implemented side-lobe contamination editing
%  Jul 15, 2004: - implemented PPI contamination editing
%  Jul 16, 2004: - implemented cross-contamination editing
%  Jul 18, 2004: - added parameter defaults
%  Jul 21, 2004: - moved bin masking from [loadrdi.m]
%  Jul 23, 2004: - implemented ensemble skipping
%  May 18, 2006: - incorporated simple fix for multi-ping ensembles
%		   provided by Mattew Alford
%		 - disabled edit_spike_filter by default
%		 - changed edit_spike_filter_max_curv default value
%  Aug 13, 2013: - BUG: edit_sidelobes did not work for UL only data
%  Jul 13, 2014: - automatically edit bins 1 when blanking distance is zero
%  Jan 23, 2015: - BUG: automatic zero blanking editing had a typo
%		 - BUG: automatic zero blanking editing did not work with DL-only data
%		 - added p.edit_{dn,up}_bad_ensembles
%  Jul 21, 2015: - made bin-masking more permissive (allow indices > #bins)
%  May 11, 2023: - added edit_depths

%======================================================================
%                    E D I T _ D A T A . M 
%                    doc: Sat Jul  3 17:13:05 2004
%                    dlm: Thu May 11 13:41:47 2023
%                    (c) 2004 A.M. Thurnherr
%                    uE-Info: 200 49 NIL 0 0 72 0 2 4 NIL ofnI
%======================================================================

%----------------------------------------------------------------------
% Initialize & Set Default Parameters
%----------------------------------------------------------------------

d.ts_edited = d.ts; % save for plotting/inspection

% Set to list of bins to remove from data.
p=setdefv(p,'edit_mask_dn_bins',[]);
p=setdefv(p,'edit_mask_up_bins',[]);

% Set to 1 to remove side-lobe contaminated data near seabed and
% surface.
p=setdefv(p,'edit_sidelobes',1);

% Set to 1 to implement time-domain spike filter on the data; 
% this removes interference from other acoustic instruments but,
% more importantly, can get rid of PPI when staggered pings
% are used. USING THIS FILTER WITH A CURVATURE CRITERION THAT IS
% TOO STRICT CAN SERIOUSLY DEGRADE THE VELOCITY PROFILES. THEREFORE,
% IT HAS BEEN DISABLED BY DEFAULT IN VERSION IX_3.
p=setdefv(p,'edit_spike_filter',0);

% Spike filtering is done using 2nd-difference
% peak detection in time. This parameter gives the maximum
% target-strength 2nd derivative that's allowed. Set to larger
% values to weaken the filtering. (Check figure 14 to see if
% filter is too strong or too weak.) Setting the value of this
% parameter too low will seriously degrade the velocity profiles,
% without any apparent sign of trouble. The optimal value of this
% parameter depends on the instrument type.
p=setdefv(p,'edit_spike_filter_max_curv',5);

% Set to 1 to remove data contaminated by previous-ping interference.
% NOTES: - Using the spike filter seems to work more robustly, as long
%     	   as staggered pings are used. Great care must be taken,
%	   however, to chose a good value for edit_spike_filter_max_curv.
p=setdefv(p,'edit_PPI',0);

% PPI layer thickness in meters; the value is taken directly from Eric
% Firing's default (2*clip_margin = 180m).
p=setdefv(p,'edit_PPI_layer_thickness',180);

% max distance from seabed at which PPI should be removed. This is
% an observed parameter and depends on the clarity of the water.
% Check Figure 14 to see whether this should be changed.
p=setdefv(p,'edit_PPI_max_hab',1000);

% set this vector to enable skipping of ensembles; skipping vector
% is wrapped around, i.e. [1 0] skips all odd ensembles, [0 1 0] skips
% ensembles 2 5 8 11.... This filter is useful to process the casts
% with only half the data to see whether the two halves agree, which
% probably means that the cast can be trusted. Note that if staggered
% ping setup is used to avoid PPI, the skipping vector should leave
% adjacent ensembles intact, i.e. use something like [1 1 0 0] and
% [0 0 1 1].
p=setdefv(p,'edit_skip_ensembles',[]);

% the following vectors can be used to edit out blocks of bad ensembles,
% caused, for example, by intermittent hardware failures. NOTE: it is
% assumed that the ensembles are numbered consecutively, starting with 1,
% i.e. the ensemble numbers in the data files are ignored. This may 
% not work as intended if the data files are trimmed during the DL/UL
% merge in [loardrdi.m]

p=setdefv(p,'edit_dn_bad_ensembles',[]);
p=setdefv(p,'edit_up_bad_ensembles',[]);

% Set to [min max] to implement bad depth-range filter
p=setdefv(p,'edit_depths',[]);


%----------------------------------------------------------------------
% Bin Masking
%----------------------------------------------------------------------

if length(d.zu)>0 && p.blnk_u==0
  disp(sprintf(' bin masking               : masking uplooker bin 1 because of zero blanking distance'));
  p.edit_mask_up_bins = [1 p.edit_mask_up_bins];
end
if p.blnk_d==0
  disp(sprintf(' bin masking               : masking downlooker bin 1 because of zero blanking distance'));
  p.edit_mask_dn_bins = [1 p.edit_mask_dn_bins];
end

if ~isempty(p.edit_mask_dn_bins) | ~isempty(p.edit_mask_up_bins)

  nbad = 0;
  if length(d.zu) > 0
    for bi=1:length(p.edit_mask_up_bins)
      if p.edit_mask_up_bins(bi)<=p.nbin_u
         bn = length(d.zu)+1 - p.edit_mask_up_bins(bi);
	 nbad = nbad + length(find(isfinite(d.weight(bn,:))));
	 d.weight(bn,:) = NaN; d.ts_edited(bn,:) = NaN;
      end
    end
  end
  if length(d.zd) > 0
    for bi=1:length(p.edit_mask_dn_bins)
      if p.edit_mask_dn_bins(bi)<=p.nbin_d
        bn = length(d.zu) + p.edit_mask_dn_bins(bi);
	nbad = nbad + length(find(isfinite(d.weight(bn,:))));
	d.weight(bn,:) = NaN; d.ts_edited(bn,:) = NaN;
      end
    end
  end

  disp(sprintf(' bin masking               : set %d weights to NaN',nbad));

end % if bin masking enabled

%----------------------------------------------------------------------
% Side-Lobe Contamination
%----------------------------------------------------------------------

if p.edit_sidelobes

  nbad = 0;
  
  % first, the uplooker: d.z is -ve distance of ADCP from surface;
  % Cell_length is in cm, i.e. 0.015*Cell_length is 1.5 x bin size
  % in m --- the same value used by Eric Firing's software
  
  if length(d.zu)==0 && d.zd(1)<0			% UL only (in DL structures)

    for b=1:length(d.zd)
      zlim(b,:) = (1 - cos(pi*d.down.Beam_angle/180)) * d.z ...
		- 0.015*d.down.Cell_length;
    end
    ibad = find(d.izm > zlim);
    nbad = nbad + length(find(isfinite(d.weight(ibad))));
    d.weight(ibad) = NaN; d.ts_edited(ibad) = NaN;
  
  elseif length(d.zu > 0)				% DL/UL combo
  
    for b=1:length(d.zu)+length(d.zd)
      zlim(b,:) = (1 - cos(pi*d.up.Beam_angle/180)) * d.z ...
		- 0.015*d.up.Cell_length;
    end
    ibad = find(d.izm > zlim);
    nbad = nbad + length(find(isfinite(d.weight(ibad))));
    d.weight(ibad) = NaN; d.ts_edited(ibad) = NaN;
  
  end
  
  % now, the downlooker: p.zbottom is the +ve depth of the sea bed; therefore,
  % -d.z - p.zbottom is the -ve distance from the sea bed
  
  for b=1:length(d.zu)+length(d.zd)
    zlim(b,:) = -p.zbottom ...
	      + (1 - cos(pi*d.down.Beam_angle/180)) * (d.z+p.zbottom) ...
	      + 0.015*d.down.Cell_length;
  end
  ibad = find(d.izm < zlim);
  nbad = nbad + length(find(isfinite(d.weight(ibad))));
  d.weight(ibad) = NaN; d.ts_edited(ibad) = NaN;
  
  disp(sprintf(' side-lobe contamination   : set %d weights to NaN',nbad));

end %if p.edit_sidelobes

%----------------------------------------------------------------------
% Bad Depth Ranges
%----------------------------------------------------------------------

if length(p.edit_depths)>0
  nbad = 0;
  
  ibad = find(d.izm<=-p.edit_depths(1) & d.izm>=-p.edit_depths(2));
  nbad = nbad + length(find(isfinite(d.weight(ibad))));
  d.weight(ibad) = NaN; d.ts_edited(ibad) = NaN;

  disp(sprintf(' bad depth range           : set %d weights to NaN',nbad));

end %if p.edit_depths

%----------------------------------------------------------------------
% Time-Domain Spike Filter
%----------------------------------------------------------------------

if p.edit_spike_filter

  nbad = 0;
  for b=1:length(d.zd)+length(d.zu)
    ibad = find(diff(diff(d.ts(b,:))) < -1*p.edit_spike_filter_max_curv) + 1;
    nbad = nbad + length(find(isfinite(d.weight(b,ibad))));
    d.weight(b,ibad) = NaN; d.ts_edited(b,ibad) = NaN;
  end
  disp(sprintf(' spike filter              : set %d weights to NaN',nbad));

end %if p.edit_spike_filter

%----------------------------------------------------------------------
% Previous-Ping Interference
%----------------------------------------------------------------------

if p.edit_PPI

  % NB: at present, PPI filtering is only implemented for the downlooker
  
  nbad = 0;
  
  % calc ping-intervals; dt(1) contains the difference (in seconds)
  % between the first two ensembles (t(2) - t(1)).
  
  dt = diff(d.time_jul)*86400;

  % adjust for multi-ping ensembles; THIS FIX, PROVIDED BY MATTHEW ALFORD,
  % DOES NOT WORK FOR IRREGULAR PINGING RATES (e.g. 10s ensembles of 3
  % pings each with 1s between pings)

  dt = dt/d.down.Pings_per_Ensemble;
  
  % use the mean sound speed below the approximate expected PPI depth;
  % this is anal but not very expensive; using 1500m/s would be nearly
  % as good.
  
  if isfield(d,'ctdprof_z') & isfield(d,'ctdprof_ss')
    guess_z = -p.zbottom + 1500 * meannan(dt) / 2;
    SS = meannan(d.ctdprof_ss(find(d.ctdprof_z > -guess_z)));
    if ~isfinite(SS), SS = 1500; end
  else
    SS = 1500;
  end
  
  % calculate the depth limits to remove the PPI for all (but the first)
  % ensembles; the beam-angle limits were found to be too conservative
  % and were replaced by a nominal layer_tickness.
  
  %PPI_min_beam_angle = 0.0 * d.down.Beam_angle;
  %PPI_max_beam_angle = 1.2 * d.down.Beam_angle;
  %PPI_max_z = -p.zbottom + SS * dt/2 * cos(pi/180 * PPI_min_beam_angle);
  %PPI_min_z = -p.zbottom + SS * dt/2 * cos(pi/180 * PPI_max_beam_angle);
  
  PPI_hab = SS * dt/2 * cos(pi/180 * d.down.Beam_angle);
  PPI_hab(find(PPI_hab > p.edit_PPI_max_hab)) = inf;
  
  PPI_min_z = -p.zbottom + PPI_hab - p.edit_PPI_layer_thickness / 2;
  PPI_max_z = PPI_min_z + p.edit_PPI_layer_thickness;
  
  % remove the contaminated data from the downlooker bins
  
  for b=length(d.zu)+1:length(d.zu)+length(d.zd)
    ibad = find(d.izm(b,2:end) > PPI_min_z & d.izm(b,2:end) < PPI_max_z) + 1;
    nbad = nbad + length(find(isfinite(d.weight(b,ibad))));
    d.weight(b,ibad) = NaN; d.ts_edited(b,ibad) = NaN;
  end
  
  disp(sprintf(' previous-ping interference: set %d weights to NaN',nbad));

end %if p.edit_PPI

%----------------------------------------------------------------------
% Ensemble Skipping
%----------------------------------------------------------------------

if ~isempty(p.edit_skip_ensembles)

  nskipped = 0;
  iskip = [];
  for i=1:length(p.edit_skip_ensembles)
    if p.edit_skip_ensembles(i)
      iskip = [iskip i:length(p.edit_skip_ensembles):length(d.time_jul)];
    end
  end

  for b=1:length(d.zd)+length(d.zu)
    nskipped = nskipped + length(find(isfinite(d.weight(b,iskip))));
    d.weight(b,iskip) = NaN; d.ts_edited(b,iskip) = NaN;
  end

  disp(sprintf(' ensemble skipping         : set %d weights to NaN',nskipped));

end % if p.edit_skip_ensembles enabled

%----------------------------------------------------------------------
% remove blocks of bad ensembles from UL and/or DL data
%----------------------------------------------------------------------

if ~isempty(p.edit_dn_bad_ensembles) || ~isempty(p.edit_up_bad_ensembles)
  if ~isempty(p.edit_dn_bad_ensembles)
    dn_bad = 0; ibad = p.edit_dn_bad_ensembles;
    for b=length(d.zu)+1:length(d.zu)+length(d.zd)
      dn_bad = dn_bad + length(find(isfinite(d.weight(b,ibad))));
      d.weight(b,ibad) = NaN; d.ts_edited(b,ibad) = NaN;
    end
  end
  if ~isempty(p.edit_up_bad_ensembles)
    up_bad = 0; ibad = p.edit_up_bad_ensembles;
    for b=length(d.zu):-1:1
      up_bad = up_bad + length(find(isfinite(d.weight(b,ibad))));
      d.weight(b,ibad) = NaN; d.ts_edited(b,ibad) = NaN;
    end
  end
  disp(sprintf(' DL/UL ensemble editing    : set %d/%d weights to NaN',dn_bad,up_bad));
end

%----------------------------------------------------------------------
% Plot Results of Editing
%----------------------------------------------------------------------

bin_no = [0];
if length(d.zu) > 0, bin_no = [-length(d.zu):1 bin_no]; end
if length(d.zd) > 0, bin_no = [bin_no 1:length(d.zd)]; end

figure(14);
clf;
orient landscape;
colormap([[1 1 1]; jet(128)]);

subplot(2,1,1);
imagesc(1:size(d.ts,2),bin_no,...
	[d.ts(1:length(d.zu),:); ...
	 ones(1,size(d.ts,2))*NaN; ...
	 d.ts(size(d.ts,1)-length(d.zd)+1:end,:)...
        ]);
csc = caxis;
xlabel('Ensemble #');
ylabel('Bin #');
title('Before Data Editing');

subplot(2,1,2);
imagesc(1:size(d.ts,2),bin_no,...
	[d.ts_edited(1:length(d.zu),:); ...
	 ones(1,size(d.ts,2))*NaN; ...
	 d.ts_edited(size(d.ts,1)-length(d.zd)+1:end,:)...
        ]);
csc = caxis;
xlabel('Ensemble #');
ylabel('Bin #');
title('After Data Editing');

streamer([p.name,'  Figure 14']);