From 86d34a21eedc600fd25ef809c98531396115f64f Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 16 Aug 2024 21:37:23 +0100 Subject: [PATCH 01/54] update for spatial flexure --- pypeit/images/rawimage.py | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 0e8fb26758..feb82b2de5 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -111,8 +111,12 @@ class RawImage: datasec_img (`numpy.ndarray`_): Image identifying which amplifier was used to read each section of the *processed* image. - spat_flexure_shift (:obj:`float`): - The spatial flexure shift in pixels, if calculated + spat_flexure_shift (`numpy.ndarray`_): + The spatial flexure shift in pixels, if calculated. This is a 2D array + of shape (nslits, 2) where spat_flexure_shift[i,0] is the shift in the + spatial direction for the left edge of slit i and spat_flexure_shift[i,1] + is the shift in the spatial direction for the right edge of slit i. + """ def __init__(self, ifile, spectrograph, det): @@ -237,7 +241,7 @@ def use_slits(self): """ if self.par is None: return False - return self.par['spat_flexure_correct'] or (self.use_flat and self.par['use_illumflat']) + return (self.par['spat_flexure_correct'] != "none") or (self.use_flat and self.par['use_illumflat']) def apply_gain(self, force=False): """ @@ -542,7 +546,7 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl msgs.error('No dark available for dark subtraction!') if self.par['subtract_scattlight'] and scattlight is None: msgs.error('Scattered light subtraction requested, but scattered light model not provided.') - if self.par['spat_flexure_correct'] and slits is None: + if (self.par['spat_flexure_correct'] != "none") and slits is None: msgs.error('Spatial flexure correction requested but no slits provided.') if self.use_flat and flatimages is None: msgs.error('Flat-field corrections requested but no flat-field images generated ' @@ -667,8 +671,9 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl # bias and dark subtraction) and before field flattening. Also the # function checks that the slits exist if running the spatial flexure # correction, so no need to do it again here. - self.spat_flexure_shift = self.spatial_flexure_shift(slits, maxlag = self.par['spat_flexure_maxlag']) \ - if self.par['spat_flexure_correct'] else None + self.spat_flexure_shift = None if self.par['spat_flexure_correct'] == "none" else \ + self.spat_flexure_shift = self.spatial_flexure_shift(slits, method=self.par['spat_flexure_correct'], + maxlag=self.par['spat_flexure_maxlag']) # - Subtract scattered light... this needs to be done before flatfielding. if self.par['subtract_scattlight']: @@ -761,7 +766,7 @@ def _squeeze(self): return _det, self.image, self.ivar, self.datasec_img, self.det_img, self.rn2img, \ self.base_var, self.img_scale, self.bpm - def spatial_flexure_shift(self, slits, force=False, maxlag = 20): + def spatial_flexure_shift(self, slits, force=False, method="detector", maxlag=20): """ Calculate a spatial shift in the edge traces due to flexure. @@ -774,11 +779,18 @@ def spatial_flexure_shift(self, slits, force=False, maxlag = 20): force (:obj:`bool`, optional): Force the image to be field flattened, even if the step log (:attr:`steps`) indicates that it already has been. + method (:obj:`str`, optional): + Method to use to calculate the spatial flexure shift. Options + are 'detector' (default), 'slit', and 'edge'. The 'detector' + method calculates the shift for all slits simultaneously, the + 'slit' method calculates the shift for each slit independently, + and the 'edge' method calculates the shift for each slit edge + independently. maxlag (:obj:'float', optional): Maximum range of lag values over which to compute the CCF. Return: - float: The calculated flexure correction + `numpy.ndarray`_: The calculated flexure correction for the edge of each slit shape is (nslits, 2) """ step = inspect.stack()[0][3] @@ -789,7 +801,7 @@ def spatial_flexure_shift(self, slits, force=False, maxlag = 20): if self.nimg > 1: msgs.error('CODING ERROR: Must use a single image (single detector or detector ' 'mosaic) to determine spatial flexure.') - self.spat_flexure_shift = flexure.spat_flexure_shift(self.image[0], slits, maxlag = maxlag) + self.spat_flexure_shift = flexure.spat_flexure_shift(self.image[0], slits, method=method, maxlag=maxlag) self.steps[step] = True # Return return self.spat_flexure_shift From eb7e76ebb03a2f1922c6ed738977c0eb85d2ef28 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 16 Aug 2024 21:37:29 +0100 Subject: [PATCH 02/54] update for spatial flexure --- pypeit/par/pypeitpar.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index aeff12e5c5..9535b87131 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -354,9 +354,15 @@ def __init__(self, trim=None, apply_gain=None, orient=None, '``slit_illum_relative=True`` in the ``flatfield`` parameter set!' # Flexure - defaults['spat_flexure_correct'] = False - dtypes['spat_flexure_correct'] = bool - descr['spat_flexure_correct'] = 'Correct slits, illumination flat, etc. for flexure' + defaults['spat_flexure_correct'] = 'none' + options['spat_flexure_correct'] = ProcessImagesPar.valid_spatial_flexure() + dtypes['spat_flexure_correct'] = str + descr['spat_flexure_correct'] = 'Correct slits, illumination flat, etc. for spatial flexure. ' \ + 'Options are: {0}'.format(', '.join(options['spat_flexure_correct'])) + \ + '"none" means no correction is performed. ' \ + '"detector" means that a single shift is applied to all slits. ' \ + '"slit" means that each slit is shifted independently.' \ + '"edge" means that each slit edge is shifted independently.' defaults['spat_flexure_maxlag'] = 20 dtypes['spat_flexure_maxlag'] = int @@ -487,6 +493,13 @@ def valid_combine_methods(): """ return ['median', 'mean' ] + @staticmethod + def valid_spatial_flexure(): + """ + Return the valid methods for combining frames. + """ + return ['none', 'detector', 'slit', 'edge'] + @staticmethod def valid_saturation_handling(): """ From de4d97aead5d88ce77463116e77fb3aa794d6125 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 16 Aug 2024 21:42:29 +0100 Subject: [PATCH 03/54] argument rename: flexure -> spat_flexure --- deprecated/find_objects.py | 2 +- deprecated/reduce.py | 8 ++++---- pypeit/calibrations.py | 2 +- pypeit/coadd2d.py | 5 +++-- pypeit/coadd3d.py | 6 +++--- pypeit/core/flexure.py | 17 +++++++++++++++-- pypeit/core/gui/skysub_regions.py | 4 ++-- pypeit/core/skysub.py | 2 +- pypeit/extraction.py | 4 ++-- pypeit/find_objects.py | 8 ++++---- pypeit/flatfield.py | 10 +++++----- pypeit/images/rawimage.py | 8 ++++---- pypeit/pypeit.py | 2 +- pypeit/scattlight.py | 2 +- pypeit/scripts/show_2dspec.py | 2 +- pypeit/slittrace.py | 30 +++++++++++++++--------------- pypeit/spec2dobj.py | 4 ++-- pypeit/wavecalib.py | 10 +++++----- pypeit/wavetilts.py | 8 ++++---- 19 files changed, 74 insertions(+), 60 deletions(-) diff --git a/deprecated/find_objects.py b/deprecated/find_objects.py index 46263e217a..819ccac820 100644 --- a/deprecated/find_objects.py +++ b/deprecated/find_objects.py @@ -191,7 +191,7 @@ def illum_profile_spatial(self, skymask=None, trim_edg=(0, 0), debug=False): skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) hist_trim = 0 # Trim the edges of the histogram to take into account edge effects gpm = self.sciImg.select_flag(invert=True) - slitid_img_init = self.slits.slit_img(pad=0, initial=True, flexure=self.spat_flexure_shift) + slitid_img_init = self.slits.slit_img(pad=0, initial=True, spat_flexure=self.spat_flexure_shift) spatScaleImg = np.ones_like(self.sciImg.image) # For each slit, grab the spatial coordinates and a spline # representation of the spatial profile from the illumflat diff --git a/deprecated/reduce.py b/deprecated/reduce.py index 0205b6cd7f..eac1185b57 100644 --- a/deprecated/reduce.py +++ b/deprecated/reduce.py @@ -222,10 +222,10 @@ def initialise_slits(self, initial=False): self.slits = self.caliBrate.slits # Select the edges to use self.slits_left, self.slits_right, _ \ - = self.slits.select_edges(initial=initial, flexure=self.spat_flexure_shift) + = self.slits.select_edges(initial=initial, spat_flexure=self.spat_flexure_shift) # Slitmask - self.slitmask = self.slits.slit_img(initial=initial, flexure=self.spat_flexure_shift, + self.slitmask = self.slits.slit_img(initial=initial, spat_flexure=self.spat_flexure_shift, exclude_flag=self.slits.bitmask.exclude_for_reducing+['BOXSLIT']) # Now add the slitmask to the mask (i.e. post CR rejection in proc) # NOTE: this uses the par defined by EdgeTraceSet; this will @@ -375,7 +375,7 @@ def prepare_extraction(self, global_sky): (np.invert(self.slits.bitmask.flagged(self.slits.mask, flag=self.slits.bitmask.exclude_for_reducing))) # Update Slitmask to remove `BOXSLIT`, i.e., we don't want to extract those - self.slitmask = self.slits.slit_img(flexure=self.spat_flexure_shift, + self.slitmask = self.slits.slit_img(spat_flexure=self.spat_flexure_shift, exclude_flag=self.slits.bitmask.exclude_for_reducing) # use the tweaked traces if they exist - DP: I'm not sure this is necessary self.sciImg.update_mask_slitmask(self.slitmask) @@ -1605,7 +1605,7 @@ def illum_profile_spatial(self, skymask=None, trim_edg=(0, 0)): skymask_now = skymask if (skymask is not None) else np.ones_like(self.sciImg.image, dtype=bool) hist_trim = 0 # Trim the edges of the histogram to take into account edge effects gpm = self.sciImg.select_flag(invert=True) - slitid_img_init = self.slits.slit_img(pad=0, initial=True, flexure=self.spat_flexure_shift) + slitid_img_init = self.slits.slit_img(pad=0, initial=True, spat_flexure=self.spat_flexure_shift) spatScaleImg = np.ones_like(self.sciImg.image) # For each slit, grab the spatial coordinates and a spline # representation of the spatial profile from the illumflat diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index 13f60553c6..164e40b767 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -666,7 +666,7 @@ def get_scattlight(self): spatbin = parse.parse_binning(binning)[1] pad = self.par['scattlight_pad'] // spatbin - offslitmask = self.slits.slit_img(pad=pad, flexure=None) == -1 + offslitmask = self.slits.slit_img(pad=pad, spat_flexure=None) == -1 # Get starting parameters for the scattered light model x0, bounds = self.spectrograph.scattered_light_archive(binning, dispname) diff --git a/pypeit/coadd2d.py b/pypeit/coadd2d.py index 65d92b3542..9f7ceaf5b6 100644 --- a/pypeit/coadd2d.py +++ b/pypeit/coadd2d.py @@ -1065,7 +1065,7 @@ def load_coadd2d_stacks(self, spec2d, chk_version=False): skymodel_stack.append(s2dobj.skymodel) sciivar_stack.append(s2dobj.ivarmodel) mask_stack.append(s2dobj.bpmmask.mask) - slitmask_stack.append(s2dobj.slits.slit_img(flexure=s2dobj.sci_spat_flexure)) + slitmask_stack.append(s2dobj.slits.slit_img(spat_flexure=s2dobj.sci_spat_flexure)) # check if exptime is consistent for all images exptime_coadd = np.percentile(exptime_stack, 50., method='higher') @@ -1589,7 +1589,8 @@ def get_maskdef_dict(self, slit_idx, ref_trace_stack): # get maskdef_objpos # find left edge slits_left, _, _ = \ - self.stack_dict['slits_list'][iexp].select_edges(flexure=self.stack_dict['spat_flexure_list'][iexp]) + self.stack_dict['slits_list'][iexp].select_edges( + spat_flexure=self.stack_dict['spat_flexure_list'][iexp]) # targeted object spat pix maskdef_obj_pixpos = \ self.stack_dict['slits_list'][iexp].maskdef_objpos[slit_idx] + self.maskdef_offset[iexp] \ diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 225747d76d..a0bf54834b 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -950,7 +950,7 @@ def get_alignments(self, spec2DObj, slits, spat_flexure=None): msgs.info("Using slit edges for astrometric transform") # If nothing better was provided, use the slit edges if alignments is None: - left, right, _ = slits.select_edges(flexure=spat_flexure) + left, right, _ = slits.select_edges(spat_flexure=spat_flexure) locations = [0.0, 1.0] traces = np.append(left[:, None, :], right[:, None, :], axis=1) else: @@ -1015,8 +1015,8 @@ def load(self): # Initialise the slit edges msgs.info("Constructing slit image") slits = spec2DObj.slits - slitid_img = slits.slit_img(pad=0, flexure=spat_flexure) - slits_left, slits_right, _ = slits.select_edges(flexure=spat_flexure) + slitid_img = slits.slit_img(pad=0, spat_flexure=spat_flexure) + slits_left, slits_right, _ = slits.select_edges(spat_flexure=spat_flexure) # The order of operations below proceeds as follows: # (1) Get science image diff --git a/pypeit/core/flexure.py b/pypeit/core/flexure.py index 02265a3426..1d7d3e649d 100644 --- a/pypeit/core/flexure.py +++ b/pypeit/core/flexure.py @@ -42,7 +42,7 @@ from IPython import embed -def spat_flexure_shift(sciimg, slits, debug=False, maxlag = 20): +def spat_flexure_shift(sciimg, slits, method="detector", maxlag=20, debug=False): """ Calculate a rigid flexure shift in the spatial dimension between the slitmask and the science image. @@ -54,14 +54,27 @@ def spat_flexure_shift(sciimg, slits, debug=False, maxlag = 20): Args: sciimg (`numpy.ndarray`_): + Science image slits (:class:`pypeit.slittrace.SlitTraceSet`): + Slits object + method (:obj:`str`, optional): + Method to use to calculate the spatial flexure shift. Options + are 'detector' (default), 'slit', and 'edge'. The 'detector' + method calculates the shift for all slits simultaneously, the + 'slit' method calculates the shift for each slit independently, + and the 'edge' method calculates the shift for each slit edge + independently. maxlag (:obj:`int`, optional): Maximum flexure searched for + debug (:obj:`bool`, optional): + Run in debug mode Returns: float: The spatial flexure shift relative to the initial slits """ + # TODO :: Need to implement different methods + # Mask -- Includes short slits and those excluded by the user (e.g. ['rdx']['slitspatnum']) slitmask = slits.slit_img(initial=True, exclude_flag=slits.bitmask.exclude_for_flexure) @@ -109,7 +122,7 @@ def spat_flexure_shift(sciimg, slits, debug=False, maxlag = 20): if debug: # Now translate the slits in the tslits_dict - all_left_flexure, all_right_flexure, mask = slits.select_edges(flexure=lag_max[0]) + all_left_flexure, all_right_flexure, mask = slits.select_edges(spat_flexure=lag_max[0]) gpm = mask == 0 viewer, ch = display.show_image(_sciimg) #display.show_slits(viewer, ch, left_flexure[:,gpm], right_flexure)[:,gpm]#, slits.id) #, args.det) diff --git a/pypeit/core/gui/skysub_regions.py b/pypeit/core/gui/skysub_regions.py index b8e268197c..b678784e2b 100644 --- a/pypeit/core/gui/skysub_regions.py +++ b/pypeit/core/gui/skysub_regions.py @@ -137,7 +137,7 @@ def __init__(self, canvas, image, frame, outname, det, slits, axes, pypeline, sp self._fitr = [] # Matplotlib shaded fit region self._fita = None - self.slits_left, self.slits_right, _ = slits.select_edges(initial=initial, flexure=flexure) + self.slits_left, self.slits_right, _ = slits.select_edges(initial=initial, spat_flexure=flexure) self.initialize_menu() self.reset_regions() @@ -178,7 +178,7 @@ def initialize(cls, det, frame, slits, pypeline, spectrograph, outname="skyregio # NOTE: SlitTraceSet objects always store the left and right # traces as 2D arrays, even if there's only one slit. nslit = slits.nslits - lordloc, rordloc, _ = slits.select_edges(initial=initial, flexure=flexure) + lordloc, rordloc, _ = slits.select_edges(initial=initial, spat_flexure=flexure) # Determine the scale of the image med = np.median(frame) diff --git a/pypeit/core/skysub.py b/pypeit/core/skysub.py index cdd97944d6..b2ec8c5511 100644 --- a/pypeit/core/skysub.py +++ b/pypeit/core/skysub.py @@ -1642,7 +1642,7 @@ def generate_mask(pypeline, skyreg, slits, slits_left, slits_right, spat_flexure specmax=spec_max, binspec=slits.binspec, binspat=slits.binspat, pad=0) # Generate the mask, and return - return (slitreg.slit_img(use_spatial=False, flexure=spat_flexure) >= 0).astype(bool) + return (slitreg.slit_img(use_spatial=False, spat_flexure=spat_flexure) >= 0).astype(bool) diff --git a/pypeit/extraction.py b/pypeit/extraction.py index 99f36f6432..a3b9f67818 100644 --- a/pypeit/extraction.py +++ b/pypeit/extraction.py @@ -297,7 +297,7 @@ def initialize_slits(self, slits, initial=False): # TODO JFH: his is an ugly hack for the present moment until we get the slits object sorted out self.slits_left, self.slits_right, _ \ - = self.slits.select_edges(initial=initial, flexure=self.spat_flexure_shift) + = self.slits.select_edges(initial=initial, spat_flexure=self.spat_flexure_shift) # This matches the logic below that is being applied to the slitmask. Better would be to clean up slits to # to return a new slits object with the desired selection criteria which would remove the ambiguity # about whether the slits and the slitmask are in sync. @@ -308,7 +308,7 @@ def initialize_slits(self, slits, initial=False): #self.slits_right = slits_right[:, gpm] # Slitmask - self.slitmask = self.slits.slit_img(initial=initial, flexure=self.spat_flexure_shift, + self.slitmask = self.slits.slit_img(initial=initial, spat_flexure=self.spat_flexure_shift, exclude_flag=self.slits.bitmask.exclude_for_reducing) # Now add the slitmask to the mask (i.e. post CR rejection in proc) # NOTE: this uses the par defined by EdgeTraceSet; this will diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index a2d11e18f6..0a9ebcc65e 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -297,7 +297,7 @@ def initialize_slits(self, slits, initial=False): # Select the edges to use # TODO JFH: his is an ugly hack for the present moment until we get the slits object sorted out self.slits_left, self.slits_right, _ \ - = self.slits.select_edges(initial=initial, flexure=self.spat_flexure_shift) + = self.slits.select_edges(initial=initial, spat_flexure=self.spat_flexure_shift) # This matches the logic below that is being applied to the slitmask. Better would be to clean up slits to # to return a new slits object with the desired selection criteria which would remove the ambiguity # about whether the slits and the slitmask are in sync. @@ -309,7 +309,7 @@ def initialize_slits(self, slits, initial=False): # Slitmask - self.slitmask = self.slits.slit_img(initial=initial, flexure=self.spat_flexure_shift, + self.slitmask = self.slits.slit_img(initial=initial, spat_flexure=self.spat_flexure_shift, exclude_flag=self.slits.bitmask.exclude_for_reducing+['BOXSLIT']) # Now add the slitmask to the mask (i.e. post CR rejection in proc) # NOTE: this uses the par defined by EdgeTraceSet; this will @@ -1069,8 +1069,8 @@ def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0), model_ivar = self.sciImg.ivar sl_ref = self.par['calibrations']['flatfield']['slit_illum_ref_idx'] # Prepare the slitmasks for the relative spectral illumination - slitmask = self.slits.slit_img(pad=0, flexure=self.spat_flexure_shift) - slitmask_trim = self.slits.slit_img(pad=-3, flexure=self.spat_flexure_shift) + slitmask = self.slits.slit_img(pad=0, spat_flexure=self.spat_flexure_shift) + slitmask_trim = self.slits.slit_img(pad=-3, spat_flexure=self.spat_flexure_shift) for nn in range(numiter): msgs.info("Performing iterative joint sky subtraction - ITERATION {0:d}/{1:d}".format(nn+1, numiter)) # TODO trim_edg is in the parset so it should be passed in here via trim_edg=tuple(self.par['reduce']['trim_edge']), diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index 646f659c1c..de82ed6fd5 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -374,7 +374,7 @@ def fit2illumflat(self, slits, frametype='illum', finecorr=False, initial=False, continue # Skip those without a bspline # DO it - _slitid_img = slits.slit_img(slitidx=slit_idx, initial=initial, flexure=spat_flexure) + _slitid_img = slits.slit_img(slitidx=slit_idx, initial=initial, spat_flexure=spat_flexure) onslit = _slitid_img == slits.spat_id[slit_idx] spat_coo = slits.spatial_coordinate_image(slitidx=slit_idx, initial=initial, @@ -672,7 +672,7 @@ def build_waveimg(self): """ msgs.info("Generating wavelength image") flex = self.wavetilts.spat_flexure - slitmask = self.slits.slit_img(initial=True, flexure=flex) + slitmask = self.slits.slit_img(initial=True, spat_flexure=flex) tilts = self.wavetilts.fit2tiltimg(slitmask, flexure=flex) # Save to class attribute for inclusion in the Flat calibration frame self.waveimg = self.wv_calib.build_waveimg(tilts, self.slits, spat_flexure=flex) @@ -1421,7 +1421,7 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s onslit_tweak_trim = self.slits.slit_img(pad=-slit_trim, slitidx=slit_idx, initial=False) == slit_spat # Setup slitimg = (slit_spat + 1) * onslit_tweak.astype(int) - 1 # Need to +1 and -1 so that slitimg=-1 when off the slit - left, right, msk = self.slits.select_edges(flexure=self.wavetilts.spat_flexure) + left, right, msk = self.slits.select_edges(spat_flexure=self.wavetilts.spat_flexure) this_left = left[:, slit_idx] this_right = right[:, slit_idx] slitlen = int(np.median(this_right - this_left)) @@ -1929,8 +1929,8 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool) modelimg = model if (model is not None) else rawimg.copy() # Setup the slits - slitid_img = slits.slit_img(pad=0, flexure=flexure) - slitid_img_trim = slits.slit_img(pad=-trim, flexure=flexure) + slitid_img = slits.slit_img(pad=0, spat_flexure=flexure) + slitid_img_trim = slits.slit_img(pad=-trim, spat_flexure=flexure) scaleImg = np.ones_like(rawimg) modelimg_copy = modelimg.copy() # Obtain the minimum and maximum wavelength of all slits diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index feb82b2de5..4141f6adea 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -1231,7 +1231,7 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): f" {tmp[13]}, {tmp[14]}, {tmp[15]}]) # Polynomial terms (coefficients of spec**index)\n" print(strprint) pad = msscattlight.pad // spatbin - offslitmask = slits.slit_img(pad=pad, flexure=None) == -1 + offslitmask = slits.slit_img(pad=pad, spat_flexure=None) == -1 from matplotlib import pyplot as plt _frame = self.image[ii, ...] vmin, vmax = 0, np.max(scatt_img) @@ -1256,7 +1256,7 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): elif self.par["scattlight"]["method"] == "frame": # Calculate a model specific for this frame pad = msscattlight.pad // spatbin - offslitmask = slits.slit_img(pad=pad, flexure=None) == -1 + offslitmask = slits.slit_img(pad=pad, spat_flexure=None) == -1 # Get starting parameters for the scattered light model x0, bounds = self.spectrograph.scattered_light_archive(binning, dispname) # Perform a fit to the scattered light @@ -1280,11 +1280,11 @@ def subtract_scattlight(self, msscattlight, slits, debug=False): # Check if a fine correction to the scattered light should be applied if do_finecorr: pad = self.par['scattlight']['finecorr_pad'] // spatbin - offslitmask = slits.slit_img(pad=pad, flexure=None) == -1 + offslitmask = slits.slit_img(pad=pad, spat_flexure=None) == -1 # Check if the user wishes to mask some inter-slit regions if self.par['scattlight']['finecorr_mask'] is not None: # Get the central trace of each slit - left, right, _ = slits.select_edges(flexure=None) + left, right, _ = slits.select_edges(spat_flexure=None) centrace = 0.5*(left+right) # Now mask user-defined inter-slit regions offslitmask = scattlight.mask_slit_regions(offslitmask, centrace, diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index 6505a0ef9d..5c5e5238ed 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -927,7 +927,7 @@ def load_skyregions(self, initial_slits=False, scifile=None, frame=None, spat_fl # NOTE : Do not include spatial flexure here! # It is included when generating the mask in the return statement below slits_left, slits_right, _ \ - = self.caliBrate.slits.select_edges(initial=initial_slits, flexure=None) + = self.caliBrate.slits.select_edges(initial=initial_slits, spat_flexure=None) maxslitlength = np.max(slits_right-slits_left) # Get the regions diff --git a/pypeit/scattlight.py b/pypeit/scattlight.py index 675da0d07b..e4bb76909b 100644 --- a/pypeit/scattlight.py +++ b/pypeit/scattlight.py @@ -122,7 +122,7 @@ def show(self, image=None, slits=None, mask=False, wcs_match=True): wcs_match : :obj:`bool`, optional Use a reference image for the WCS and match all image in other channels to it. """ - offslitmask = slits.slit_img(pad=0, flexure=None) == -1 if mask else 1 + offslitmask = slits.slit_img(pad=0, spat_flexure=None) == -1 if mask else 1 # Prepare the frames _data = self.scattlight_raw if image is None else image diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index 07d9b6ae32..5db4498dfc 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -258,7 +258,7 @@ def main(args): if spec2DObj.sci_spat_flexure is not None: msgs.info(f'Offseting slits by {spec2DObj.sci_spat_flexure}') slit_left, slit_right, slit_mask \ - = spec2DObj.slits.select_edges(flexure=spec2DObj.sci_spat_flexure) + = spec2DObj.slits.select_edges(spat_flexure=spec2DObj.sci_spat_flexure) slit_spat_id = spec2DObj.slits.spat_id slit_mask_id = spec2DObj.slits.maskdef_id slit_slid_IDs = spec2DObj.slits.slitord_id diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 9405b51ad2..1fa0fae067 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -542,7 +542,7 @@ def get_radec_image(self, wcs, alignSplines, tilts, slit_compute=None, slice_off decimg = np.zeros((self.nspec, self.nspat)) minmax = np.zeros((self.nslits, 2)) # Get the slit information - slitid_img_init = self.slit_img(pad=0, initial=initial, flexure=flexure) + slitid_img_init = self.slit_img(pad=0, initial=initial, spat_flexure=flexure) for slit_idx, spatid in enumerate(self.spat_id): if slit_idx not in slit_compute: continue @@ -563,7 +563,7 @@ def get_radec_image(self, wcs, alignSplines, tilts, slit_compute=None, slice_off decimg[onslit] = world_dec.copy() return raimg, decimg, minmax - def select_edges(self, initial=False, flexure=None): + def select_edges(self, initial=False, spat_flexure=None): """ Select between the initial or tweaked slit edges and allow for flexure correction. @@ -578,7 +578,7 @@ def select_edges(self, initial=False, flexure=None): initial (:obj:`bool`, optional): To use the initial edges regardless of the presence of the tweaked edges, set this to True. - flexure (:obj:`float`, optional): + spat_flexure (:obj:`float`, optional): If provided, offset each slit by this amount Returns: @@ -593,15 +593,15 @@ def select_edges(self, initial=False, flexure=None): left, right = self.left_init, self.right_init # Add in spatial flexure? - if flexure: - self.left_flexure = left + flexure - self.right_flexure = right + flexure + if spat_flexure is not None: + self.left_flexure = left + spat_flexure[:,0] + self.right_flexure = right + spat_flexure[:,1] left, right = self.left_flexure, self.right_flexure # Return return left.copy(), right.copy(), self.mask.copy() - def slit_img(self, pad=None, slitidx=None, initial=False, flexure=None, exclude_flag=None, + def slit_img(self, pad=None, slitidx=None, initial=False, spat_flexure=None, exclude_flag=None, use_spatial=True): r""" Construct an image identifying each pixel with its associated @@ -651,7 +651,7 @@ def slit_img(self, pad=None, slitidx=None, initial=False, flexure=None, exclude_ Warning -- This could conflict with input slitids, i.e. avoid using both use_spatial (bool, optional): If True, use self.spat_id value instead of 0-based indices - flexure (:obj:`float`, optional): + spat_flexure (:obj:`float`, optional): If provided, offset each slit by this amount Done in select_edges() @@ -673,7 +673,7 @@ def slit_img(self, pad=None, slitidx=None, initial=False, flexure=None, exclude_ spat = np.arange(self.nspat) spec = np.arange(self.nspec) - left, right, _ = self.select_edges(initial=initial, flexure=flexure) + left, right, _ = self.select_edges(initial=initial, spat_flexure=spat_flexure) # Choose the slits to use if slitidx is not None: @@ -759,7 +759,7 @@ def spatial_coordinate_image(self, slitidx=None, full=False, slitid_img=None, msgs.error('Provided slit ID image does not have the correct shape!') # Choose the slit edges to use - left, right, _ = self.select_edges(initial=initial, flexure=flexure_shift) + left, right, _ = self.select_edges(initial=initial, spat_flexure=flexure_shift) # Slit width slitwidth = right - left @@ -806,7 +806,7 @@ def spatial_coordinates(self, initial=False, flexure=None): spatial coordinates. """ # TODO -- Confirm it makes sense to pass in flexure - left, right, _ = self.select_edges(initial=initial, flexure=flexure) + left, right, _ = self.select_edges(initial=initial, spat_flexure=flexure) return SlitTraceSet.slit_spat_pos(left, right, self.nspat) @staticmethod @@ -880,9 +880,9 @@ def mask_add_missing_obj(self, sobjs, spat_flexure, fwhm, boxcar_rad): cut_sobjs = sobjs # get slits edges init; includes flexure - left_init, _, _ = self.select_edges(initial=True, flexure=spat_flexure) + left_init, _, _ = self.select_edges(initial=True, spat_flexure=spat_flexure) # get slits edges tweaked; includes flexure - left_tweak, right_tweak, _ = self.select_edges(initial=False, flexure=spat_flexure) + left_tweak, right_tweak, _ = self.select_edges(initial=False, spat_flexure=spat_flexure) # midpoint in the spectral direction specmid = left_init[:,0].size//2 @@ -1029,7 +1029,7 @@ def assign_maskinfo(self, sobjs, plate_scale, spat_flexure, TOLER=1.): 'Matching tolerance includes user-provided tolerance, slit tracing uncertainties and object size.') # get slits edges init - left_init, right_init, _ = self.select_edges(initial=True, flexure=spat_flexure) # includes flexure + left_init, right_init, _ = self.select_edges(initial=True, spat_flexure=spat_flexure) # includes flexure # midpoint in the spectral direction specmid = left_init[:, 0].size // 2 @@ -1290,7 +1290,7 @@ def get_maskdef_offset(self, sobjs, platescale, spat_flexure, slitmask_off, brig align_maskdef_ids = obj_maskdef_id[flag_align == 1] # get slits edges init - left_init, _, _ = self.select_edges(initial=True, flexure=spat_flexure) # includes flexure + left_init, _, _ = self.select_edges(initial=True, spat_flexure=spat_flexure) # includes flexure # midpoint in the spectral direction specmid = left_init[:, 0].size // 2 diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index 6aaae67396..55c3c93740 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -331,7 +331,7 @@ def update_slits(self, spec2DObj): self.slits.mask[gpm] = spec2DObj.slits.mask[gpm] # Slitmask - slitmask = spec2DObj.slits.slit_img(flexure=spec2DObj.sci_spat_flexure, + slitmask = spec2DObj.slits.slit_img(spat_flexure=spec2DObj.sci_spat_flexure, exclude_flag=spec2DObj.slits.bitmask.exclude_for_reducing) # Fill in the image for slit_idx, spat_id in enumerate(spec2DObj.slits.spat_id[gpm]): @@ -661,7 +661,7 @@ def write_to_fits(self, outfile, pri_hdr=None, update_det=None, msgs.error("Original spec2D object has a different version. Too risky to continue. Rerun both") # Generate the slit "mask" slitmask = _allspecobj[det].slits.slit_img( - flexure=_allspecobj[det].sci_spat_flexure) + spat_flexure=_allspecobj[det].sci_spat_flexure) # Save the new one in a copy new_Spec2DObj = deepcopy(self[det]) # Replace with the old diff --git a/pypeit/wavecalib.py b/pypeit/wavecalib.py index 4e576ee53f..9f00120579 100644 --- a/pypeit/wavecalib.py +++ b/pypeit/wavecalib.py @@ -280,8 +280,8 @@ def build_fwhmimg(self, tilts, slits, initial=False, spat_flexure=None): if (spat_flexure is not None) and (not isinstance(spat_flexure, float)): msgs.error("Spatial flexure must be None or float") # Generate the slit mask and slit edges - pad slitmask by 1 for edge effects - slitmask = slits.slit_img(pad=1, initial=initial, flexure=spat_flexure) - slits_left, slits_right, _ = slits.select_edges(initial=initial, flexure=spat_flexure) + slitmask = slits.slit_img(pad=1, initial=initial, spat_flexure=spat_flexure) + slits_left, slits_right, _ = slits.select_edges(initial=initial, spat_flexure=spat_flexure) # Build a map of the spectral FWHM fwhmimg = np.zeros(tilts.shape) for sl, spat_id in enumerate(slits.spat_id): @@ -335,7 +335,7 @@ def build_waveimg(self, tilts, slits, spat_flexure=None, spec_flexure=None): ok_slits = np.logical_not(bpm) # image = np.zeros_like(tilts) - slitmask = slits.slit_img(flexure=spat_flexure, exclude_flag=slits.bitmask.exclude_for_reducing) + slitmask = slits.slit_img(spat_flexure=spat_flexure, exclude_flag=slits.bitmask.exclude_for_reducing) # Separate detectors for the 2D solutions? if self.par['ech_separate_2d']: @@ -555,7 +555,7 @@ def __init__(self, msarc, slits, spectrograph, par, lamps, # Load up slits # TODO -- Allow for flexure - slits_left, slits_right, mask = self.slits.select_edges(initial=True, flexure=None) # Grabs all, init slits + flexure + slits_left, slits_right, mask = self.slits.select_edges(initial=True, spat_flexure=None) # Grabs all, init slits + flexure self.orders = self.slits.ech_order # Can be None # self.spat_coo = self.slits.spatial_coordinates() # All slits, even masked # Internal mask for failed wv_calib analysis @@ -567,7 +567,7 @@ def __init__(self, msarc, slits, spectrograph, par, lamps, self.wvc_bpm_init = self.wvc_bpm.copy() # Slitmask -- Grabs only unmasked, initial slits #self.slitmask_science = self.slits.slit_img(initial=True, flexure=None, exclude_flag=['BOXSLIT']) - self.slitmask_science = self.slits.slit_img(initial=True, flexure=None) + self.slitmask_science = self.slits.slit_img(initial=True, spat_flexure=None) # Resize self.shape_science = self.slitmask_science.shape self.shape_arc = self.msarc.image.shape diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index cf9977744a..4fe808b86d 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -206,8 +206,8 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False, cal_file = Path(self.calib_dir).absolute() / self.slits_filename if cal_file.exists(): slits = slittrace.SlitTraceSet.from_file(cal_file, chk_version=chk_version) - _slitmask = slits.slit_img(initial=True, flexure=self.spat_flexure) - _left, _right, _mask = slits.select_edges(flexure=self.spat_flexure) + _slitmask = slits.slit_img(initial=True, spat_flexure=self.spat_flexure) + _left, _right, _mask = slits.select_edges(spat_flexure=self.spat_flexure) gpm = _mask == 0 # resize slitmask = arc.resize_mask2arc(tilt_img_dict.image.shape, _slitmask) @@ -332,7 +332,7 @@ def __init__(self, mstilt, slits, spectrograph, par, wavepar, det=1, qa_path=Non # TODO -- Tidy this up into one or two methods? # Load up all slits # TODO -- Discuss further with JFH - all_left, all_right, mask = self.slits.select_edges(initial=True, flexure=self.spat_flexure) # Grabs all, initial slits + all_left, all_right, mask = self.slits.select_edges(initial=True, spat_flexure=self.spat_flexure) # Grabs all, initial slits # self.tilt_bpm = np.invert(mask == 0) # At this point of the reduction the only bitmask flags that may have been generated are 'USERIGNORE', # 'SHORTSLIT', 'BOXSLIT' and 'BADWVCALIB'. Here we use only 'USERIGNORE' and 'SHORTSLIT' to create the bpm mask @@ -340,7 +340,7 @@ def __init__(self, mstilt, slits, spectrograph, par, wavepar, det=1, qa_path=Non self.tilt_bpm_init = self.tilt_bpm.copy() # Slitmask # TODO -- Discuss further with JFH - self.slitmask_science = self.slits.slit_img(initial=True, flexure=self.spat_flexure, exclude_flag=['BOXSLIT']) # All unmasked slits + self.slitmask_science = self.slits.slit_img(initial=True, spat_flexure=self.spat_flexure, exclude_flag=['BOXSLIT']) # All unmasked slits # Resize # TODO: Should this be the bpm or *any* flag? gpm = self.mstilt.select_flag(flag='BPM', invert=True) if self.mstilt is not None \ From b2bcc5fddd234974c02da426fc7c0edebb7a1d2d Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 16 Aug 2024 21:44:18 +0100 Subject: [PATCH 04/54] argument rename: flexure -> spat_flexure --- pypeit/images/rawimage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 4141f6adea..5a945d3ca6 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -867,7 +867,7 @@ def flatfield(self, flatimages, slits=None, force=False, debug=False): illum_flat = flatimages.fit2illumflat(slits, spat_flexure=self.spat_flexure_shift, finecorr=False) illum_flat *= flatimages.fit2illumflat(slits, spat_flexure=self.spat_flexure_shift, finecorr=True) if debug: - left, right = slits.select_edges(flexure=self.spat_flexure_shift) + left, right = slits.select_edges(spat_flexure=self.spat_flexure_shift) viewer, ch = display.show_image(illum_flat, chname='illum_flat') display.show_slits(viewer, ch, left, right) # , slits.id) # From 9790a334eb1cfed6322d4d2043da1d37a711454b Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 16 Aug 2024 21:50:33 +0100 Subject: [PATCH 05/54] update spatial flexure --- pypeit/images/pypeitimage.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pypeit/images/pypeitimage.py b/pypeit/images/pypeitimage.py index dc8e07b591..6a62901c3c 100644 --- a/pypeit/images/pypeitimage.py +++ b/pypeit/images/pypeitimage.py @@ -128,9 +128,12 @@ class PypeItImage(datamodel.DataContainer): 'exptime': dict(otype=(int, float), descr='Effective exposure time (s)'), 'noise_floor': dict(otype=float, descr='Noise floor included in variance'), 'shot_noise': dict(otype=bool, descr='Shot-noise included in variance'), - 'spat_flexure': dict(otype=float, + 'spat_flexure': dict(otype=np.ndarray, atype=np.floating, descr='Shift, in spatial pixels, between this image ' - 'and SlitTrace'), + 'and SlitTrace. Shape is (nslits, 2), where' + 'spat_flexure[i,0] is the spatial shift of the left ' + 'edge of slit i and spat_flexure[i,1] is the spatial ' + 'shift of the right edge of slit i.'), 'filename': dict(otype=str, descr='Filename for the image'),} """Data model components.""" @@ -782,9 +785,10 @@ def sub(self, other): # Spatial flexure spat_flexure = self.spat_flexure if other.spat_flexure is not None and spat_flexure is not None \ - and other.spat_flexure != spat_flexure: - msgs.warn(f'Spatial flexure different for images being subtracted ({spat_flexure} ' - f'vs. {other.spat_flexure}). Adopting {np.max(np.abs([spat_flexure, other.spat_flexure]))}.') + and np.array_equal(other.spat_flexure, spat_flexure): + msgs.warn(f'Spatial flexure different for images being subtracted. Adopting ' \ + f'the maximum spatial flexure of each individual edge.') + spat_flexure = np.maximum(spat_flexure, other.spat_flexure) # Create a copy of the detector, if it is defined, to be used when # creating the new pypeit image below From 3e83eccf7ea3f5f0a4980be212a04ae4d242d7b6 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 16 Aug 2024 22:00:42 +0100 Subject: [PATCH 06/54] update spatial flexure arguments --- pypeit/flatfield.py | 4 +- pypeit/slittrace.py | 140 ++++++++++++++++++++++++++++++++------------ 2 files changed, 104 insertions(+), 40 deletions(-) diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index de82ed6fd5..4972df7c6d 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -379,7 +379,7 @@ def fit2illumflat(self, slits, frametype='illum', finecorr=False, initial=False, spat_coo = slits.spatial_coordinate_image(slitidx=slit_idx, initial=initial, slitid_img=_slitid_img, - flexure_shift=spat_flexure) + spat_flexure=spat_flexure) if finecorr: spec_coo = np.where(onslit)[0] / (slits.nspec - 1) illumflat[onslit] = spat_bsplines[slit_idx].eval(spat_coo[onslit], spec_coo) @@ -1431,7 +1431,7 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s this_wave = self.waveimg[this_slit] xpos_img = self.slits.spatial_coordinate_image(slitidx=slit_idx, slitid_img=slitimg, - flexure_shift=self.wavetilts.spat_flexure) + spat_flexure=self.wavetilts.spat_flexure) # Generate the trimmed versions for fitting this_slit_trim = np.where(onslit_tweak_trim & self.rawflatimg.select_flag(invert=True)) this_wave_trim = self.waveimg[this_slit_trim] diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 1fa0fae067..47e7d256e2 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -578,8 +578,13 @@ def select_edges(self, initial=False, spat_flexure=None): initial (:obj:`bool`, optional): To use the initial edges regardless of the presence of the tweaked edges, set this to True. - spat_flexure (:obj:`float`, optional): - If provided, offset each slit by this amount + spat_flexure (`numpy.ndarray`_, optional): + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. Returns: tuple: Returns the full arrays containing the left and right @@ -651,9 +656,13 @@ def slit_img(self, pad=None, slitidx=None, initial=False, spat_flexure=None, exc Warning -- This could conflict with input slitids, i.e. avoid using both use_spatial (bool, optional): If True, use self.spat_id value instead of 0-based indices - spat_flexure (:obj:`float`, optional): - If provided, offset each slit by this amount - Done in select_edges() + spat_flexure (`numpy.ndarray`_, optional): + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. Returns: `numpy.ndarray`_: The image with the slit index @@ -701,7 +710,7 @@ def slit_img(self, pad=None, slitidx=None, initial=False, spat_flexure=None, exc return slitid_img def spatial_coordinate_image(self, slitidx=None, full=False, slitid_img=None, - pad=None, initial=False, flexure_shift=None): + pad=None, initial=False, spat_flexure=None): r""" Generate an image with the normalized spatial coordinate within each slit. @@ -738,6 +747,13 @@ def spatial_coordinate_image(self, slitidx=None, full=False, slitid_img=None, :attr:`right`) are used. To use the nominal edges regardless of the presence of the tweaked edges, set this to True. See :func:`select_edges`. + spat_flexure (`numpy.ndarray`_, optional): + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. Returns: `numpy.ndarray`_: Array specifying the spatial coordinate @@ -759,7 +775,7 @@ def spatial_coordinate_image(self, slitidx=None, full=False, slitid_img=None, msgs.error('Provided slit ID image does not have the correct shape!') # Choose the slit edges to use - left, right, _ = self.select_edges(initial=initial, spat_flexure=flexure_shift) + left, right, _ = self.select_edges(initial=initial, spat_flexure=spat_flexure) # Slit width slitwidth = right - left @@ -785,7 +801,7 @@ def spatial_coordinate_image(self, slitidx=None, full=False, slitid_img=None, coo_img = coo return coo_img - def spatial_coordinates(self, initial=False, flexure=None): + def spatial_coordinates(self, initial=False, spat_flexure=None): """ Return a fiducial coordinate for each slit. @@ -793,20 +809,27 @@ def spatial_coordinates(self, initial=False, flexure=None): :func:`slit_spat_pos`. Args: - original (:obj:`bool`, optional): + initial (:obj:`bool`, optional): By default, the method will use the tweaked slit edges if they have been defined. If they haven't - been, the nominal edges (:attr:`left` and - :attr:`right`) are used. To use the nominal edges + been, the initial edges (:attr:`left` and + :attr:`right`) are used. To use the initial edges regardless of the presence of the tweaked edges, set this to True. See :func:`select_edges`. + spat_flexure (`numpy.ndarray`_, optional): + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. Returns: `numpy.ndarray`_: Vector with the list of floating point spatial coordinates. """ # TODO -- Confirm it makes sense to pass in flexure - left, right, _ = self.select_edges(initial=initial, spat_flexure=flexure) + left, right, _ = self.select_edges(initial=initial, spat_flexure=spat_flexure) return SlitTraceSet.slit_spat_pos(left, right, self.nspat) @staticmethod @@ -847,8 +870,13 @@ def mask_add_missing_obj(self, sobjs, spat_flexure, fwhm, boxcar_rad): Args: sobjs (:class:`~pypeit.specobjs.SpecObjs`): List of SpecObj that have been found and traced - spat_flexure (:obj:`float`): - Shifts, in spatial pixels, between this image and SlitTrace + spat_flexure (`numpy.ndarray`_): + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. fwhm (:obj:`float`): FWHM in pixels to be used in the optimal extraction boxcar_rad (:obj:`float`): @@ -985,11 +1013,21 @@ def assign_maskinfo(self, sobjs, plate_scale, spat_flexure, TOLER=1.): Modified in place. Args: - sobjs (:class:`pypeit.specobjs.SpecObjs`): List of SpecObj that have been found and traced. - plate_scale (:obj:`float`): platescale for the current detector. - spat_flexure (:obj:`float`): Shifts, in spatial pixels, between this image and SlitTrace. - det_buffer (:obj:`int`): Minimum separation between detector edges and a slit edge. - TOLER (:obj:`float`, optional): Matching tolerance in arcsec. + sobjs (:class:`pypeit.specobjs.SpecObjs`): + List of SpecObj that have been found and traced. + plate_scale (:obj:`float`): + platescale for the current detector. + spat_flexure (`numpy.ndarray`_): + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. + det_buffer (:obj:`int`): + Minimum separation between detector edges and a slit edge. + TOLER (:obj:`float`, optional): + Matching tolerance in arcsec. Returns: :class:`pypeit.specobjs.SpecObjs`: Updated list of SpecObj that have been found and traced. @@ -1238,17 +1276,28 @@ def get_maskdef_offset(self, sobjs, platescale, spat_flexure, slitmask_off, brig Determine the Slitmask offset (pixels) from position expected by the slitmask design Args: - sobjs (:class:`pypeit.specobjs.SpecObjs`): List of SpecObj that have been found and traced - platescale (:obj:`float`): Platescale - spat_flexure (:obj:`float`): Shifts, in spatial pixels, between this image and SlitTrace - slitmask_off (:obj:`float`): User provided slitmask offset in pixels - bright_maskdefid (:obj:`str`): User provided maskdef_id of a bright object to be used to measure offset - snr_thrshd (:obj:`float`): Objects detected above this S/N ratio threshold will be use to - compute the slitmask offset - use_alignbox (:obj:`bool`): Flag that determines if the alignment boxes are used to measure the offset - dither_off (:obj:`float`, optional): dither offset recorded in the header of the observations - - + sobjs (:class:`pypeit.specobjs.SpecObjs`): + List of SpecObj that have been found and traced + platescale (:obj:`float`): + Platescale of this detector + spat_flexure (`numpy.ndarray`_): + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. + slitmask_off (:obj:`float`): + User provided slitmask offset in pixels + bright_maskdefid (:obj:`str`): + User provided maskdef_id of a bright object to be used to measure offset + snr_thrshd (:obj:`float`): + Objects detected above this S/N ratio threshold will be use to + compute the slitmask offset + use_alignbox (:obj:`bool`): + Flag that determines if the alignment boxes are used to measure the offset + dither_off (:obj:`float`, optional): + dither offset recorded in the header of the observations """ if self.maskdef_objpos is None: msgs.error('An array of object positions predicted by the slitmask design must be provided.') @@ -1556,13 +1605,25 @@ def get_maskdef_objpos_offset_alldets(sobjs, calib_slits, spat_flexure, platesca This info is recorded in the `SlitTraceSet` datamodel. Args: - sobjs (:class:`pypeit.specobjs.SpecObjs`): List of SpecObj that have been found and traced - calib_slits (:obj:`list`): List of `SlitTraceSet` with information on the traced slit edges - spat_flexure (:obj:`list`): List of shifts, in spatial pixels, between this image and SlitTrace - platescale (:obj:`list`): List of platescale for every detector - det_buffer (:obj:`int`): Minimum separation between detector edges and a slit edge - slitmask_par (:class:`pypeit.par.pypeitpar.PypeItPar`): slitmask PypeIt parameters - dither_off (:obj:`float`, optional): dither offset recorded in the header of the observations + sobjs (:class:`pypeit.specobjs.SpecObjs`): + List of SpecObj that have been found and traced + calib_slits (:obj:`list`): + List of `SlitTraceSet` with information on the traced slit edges + spat_flexure (`numpy.ndarray`_, optional): + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. + platescale (:obj:`list`): + List of platescale for every detector + det_buffer (:obj:`int`): + Minimum separation between detector edges and a slit edge + slitmask_par (:class:`pypeit.par.pypeitpar.PypeItPar`): + slitmask PypeIt parameters + dither_off (:obj:`float`, optional): + dither offset recorded in the header of the observations Returns: List of `SlitTraceSet` with updated information on the traced slit edges @@ -1691,7 +1752,10 @@ def assign_addobjs_alldets(sobjs, calib_slits, spat_flexure, platescale, slitmas calib_slits (`numpy.ndarray`_): Array of `SlitTraceSet` with information on the traced slit edges. spat_flexure (:obj:`list`): - List of shifts, in spatial pixels, between this image and SlitTrace. + List of spatial flexure shifts, in spatial pixels, between this image and SlitTrace. + Each element of this list should be a `numpy.ndarray`_ of shape (nslits, 2), + where the first column is the shift to apply to the left edge of each slit + and the second column is the shift to apply to the right edge of each slit. platescale (:obj:`list`): List of platescale for every detector. slitmask_par (:class:`~pypeit.par.pypeitpar.PypeItPar`): From da607cd8b2aa00485534902e4f87e604360d5c43 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 16 Aug 2024 22:05:16 +0100 Subject: [PATCH 07/54] update spatial flexure tests --- pypeit/tests/test_spec2dobj.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pypeit/tests/test_spec2dobj.py b/pypeit/tests/test_spec2dobj.py index 2a639153fe..ae30d78b49 100644 --- a/pypeit/tests/test_spec2dobj.py +++ b/pypeit/tests/test_spec2dobj.py @@ -25,8 +25,9 @@ def init_dict(): sciimg = np.ones((1000,1000)).astype(float) # Slits - left = np.full((1000, 3), 2, dtype=float) - right = np.full((1000, 3), 8, dtype=float) + nslits = 3 + left = np.full((1000, nslits), 2, dtype=float) + right = np.full((1000, nslits), 8, dtype=float) left[:,1] = 15. right[:,1] = 21. left[:,2] = 25. @@ -37,6 +38,7 @@ def init_dict(): spec_flex_table = Table() spec_flex_table['spat_id'] = slits.spat_id spec_flex_table['sci_spec_flexure'] = np.zeros(left.shape[1]) + spat_flexure = np.full((nslits, 2), 3.5) # return dict(sciimg = sciimg, ivarraw = 0.1 * np.ones_like(sciimg), @@ -52,7 +54,7 @@ def init_dict(): maskdef_designtab=None, tilts=np.ones_like(sciimg).astype(float), #tilts=wavetilts.WaveTilts(**test_wavetilts.instant_dict), - sci_spat_flexure=3.5, + sci_spat_flexure=spat_flexure, sci_spec_flexure=spec_flex_table, vel_type='HELIOCENTRIC', vel_corr=1.0+1.0e-5) From 0a3affd9c7d6c3a0270a67a29346adeedf373ce7 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 17 Aug 2024 11:11:34 +0100 Subject: [PATCH 08/54] update spatial flexure tests --- pypeit/calibrations.py | 4 ++-- pypeit/core/flexure.py | 7 +++---- pypeit/spec2dobj.py | 12 +++++++++--- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index 164e40b767..09d5727c06 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -1143,8 +1143,8 @@ def get_tilts(self): return self.wavetilts # Get flexure - _spat_flexure = self.mstilt.spat_flexure \ - if self.par['tiltframe']['process']['spat_flexure_correct'] else None + _spat_flexure = self.mstilt.spat_flexure if self.par['tiltframe']['process']['spat_flexure_correct'] != "none" \ + else None # Build buildwaveTilts = wavetilts.BuildWaveTilts( diff --git a/pypeit/core/flexure.py b/pypeit/core/flexure.py index 1d7d3e649d..5c3db3935a 100644 --- a/pypeit/core/flexure.py +++ b/pypeit/core/flexure.py @@ -98,9 +98,8 @@ def spat_flexure_shift(sciimg, slits, method="detector", maxlag=20, debug=False) # No peak? -- e.g. data fills the entire detector if len(tampl) == 0: msgs.warn('No peak found in spatial flexure. Assuming there is none...') - - return 0. - + return np.zeros((slits.nslits, 2), dtype=float) + # Find the peak xcorr_max = np.interp(pix_max, np.arange(lags.shape[0]), xcorr_norm) lag_max = np.interp(pix_max, np.arange(lags.shape[0]), lags) @@ -128,7 +127,7 @@ def spat_flexure_shift(sciimg, slits, method="detector", maxlag=20, debug=False) #display.show_slits(viewer, ch, left_flexure[:,gpm], right_flexure)[:,gpm]#, slits.id) #, args.det) #embed(header='83 of flexure.py') - return lag_max[0] + return np.full((slits.nslits, 2), lag_max[0]) def spec_flex_shift(obj_skyspec, sky_file=None, arx_skyspec=None, arx_fwhm_pix=None, diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index 55c3c93740..af0321edbe 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -85,9 +85,12 @@ class Spec2DObj(datamodel.DataContainer): descr='Table with WaveCalib diagnostic info'), 'maskdef_designtab': dict(otype=table.Table, descr='Table with slitmask design and object info'), - 'sci_spat_flexure': dict(otype=float, - descr='Shift, in spatial pixels, between this image ' - 'and SlitTrace'), + 'sci_spat_flexure': dict(otype=table.Table, + descr='Shift, in spatial pixels, between this image ' + 'and SlitTrace. Shape is (nslits, 2), where' + 'spat_flexure[i,0] is the spatial shift of the left ' + 'edge of slit i and spat_flexure[i,1] is the spatial ' + 'shift of the right edge of slit i.'), 'sci_spec_flexure': dict(otype=table.Table, descr='Global shift of the spectrum to correct for spectral' 'flexure (pixels). This is based on the sky spectrum at' @@ -244,6 +247,9 @@ def _bundle(self): # maskdef_designtab elif key == 'maskdef_designtab': d.append(dict(maskdef_designtab=self.maskdef_designtab)) + # Spatial flexure + elif key == 'sci_spat_flexure': + d.append(dict(sci_spat_flexure=self.sci_spat_flexure)) # Spectral flexure elif key == 'sci_spec_flexure': d.append(dict(sci_spec_flexure=self.sci_spec_flexure)) From 41cad8523bdc854af47f06198c946a5e3420f890 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 17 Aug 2024 13:08:53 +0100 Subject: [PATCH 09/54] update spatial flexure --- deprecated/reduce.py | 4 ++-- pypeit/extraction.py | 2 +- pypeit/find_objects.py | 19 +++++++++++++------ pypeit/flatfield.py | 15 ++++++++++----- pypeit/images/rawimage.py | 9 ++++++--- pypeit/metadata.py | 8 ++++++-- pypeit/pypeit.py | 20 +++++++++++++------- pypeit/spectrographs/keck_kcwi.py | 2 +- pypeit/wavetilts.py | 18 ++++++++++++------ 9 files changed, 64 insertions(+), 33 deletions(-) diff --git a/deprecated/reduce.py b/deprecated/reduce.py index eac1185b57..fa5a803ec2 100644 --- a/deprecated/reduce.py +++ b/deprecated/reduce.py @@ -339,7 +339,7 @@ def run_objfind(self, std_trace=None, show_peaks=False): else: tilt_flexure_shift = self.spat_flexure_shift msgs.info("Generating tilts image") - self.tilts = self.waveTilts.fit2tiltimg(self.slitmask, flexure=tilt_flexure_shift) + self.tilts = self.waveTilts.fit2tiltimg(self.slitmask, spat_flexure=tilt_flexure_shift) # # First pass object finding @@ -391,7 +391,7 @@ def prepare_extraction(self, global_sky): else: tilt_flexure_shift = self.spat_flexure_shift msgs.info("Generating tilts image") - self.tilts = self.waveTilts.fit2tiltimg(self.slitmask, flexure=tilt_flexure_shift) + self.tilts = self.waveTilts.fit2tiltimg(self.slitmask, spat_flexure=tilt_flexure_shift) # Wavelengths (on unmasked slits) msgs.info("Generating wavelength image") diff --git a/pypeit/extraction.py b/pypeit/extraction.py index a3b9f67818..12e57972f4 100644 --- a/pypeit/extraction.py +++ b/pypeit/extraction.py @@ -219,7 +219,7 @@ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_ else: tilt_flexure_shift = self.spat_flexure_shift msgs.info("Generating tilts image from fit in waveTilts") - self.tilts = self.waveTilts.fit2tiltimg(self.slitmask, flexure=tilt_flexure_shift) + self.tilts = self.waveTilts.fit2tiltimg(self.slitmask, spat_flexure=tilt_flexure_shift) elif waveTilts is None and tilts is not None: msgs.info("Using user input tilts image") self.tilts = tilts diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index 0a9ebcc65e..84981ef681 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -92,7 +92,13 @@ class FindObjects: slits (:class:`~pypeit.slittrace.SlitTraceSet`): sobjs_obj (:class:`pypeit.specobjs.SpecObjs`): Objects found - spat_flexure_shift (:obj:`float`): + spat_flexure_shift (`numpy.ndarray`_, optional): + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. tilts (`numpy.ndarray`_): WaveTilts images generated on-the-spot waveimg (`numpy.ndarray`_): @@ -153,8 +159,8 @@ def __init__(self, sciImg, slits, spectrograph, par, objtype, wv_calib=None, wav # frames. Is that okay for this usage? # Flexure self.spat_flexure_shift = None - if (objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_correct']) or \ - (objtype == 'standard' and self.par['calibrations']['standardframe']['process']['spat_flexure_correct']): + if (objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_correct'] != "none") or \ + (objtype == 'standard' and self.par['calibrations']['standardframe']['process']['spat_flexure_correct'] != "none"): self.spat_flexure_shift = self.sciImg.spat_flexure elif objtype == 'science_coadd2d': self.spat_flexure_shift = None @@ -222,14 +228,15 @@ def __init__(self, sciImg, slits, spectrograph, par, objtype, wv_calib=None, wav self.waveTilts = waveTilts self.waveTilts.is_synced(self.slits) # Deal with Flexure - if self.par['calibrations']['tiltframe']['process']['spat_flexure_correct']: - _spat_flexure = 0. if self.spat_flexure_shift is None else self.spat_flexure_shift + if self.par['calibrations']['tiltframe']['process']['spat_flexure_correct'] != "none": + _spat_flexure = np.zeros((slits.nslits, 2)) if self.spat_flexure_shift is None \ + else self.spat_flexure_shift # If they both shifted the same, there will be no reason to shift the tilts tilt_flexure_shift = _spat_flexure - self.waveTilts.spat_flexure else: tilt_flexure_shift = self.spat_flexure_shift msgs.info("Generating tilts image from fit in waveTilts") - self.tilts = self.waveTilts.fit2tiltimg(self.slitmask, flexure=tilt_flexure_shift) + self.tilts = self.waveTilts.fit2tiltimg(self.slitmask, spat_flexure=tilt_flexure_shift) elif waveTilts is None and tilts is not None: msgs.info("Using user input tilts image") self.tilts = tilts diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index 4972df7c6d..ce844f8e25 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -348,15 +348,20 @@ def fit2illumflat(self, slits, frametype='illum', finecorr=False, initial=False, zeroth order correction (finecorr=False) initial (bool, optional): If True, the initial slit edges will be used - spat_flexure (float, optional): - Spatial flexure in pixels + spat_flexure (`numpy.ndarray`_, optional): + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. Returns: `numpy.ndarray`_: An image of the spatial illumination profile for all slits. """ # Check spatial flexure type - if spat_flexure is not None and not isinstance(spat_flexure, float): - msgs.error('Spatial flexure must be None or float.') + if spat_flexure is not None and not isinstance(spat_flexure, np.ndarray): + msgs.error('Spatial flexure must be None or a 2D numpy array.') # Initialise the returned array illumflat = np.ones(self.shape, dtype=float) # Load spatial bsplines @@ -673,7 +678,7 @@ def build_waveimg(self): msgs.info("Generating wavelength image") flex = self.wavetilts.spat_flexure slitmask = self.slits.slit_img(initial=True, spat_flexure=flex) - tilts = self.wavetilts.fit2tiltimg(slitmask, flexure=flex) + tilts = self.wavetilts.fit2tiltimg(slitmask, spat_flexure=flex) # Save to class attribute for inclusion in the Flat calibration frame self.waveimg = self.wv_calib.build_waveimg(tilts, self.slits, spat_flexure=flex) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 5a945d3ca6..fce3667515 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -671,9 +671,12 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl # bias and dark subtraction) and before field flattening. Also the # function checks that the slits exist if running the spatial flexure # correction, so no need to do it again here. - self.spat_flexure_shift = None if self.par['spat_flexure_correct'] == "none" else \ - self.spat_flexure_shift = self.spatial_flexure_shift(slits, method=self.par['spat_flexure_correct'], - maxlag=self.par['spat_flexure_maxlag']) + self.spat_flexure_shift = self.spatial_flexure_shift(slits, method=self.par['spat_flexure_correct'], + maxlag=self.par['spat_flexure_maxlag']) \ + if self.par['spat_flexure_correct'] != "none" else None + + # self.spat_flexure_shift = self.spatial_flexure_shift(slits, maxlag=self.par['spat_flexure_maxlag']) \ + # if self.par['spat_flexure_correct'] else None # - Subtract scattered light... this needs to be done before flatfielding. if self.par['subtract_scattlight']: diff --git a/pypeit/metadata.py b/pypeit/metadata.py index faabcea586..87132cdc5a 100644 --- a/pypeit/metadata.py +++ b/pypeit/metadata.py @@ -666,6 +666,10 @@ def n_configs(self): 'unique_configurations first.') return len(list(self.configs.keys())) + @property + def MASKED_VALUE(self): + return -9999 + def unique_configurations(self, force=False, copy=False, rm_none=False): """ Return the unique instrument configurations. @@ -1489,7 +1493,7 @@ def get_frame_types(self, flag_unknown=False, user=None, merge=True): msgs.info("Typing completed!") return self.set_frame_types(type_bits, merge=merge) - def set_pypeit_cols(self, write_bkg_pairs=False, write_manual=False, write_shift = False): + def set_pypeit_cols(self, write_bkg_pairs=False, write_manual=False, write_shift=False): """ Generate the list of columns to be included in the fitstbl (nearly the complete list). @@ -1559,7 +1563,7 @@ def set_combination_groups(self, assign_objects=True): if 'bkg_id' not in self.keys(): self['bkg_id'] = -1 if 'shift' not in self.keys(): - self['shift'] = 0 + self['shift'] = self.MASKED_VALUE # NOTE: Importantly, this if statement means that, if the user has # defined any non-negative combination IDs in their pypeit file, none of diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index 5c5e5238ed..cb8ea6d3da 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -811,17 +811,23 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): spat_flexure = None # use the flexure correction in the "shift" column manual_flexure = self.fitstbl[frames[0]]['shift'] - if (self.objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_correct']) or \ - (self.objtype == 'standard' and self.par['calibrations']['standardframe']['process']['spat_flexure_correct']) or \ + if (self.objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_correct'] != "none") or \ + (self.objtype == 'standard' and self.par['calibrations']['standardframe']['process']['spat_flexure_correct'] != "none") or \ manual_flexure: - if (manual_flexure or manual_flexure == 0) and not (np.issubdtype(self.fitstbl[frames[0]]["shift"], np.integer)): - msgs.info(f'Implementing manual flexure of {manual_flexure}') - spat_flexure = np.float64(manual_flexure) + if (manual_flexure != self.fitstbl.MASKED_VALUE) and np.issubdtype(self.fitstbl[frames[0]]["shift"], np.integer): + msgs.info(f'Implementing manual spatial flexure of {manual_flexure}') + spat_flexure = np.full((self.caliBrate.slits.nslits, 2), np.float64(manual_flexure)) sciImg.spat_flexure = spat_flexure else: - msgs.info(f'Using auto-computed flexure') + msgs.info(f'Using auto-computed spatial flexure') spat_flexure = sciImg.spat_flexure - msgs.info(f'Flexure being used is: {spat_flexure}') + # Print the flexure values + if np.all(spat_flexure == spat_flexure[0, 0]): + msgs.info(f'Spatial flexure is: {spat_flexure[0, 0]}') + else: + # Print the flexure values for each slit separately + for slit in range(spat_flexure.shape[0]): + msgs.info(f'Spatial flexure for slit {self.caliBrate.slits.spat_id[slit]} is: left={spat_flexure[slit, 0]} right={spat_flexure[slit, 1]}') # Build the initial sky mask initial_skymask = self.load_skyregions(initial_slits=self.spectrograph.pypeline != 'SlicerIFU', scifile=sciImg.files[0], frame=frames[0], spat_flexure=spat_flexure) diff --git a/pypeit/spectrographs/keck_kcwi.py b/pypeit/spectrographs/keck_kcwi.py index abb0bd1179..c4e03be5ce 100644 --- a/pypeit/spectrographs/keck_kcwi.py +++ b/pypeit/spectrographs/keck_kcwi.py @@ -317,7 +317,7 @@ def default_pypeit_par(cls): # Illumination corrections par['scienceframe']['process']['use_illumflat'] = True # illumflat is applied when building the relative scale image in reduce.py, so should be applied to scienceframe too. par['scienceframe']['process']['use_specillum'] = True # apply relative spectral illumination - par['scienceframe']['process']['spat_flexure_correct'] = False # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames + par['scienceframe']['process']['spat_flexure_correct'] = "detector" # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames par['scienceframe']['process']['use_biasimage'] = True # Need to use bias frames for KCWI, because the bias level varies monotonically with spatial and spectral direction par['scienceframe']['process']['use_darkimage'] = False diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index 4fe808b86d..df2c6d3824 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -123,7 +123,7 @@ def is_synced(self, slits): msgs.error('Your tilt solutions are out of sync with your slits. Remove calibrations ' 'and restart from scratch.') - def fit2tiltimg(self, slitmask, flexure=None): + def fit2tiltimg(self, slitmask, spat_flexure=None): """ Generate a tilt image from the fit parameters @@ -132,7 +132,7 @@ def fit2tiltimg(self, slitmask, flexure=None): Args: slitmask (`numpy.ndarray`_): ?? - flexure (float, optional): + spat_flexure (float, optional): Spatial shift of the tilt image onto the desired frame (typically a science image) @@ -142,16 +142,21 @@ def fit2tiltimg(self, slitmask, flexure=None): """ msgs.info("Generating a tilts image from the fit parameters") - _flexure = 0. if flexure is None else flexure + _flexure = 0. if spat_flexure is None else spat_flexure final_tilts = np.zeros_like(slitmask).astype(float) gdslit_spat = np.unique(slitmask[slitmask >= 0]).astype(int) # Loop for slit_spat in gdslit_spat: slit_idx = self.spatid_to_zero(slit_spat) + # Determine the spatial flexure to use + if spat_flexure is None: + this_spat_shift = np.zeros(2) + else: + this_spat_shift = -1*_flexure[slit_idx, :] # Calculate coeff_out = self.coeffs[:self.spec_order[slit_idx]+1,:self.spat_order[slit_idx]+1,slit_idx] - _tilts = tracewave.fit2tilts(final_tilts.shape, coeff_out, self.func2d, spat_shift=-1*_flexure) + _tilts = tracewave.fit2tilts(final_tilts.shape, coeff_out, self.func2d, spat_shift=this_spat_shift) # Fill thismask_science = slitmask == slit_spat final_tilts[thismask_science] = _tilts[thismask_science] @@ -165,9 +170,10 @@ def spatid_to_zero(self, spat_id): Args: spat_id (int): + Slit spat_id Returns: - int: + int: index of slit corresponding to spat_id """ mtch = self.spat_id == spat_id @@ -223,7 +229,7 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False, wv_calib_name = wavecalib.WaveCalib.construct_file_name(self.calib_key, calib_dir=self.calib_dir) if Path(wv_calib_name).absolute().exists(): wv_calib = wavecalib.WaveCalib.from_file(wv_calib_name, chk_version=chk_version) - tilts = self.fit2tiltimg(slitmask, flexure=self.spat_flexure) + tilts = self.fit2tiltimg(slitmask, spat_flexure=self.spat_flexure) waveimg = wv_calib.build_waveimg(tilts, slits, spat_flexure=self.spat_flexure) else: msgs.warn('Could not load Wave image to show with tilts image.') From da18826288ad737d5da686af184ce1c26d20faba Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 17 Aug 2024 13:21:53 +0100 Subject: [PATCH 10/54] masked value for metadata --- pypeit/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/metadata.py b/pypeit/metadata.py index 87132cdc5a..4f1ed3e420 100644 --- a/pypeit/metadata.py +++ b/pypeit/metadata.py @@ -1599,7 +1599,7 @@ def set_user_added_columns(self): if 'manual' not in self.keys(): self['manual'] = '' if 'shift' not in self.keys(): - self['shift'] = 0 + self['shift'] = self.MASKED_VALUE def write_sorted(self, ofile, overwrite=True, ignore=None, write_bkg_pairs=False, write_manual=False): From 888a7544fec3e745090db7c47b3bf2ec8b00f7d7 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 17 Aug 2024 13:22:18 +0100 Subject: [PATCH 11/54] allow ndarray for flexure --- pypeit/wavecalib.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/pypeit/wavecalib.py b/pypeit/wavecalib.py index 9f00120579..4f936fcd7b 100644 --- a/pypeit/wavecalib.py +++ b/pypeit/wavecalib.py @@ -270,15 +270,20 @@ def build_fwhmimg(self, tilts, slits, initial=False, spat_flexure=None): Properties of the slits initial (bool, optional): If True, the initial slit locations will be used. Otherwise, the tweaked edges will be used. - spat_flexure (float, optional): - Spatial flexure correction in pixels. + spat_flexure (`numpy.ndarray`_, optional): + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. Returns: `numpy.ndarray`_: The spectral FWHM image. """ # Check spatial flexure type - if (spat_flexure is not None) and (not isinstance(spat_flexure, float)): - msgs.error("Spatial flexure must be None or float") + if (spat_flexure is not None) and (not isinstance(spat_flexure, np.ndarray)): + msgs.error("Spatial flexure must be None or numpy.ndarray.") # Generate the slit mask and slit edges - pad slitmask by 1 for edge effects slitmask = slits.slit_img(pad=1, initial=initial, spat_flexure=spat_flexure) slits_left, slits_right, _ = slits.select_edges(initial=initial, spat_flexure=spat_flexure) @@ -303,8 +308,13 @@ def build_waveimg(self, tilts, slits, spat_flexure=None, spec_flexure=None): Image holding tilts slits (:class:`pypeit.slittrace.SlitTraceSet`): Properties of the slits - spat_flexure (float, optional): - Spatial flexure correction in pixels. + spat_flexure (`numpy.ndarray`_, optional): + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. spec_flexure (float, `numpy.ndarray`_, optional): Spectral flexure correction in pixels. If a float, the same spectral flexure correction will be applied @@ -317,8 +327,8 @@ def build_waveimg(self, tilts, slits, spat_flexure=None, spec_flexure=None): `numpy.ndarray`_: The wavelength image. """ # Check spatial flexure type - if (spat_flexure is not None) and (not isinstance(spat_flexure, float)): - msgs.error("Spatial flexure must be None or float") + if (spat_flexure is not None) and (not isinstance(spat_flexure, np.ndarray)): + msgs.error("Spatial flexure must be None or numpy.ndarray") # Check spectral flexure type if spec_flexure is None: spec_flex = np.zeros(slits.nslits) elif isinstance(spec_flexure, float): spec_flex = spec_flexure*np.ones(slits.nslits) From e3f612274dd33e53ea1efd33c2c79c29c93399d5 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 17 Aug 2024 13:36:55 +0100 Subject: [PATCH 12/54] update spec2d for flexure --- pypeit/spec2dobj.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index af0321edbe..3547f84b8c 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -48,7 +48,7 @@ class Spec2DObj(datamodel.DataContainer): Primary header if instantiated from a FITS file """ - version = '1.1.1' + version = '1.1.2' # TODO 2d data model should be expanded to include: # waveimage -- flexure and heliocentric corrections should be applied to the final waveimage and since this is unique to @@ -85,7 +85,7 @@ class Spec2DObj(datamodel.DataContainer): descr='Table with WaveCalib diagnostic info'), 'maskdef_designtab': dict(otype=table.Table, descr='Table with slitmask design and object info'), - 'sci_spat_flexure': dict(otype=table.Table, + 'sci_spat_flexure': dict(otype=np.ndarray, atype=np.floating, descr='Shift, in spatial pixels, between this image ' 'and SlitTrace. Shape is (nslits, 2), where' 'spat_flexure[i,0] is the spatial shift of the left ' From 65b84b95c5f5ea808f83b3c6457edebb03d89614 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 17 Aug 2024 13:37:15 +0100 Subject: [PATCH 13/54] array spatial flexure during findobj --- pypeit/find_objects.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index 84981ef681..a737bf17b3 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -1223,8 +1223,8 @@ def calculate_flexure(self, global_sky): new_slitshift = self.slitshift + this_slitshift # Now report the flexure values for slit_idx, slit_spat in enumerate(self.slits.spat_id): - msgs.info("Flexure correction, slit {0:d} (spat id={1:d}): {2:.3f} pixels".format(1+slit_idx, slit_spat, - self.slitshift[slit_idx])) + msgs.info("Spectral flexure correction, slit {0:d} (spat id={1:d}): {2:.3f} pixels".format(1+slit_idx, slit_spat, + new_slitshift[slit_idx])) # Save QA # TODO :: Need to implement QA once the flexure code has been tidied up, and this routine has been moved # out of the find_objects() class. From 0802abfcd5b0920e4291fdf79f5703beddffae9e Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 17 Aug 2024 13:52:41 +0100 Subject: [PATCH 14/54] speed up fit2tilts when only calculating one slit at a time. --- pypeit/core/tracewave.py | 71 +++++++++++++++++++++++++++++++++------- 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/pypeit/core/tracewave.py b/pypeit/core/tracewave.py index 622094a9be..52c9065847 100644 --- a/pypeit/core/tracewave.py +++ b/pypeit/core/tracewave.py @@ -851,7 +851,7 @@ def fit_tilts(trc_tilt_dict, thismask, slit_cen, spat_order=3, spec_order=4, max # msgs.info("RMS/FWHM: {}".format(rms_real/fwhm)) -def fit2tilts(shape, coeff2, func2d, spat_shift=None): +def fit2tilts(shape, coeff2, func2d, spec_eval=None, spat_eval=None, spat_shift=None): """ Evaluate the wavelength tilt model over the full image. @@ -863,10 +863,22 @@ def fit2tilts(shape, coeff2, func2d, spat_shift=None): result of griddata tilt fit func2d : str the 2d function used to fit the tilts - spat_shift : float, optional + spat_shift : float, `numpy.ndarray`_, optional Spatial shift to be added to image pixels before evaluation If you are accounting for flexure, then you probably wish to - input -1*flexure_shift into this parameter. + input -1*flexure_shift into this parameter. Note that this + should either be a float or a 1D array with two elements + (and both of these elements must be equal). + spat_eval : `numpy.ndarray`_, optional + 1D array indicating how spatial pixel locations move across the + image. If spat_eval is provided, spec_eval must also be provided. + spat_shift is ignored when spat_eval and spec_eval are provided. + If you wish to account for spatial flexure, then you should include + the spatial flexure into this parameter. spat_eval should be given + as spatial_coordinates + spatial_flexure. + spec_eval : `numpy.ndarray`_, optional + 1D array indicating how spectral pixel locations move across the + image. If spec_eval is provided, spat_eval must also be provided. Returns ------- @@ -876,18 +888,53 @@ def fit2tilts(shape, coeff2, func2d, spat_shift=None): """ # Init - _spat_shift = 0. if spat_shift is None else spat_shift + # TODO :: Need to deal with the spatial flexure here... + # :: One possibility is to add another function that evaluates the tilts + # :: only at the slit locations. This might require that the slit edges + # :: are provided as input as well (possibly to another function)?. + if spec_eval is not None and spat_eval is not None: + if spat_shift is not None: + msgs.warn('spat_shift is ignored when spec_eval and spat_eval are provided.') + _spec_eval = spec_eval + _spat_eval = spat_eval + else: + # Print a warning just in case only one was provided + if (spec_eval is None and spat_eval is not None) or (spec_eval is not None and spat_eval is None): + msgs.warn('Both spec_eval and spat_eval must be provided.' + msgs.newline() + + 'Only one variable provided, so a new (full) grid will be generated.') + # Check the flexure + if spat_shift is None: + _spat_shift = 0. + elif isinstance(spat_shift, (int, float)): + _spat_shift = spat_shift + elif isinstance(spat_shift, np.ndarray): + if spat_shift.size != 2: + msgs.error('spat_shift must be a 2-element array.') + elif spat_shift[0] != spat_shift[1]: + msgs.error('The two elements of spat_shift must be equal.' + msgs.newline() + + 'To include different spatial shifts at the two ends of the slit, ' + msgs.newline() + + 'you must provide the variables spec_eval and spat_eval instead of spat_shift.') + else: + _spat_shift = spat_shift[0] + else: + msgs.error('spat_shift must be either None, a float, or a 1D array with two identical elements.' + msgs.newline() + + 'To include different spatial shifts at the two ends of the slit, ' + msgs.newline() + + 'you must provide the variables spec_eval and spat_eval instead of spat_shift.') + + # Setup the evaluation grid + nspec, nspat = shape + xnspecmin1 = float(nspec - 1) + xnspatmin1 = float(nspat - 1) + spec_vec = np.arange(nspec) + spat_vec = np.arange(nspat) - _spat_shift + spat_img, spec_img = np.meshgrid(spat_vec, spec_vec) + _spec_eval = spec_img / xnspecmin1 + _spat_eval = spat_img / xnspatmin1 + # Compute the tilts image - nspec, nspat = shape - xnspecmin1 = float(nspec - 1) - xnspatmin1 = float(nspat - 1) - spec_vec = np.arange(nspec) - spat_vec = np.arange(nspat) - _spat_shift - spat_img, spec_img = np.meshgrid(spat_vec, spec_vec) - # pypeitFit = fitting.PypeItFit(fitc=coeff2, minx=0.0, maxx=1.0, minx2=0.0, maxx2=1.0, func=func2d) - tilts = pypeitFit.eval(spec_img / xnspecmin1, x2=spat_img / xnspatmin1) + tilts = pypeitFit.eval(_spec_eval, x2=_spat_eval) # Added this to ensure that tilts are never crazy values due to extrapolation of fits which can break # wavelength solution fitting return np.fmax(np.fmin(tilts, 1.2), -0.2) From 8f67040c9d5f2251bf29a5704e53c74fb05d800b Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 17 Aug 2024 13:53:05 +0100 Subject: [PATCH 15/54] speed up fit2tilts when only calculating one slit at a time. --- pypeit/core/tracewave.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pypeit/core/tracewave.py b/pypeit/core/tracewave.py index 52c9065847..8e3a14d8f0 100644 --- a/pypeit/core/tracewave.py +++ b/pypeit/core/tracewave.py @@ -863,12 +863,6 @@ def fit2tilts(shape, coeff2, func2d, spec_eval=None, spat_eval=None, spat_shift= result of griddata tilt fit func2d : str the 2d function used to fit the tilts - spat_shift : float, `numpy.ndarray`_, optional - Spatial shift to be added to image pixels before evaluation - If you are accounting for flexure, then you probably wish to - input -1*flexure_shift into this parameter. Note that this - should either be a float or a 1D array with two elements - (and both of these elements must be equal). spat_eval : `numpy.ndarray`_, optional 1D array indicating how spatial pixel locations move across the image. If spat_eval is provided, spec_eval must also be provided. @@ -879,6 +873,12 @@ def fit2tilts(shape, coeff2, func2d, spec_eval=None, spat_eval=None, spat_shift= spec_eval : `numpy.ndarray`_, optional 1D array indicating how spectral pixel locations move across the image. If spec_eval is provided, spat_eval must also be provided. + spat_shift : float, `numpy.ndarray`_, optional + Spatial shift to be added to image pixels before evaluation + If you are accounting for flexure, then you probably wish to + input -1*flexure_shift into this parameter. Note that this + should either be a float or a 1D array with two elements + (and both of these elements must be equal). Returns ------- From 233a03f58d5295ee8a95056613830449da0a1d91 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 17 Aug 2024 21:38:08 +0100 Subject: [PATCH 16/54] speed up fit2tilts when only calculating one slit at a time. --- pypeit/core/tracewave.py | 59 +++++++++++++++++++++++++++++----- pypeit/extraction.py | 3 +- pypeit/find_objects.py | 3 +- pypeit/flatfield.py | 9 +++--- pypeit/slittrace.py | 8 ++--- pypeit/wavetilts.py | 68 +++++++++++++++++++++++++--------------- 6 files changed, 106 insertions(+), 44 deletions(-) diff --git a/pypeit/core/tracewave.py b/pypeit/core/tracewave.py index 8e3a14d8f0..2c8f7c6f8f 100644 --- a/pypeit/core/tracewave.py +++ b/pypeit/core/tracewave.py @@ -851,6 +851,55 @@ def fit_tilts(trc_tilt_dict, thismask, slit_cen, spat_order=3, spec_order=4, max # msgs.info("RMS/FWHM: {}".format(rms_real/fwhm)) +def fit2tilts_prepareSlit(img_shape, slit_left, slit_right, thismask_science, spat_flexure=None): + """ + Prepare the slit for the fit2tilts function + + Parameters + ---------- + img_shape : :obj:`tuple` + Shape of the science image + slit_left : `numpy.ndarray`_ + Left slit edge + slit_right : `numpy.ndarray`_ + Right slit edge + thismask_science : `numpy.ndarray`_ + Boolean mask for the science pixels in this slit (True = on slit) + spat_flexure : `numpy.ndarray`_, optional + Spatial flexure in pixels. This should be a two element array with the first element being the flexure at the + left edge of the slit and the second element being the flexure at the right edge of the slit. If None, no + flexure is applied. + + Returns + ------- + :obj:`tuple` of two `numpy.ndarray`_ + Tuple containing the spectral and spatial coordinates of the slit, including spatial flexure. + These variables are to be used in the fit2tilts function. + """ + # Check the spatial flexure input + if spat_flexure is not None and len(spat_flexure) != 2: + msgs.error('Spatial flexure must be a two element array') + _spat_flexure = np.zeros(2) if spat_flexure is None else spat_flexure + # Check dimensions + if len(slit_left) != len(slit_right): + msgs.error('Slit left and right edges must have the same length') + if len(slit_left) != img_shape[0]: + msgs.error('Slit edges must have the same length as the spectral dimension of the image') + + # Prepare the spatial and spectral coordinates + nspec, nspat = img_shape + spec_vec = np.arange(nspec) + spat_vec = np.arange(nspat) + spat_img, spec_img = np.meshgrid(spat_vec, spec_vec) + # Evaluate the slit coordinates + _spec_eval = spec_img[thismask_science] / (nspec - 1) + flex_coord = (spat_img - slit_left[:, None]) / (slit_right - slit_left)[:, None] + # Linearly interpolate the spatial flexure over the slit coordinates + spat_eval = spat_img + _spat_flexure[0] + flex_coord * (_spat_flexure[1] - _spat_flexure[0]) + _spat_eval = spat_eval[thismask_science] / (nspat - 1) + return _spec_eval, _spat_eval + + def fit2tilts(shape, coeff2, func2d, spec_eval=None, spat_eval=None, spat_shift=None): """ Evaluate the wavelength tilt model over the full image. @@ -883,15 +932,9 @@ def fit2tilts(shape, coeff2, func2d, spec_eval=None, spat_eval=None, spat_shift= Returns ------- tilts : `numpy.ndarray`_, float - Image indicating how spectral pixel locations move across the - image. This output is used in the pipeline. - + Image indicating how spectral pixel locations move across the image. """ - # Init - # TODO :: Need to deal with the spatial flexure here... - # :: One possibility is to add another function that evaluates the tilts - # :: only at the slit locations. This might require that the slit edges - # :: are provided as input as well (possibly to another function)?. + # Determine which mode to run this function if spec_eval is not None and spat_eval is not None: if spat_shift is not None: msgs.warn('spat_shift is ignored when spec_eval and spat_eval are provided.') diff --git a/pypeit/extraction.py b/pypeit/extraction.py index 12e57972f4..2a9c2a0acf 100644 --- a/pypeit/extraction.py +++ b/pypeit/extraction.py @@ -219,7 +219,8 @@ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_ else: tilt_flexure_shift = self.spat_flexure_shift msgs.info("Generating tilts image from fit in waveTilts") - self.tilts = self.waveTilts.fit2tiltimg(self.slitmask, spat_flexure=tilt_flexure_shift) + self.tilts = self.waveTilts.fit2tiltimg(self.slitmask, self.slits_left, self.slits_right, + spat_flexure=tilt_flexure_shift) elif waveTilts is None and tilts is not None: msgs.info("Using user input tilts image") self.tilts = tilts diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index a737bf17b3..cd5c4a2a8d 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -236,7 +236,8 @@ def __init__(self, sciImg, slits, spectrograph, par, objtype, wv_calib=None, wav else: tilt_flexure_shift = self.spat_flexure_shift msgs.info("Generating tilts image from fit in waveTilts") - self.tilts = self.waveTilts.fit2tiltimg(self.slitmask, spat_flexure=tilt_flexure_shift) + self.tilts = self.waveTilts.fit2tiltimg(self.slitmask, self.slits_left, self.slits_right, + spat_flexure=tilt_flexure_shift) elif waveTilts is None and tilts is not None: msgs.info("Using user input tilts image") self.tilts = tilts diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index ce844f8e25..d4a9d9bfcb 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -676,11 +676,12 @@ def build_waveimg(self): Generate an image of the wavelength of each pixel. """ msgs.info("Generating wavelength image") - flex = self.wavetilts.spat_flexure - slitmask = self.slits.slit_img(initial=True, spat_flexure=flex) - tilts = self.wavetilts.fit2tiltimg(slitmask, spat_flexure=flex) + spat_flexure = self.wavetilts.spat_flexure + left, right, msk = self.slits.select_edges(spat_flexure=spat_flexure) + slitmask = self.slits.slit_img(initial=True, spat_flexure=spat_flexure) + tilts = self.wavetilts.fit2tiltimg(slitmask, left, right, spat_flexure=spat_flexure) # Save to class attribute for inclusion in the Flat calibration frame - self.waveimg = self.wv_calib.build_waveimg(tilts, self.slits, spat_flexure=flex) + self.waveimg = self.wv_calib.build_waveimg(tilts, self.slits, spat_flexure=spat_flexure) def show(self, wcs_match=True): """ diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 47e7d256e2..62d19380b8 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -599,8 +599,8 @@ def select_edges(self, initial=False, spat_flexure=None): # Add in spatial flexure? if spat_flexure is not None: - self.left_flexure = left + spat_flexure[:,0] - self.right_flexure = right + spat_flexure[:,1] + self.left_flexure = left + spat_flexure[:, 0] + self.right_flexure = right + spat_flexure[:, 1] left, right = self.left_flexure, self.right_flexure # Return @@ -828,14 +828,14 @@ def spatial_coordinates(self, initial=False, spat_flexure=None): `numpy.ndarray`_: Vector with the list of floating point spatial coordinates. """ - # TODO -- Confirm it makes sense to pass in flexure + # TODO -- Confirm it makes sense to pass in spatial flexure left, right, _ = self.select_edges(initial=initial, spat_flexure=spat_flexure) return SlitTraceSet.slit_spat_pos(left, right, self.nspat) @staticmethod def slit_spat_pos(left, right, nspat): r""" - Return a fidicial, normalized spatial coordinate for each slit. + Return a fiducial, normalized spatial coordinate for each slit. The fiducial coordinates are given by:: diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index df2c6d3824..1ce89e0927 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -123,43 +123,53 @@ def is_synced(self, slits): msgs.error('Your tilt solutions are out of sync with your slits. Remove calibrations ' 'and restart from scratch.') - def fit2tiltimg(self, slitmask, spat_flexure=None): + def fit2tiltimg(self, slitmask, slits_left, slits_right, spat_flexure=None): """ Generate a tilt image from the fit parameters - - Mainly to allow for flexure + Mainly to allow for spatial flexure Args: slitmask (`numpy.ndarray`_): - ?? - spat_flexure (float, optional): - Spatial shift of the tilt image onto the desired frame - (typically a science image) + An image identifying the slit/order at each pixel. Pixels + without a slit are marked with -1. + slits_left (`numpy.ndarray`_): + Left slit edges + slits_right (`numpy.ndarray`_): + Right slit edges + spat_flexure (`numpy.ndarray`_, optional): + If provided, this is the shift, in spatial pixels, of the tilt + image onto the desired frame (typically a science image). The + shifts apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. Returns: `numpy.ndarray`_: New tilt image - """ msgs.info("Generating a tilts image from the fit parameters") - _flexure = 0. if spat_flexure is None else spat_flexure + # Check the optional inputs + _flexure = np.zeros_like((slits_left.shape[1], 2)) if spat_flexure is None else spat_flexure + # Setup the output image final_tilts = np.zeros_like(slitmask).astype(float) gdslit_spat = np.unique(slitmask[slitmask >= 0]).astype(int) - # Loop + # Loop through all good slits for slit_spat in gdslit_spat: + # Get the slit index slit_idx = self.spatid_to_zero(slit_spat) - # Determine the spatial flexure to use - if spat_flexure is None: - this_spat_shift = np.zeros(2) - else: - this_spat_shift = -1*_flexure[slit_idx, :] - # Calculate - coeff_out = self.coeffs[:self.spec_order[slit_idx]+1,:self.spat_order[slit_idx]+1,slit_idx] - _tilts = tracewave.fit2tilts(final_tilts.shape, coeff_out, self.func2d, spat_shift=this_spat_shift) - # Fill - thismask_science = slitmask == slit_spat - final_tilts[thismask_science] = _tilts[thismask_science] + # Prepare the coefficients + coeff_out = self.coeffs[:self.spec_order[slit_idx]+1, :self.spat_order[slit_idx]+1, slit_idx] + # Extract the spectral and spatial coordinates for this slit + thismask_science = (slitmask == slit_spat) + _spec_eval, _spat_eval = tracewave.fit2tilts_prepareSlit(slitmask.shape, + slits_left[:, slit_idx], slits_right[:, slit_idx], + thismask_science, spat_flexure[slit_idx, :]) + # Calculate the tilts + final_tilts[thismask_science] = tracewave.fit2tilts(final_tilts.shape, coeff_out, self.func2d, + spec_eval=_spec_eval, spat_eval=_spat_eval) # Return return final_tilts @@ -174,7 +184,7 @@ def spatid_to_zero(self, spat_id): Returns: int: index of slit corresponding to spat_id - + TODO :: This code is a direct copy of the slits method (and only used in the function above. Should be consolidated... """ mtch = self.spat_id == spat_id return np.where(mtch)[0][0] @@ -229,7 +239,7 @@ def show(self, waveimg=None, wcs_match=True, in_ginga=True, show_traces=False, wv_calib_name = wavecalib.WaveCalib.construct_file_name(self.calib_key, calib_dir=self.calib_dir) if Path(wv_calib_name).absolute().exists(): wv_calib = wavecalib.WaveCalib.from_file(wv_calib_name, chk_version=chk_version) - tilts = self.fit2tiltimg(slitmask, spat_flexure=self.spat_flexure) + tilts = self.fit2tiltimg(slitmask, _left, _right, spat_flexure=self.spat_flexure) waveimg = wv_calib.build_waveimg(tilts, slits, spat_flexure=self.spat_flexure) else: msgs.warn('Could not load Wave image to show with tilts image.') @@ -712,6 +722,9 @@ def run(self, doqa=True, debug=False, show=False): self.spat_order = np.zeros(self.slits.nslits, dtype=int) self.spec_order = np.zeros(self.slits.nslits, dtype=int) + # Grab the slit edges + slits_left, slits_right, _ = self.slits.select_edges(initial=True, spat_flexure=self.spat_flexure) + # Loop on all slits for slit_idx, slit_spat in enumerate(self.slits.spat_id): if self.tilt_bpm[slit_idx]: @@ -777,10 +790,13 @@ def run(self, doqa=True, debug=False, show=False): # Tilts are created with the size of the original slitmask, # which corresonds to the same binning as the science # images, trace images, and pixelflats etc. - self.tilts = tracewave.fit2tilts(self.slitmask_science.shape, coeff_out, - self.par['func2d']) - # Save to final image thismask_science = self.slitmask_science == slit_spat + _spec_eval, _spat_eval = tracewave.fit2tilts_prepareSlit(self.slitmask_science.shape, + slits_left[:, slit_idx], slits_right[:, slit_idx], + thismask_science, self.spat_flexure[slit_idx, :]) + self.tilts[thismask_science] = tracewave.fit2tilts(self.slitmask_science.shape, coeff_out, self.par['func2d'], + spec_eval=_spec_eval, spat_eval=_spat_eval) + # Save to final image self.final_tilts[thismask_science] = self.tilts[thismask_science] if show: From b2f56e2c94be47bbff12affd4d41163620ef4669 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 18 Aug 2024 09:32:48 +0100 Subject: [PATCH 17/54] fix None spatflex --- pypeit/flatfield.py | 2 +- pypeit/pypeit.py | 12 +++++++++--- pypeit/spec2dobj.py | 10 +++++----- pypeit/wavetilts.py | 19 +++++++++++-------- 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index d4a9d9bfcb..5d9a833b9b 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -969,7 +969,7 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): # Create the tilts image for this slit # TODO -- JFH Confirm the sign of this shift is correct! - _flexure = 0. if self.wavetilts.spat_flexure is None else self.wavetilts.spat_flexure + _flexure = np.zeros(2) if self.wavetilts.spat_flexure is None else self.wavetilts.spat_flexure[slit_idx, :] tilts = tracewave.fit2tilts(rawflat.shape, self.wavetilts['coeffs'][:,:,slit_idx], self.wavetilts['func2d'], spat_shift=-1*_flexure) # Convert the tilt image to an image with the spectral pixel index diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index cb8ea6d3da..f9e9621117 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -808,7 +808,7 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): sciImg = bkg_redux_sciimg.sub(bgimg) # Flexure - spat_flexure = None + spat_flexure = np.zeros((self.caliBrate.slits.nslits,1)) # No spatial flexure, unless we find it below # use the flexure correction in the "shift" column manual_flexure = self.fitstbl[frames[0]]['shift'] if (self.objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_correct'] != "none") or \ @@ -819,8 +819,14 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): spat_flexure = np.full((self.caliBrate.slits.nslits, 2), np.float64(manual_flexure)) sciImg.spat_flexure = spat_flexure else: - msgs.info(f'Using auto-computed spatial flexure') - spat_flexure = sciImg.spat_flexure + if sciImg.spat_flexure is not None: + msgs.info(f'Using auto-computed spatial flexure') + spat_flexure = sciImg.spat_flexure + else: + msgs.info('Assuming no spatial flexure correction') + else: + msgs.info('Assuming no spatial flexure correction') + # Print the flexure values if np.all(spat_flexure == spat_flexure[0, 0]): msgs.info(f'Spatial flexure is: {spat_flexure[0, 0]}') diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index 3547f84b8c..a44ee48a22 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -72,8 +72,8 @@ class Spec2DObj(datamodel.DataContainer): 'tilts': dict(otype=np.ndarray, atype=np.floating, descr='2D tilts image (float64)'), 'scaleimg': dict(otype=np.ndarray, atype=np.floating, - descr='2D multiplicative scale image [or a single scalar as an array] that has been applied to ' - 'the science image (float32)'), + descr='2D multiplicative scale image [or a single scalar as an array] ' + 'that has been applied to the science image (float32)'), 'waveimg': dict(otype=np.ndarray, atype=np.floating, descr='2D wavelength image in vacuum (float64)'), 'bpmmask': dict(otype=imagebitmask.ImageBitMaskArray, @@ -87,13 +87,13 @@ class Spec2DObj(datamodel.DataContainer): descr='Table with slitmask design and object info'), 'sci_spat_flexure': dict(otype=np.ndarray, atype=np.floating, descr='Shift, in spatial pixels, between this image ' - 'and SlitTrace. Shape is (nslits, 2), where' + 'and SlitTrace. Shape is (nslits, 2), where ' 'spat_flexure[i,0] is the spatial shift of the left ' 'edge of slit i and spat_flexure[i,1] is the spatial ' 'shift of the right edge of slit i.'), 'sci_spec_flexure': dict(otype=table.Table, - descr='Global shift of the spectrum to correct for spectral' - 'flexure (pixels). This is based on the sky spectrum at' + descr='Global shift of the spectrum to correct for spectral ' + 'flexure (pixels). This is based on the sky spectrum at ' 'the center of each slit'), 'vel_type': dict(otype=str, descr='Type of reference frame correction (if any). ' 'Options are listed in the routine: ' diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index 1ce89e0927..c7f2b5026e 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -45,7 +45,7 @@ class WaveTilts(calibframe.CalibFrame): included in the output. """ - version = '1.2.0' + version = '1.2.1' # Calibration frame attributes calib_type = 'Tilts' @@ -71,7 +71,12 @@ class WaveTilts(calibframe.CalibFrame): 'spec_order': dict(otype=np.ndarray, atype=np.integer, descr='Order for spectral fit (nslit)'), 'func2d': dict(otype=str, descr='Function used for the 2D fit'), - 'spat_flexure': dict(otype=float, descr='Flexure shift from the input TiltImage'), + 'spat_flexure': dict(otype=np.ndarray, atype=np.floating, + descr='Spatial flexure shift, in spatial pixels, between TiltImage ' + 'and SlitTrace. Shape is (nslits, 2), where ' + 'spat_flexure[i,0] is the spatial shift of the left ' + 'edge of slit i and spat_flexure[i,1] is the spatial ' + 'shift of the right edge of slit i.'), 'slits_filename': dict(otype=str, descr='Path to SlitTraceSet file. This helps to ' 'find the Slits calibration file when running ' 'pypeit_chk_tilts()'), @@ -151,7 +156,7 @@ def fit2tiltimg(self, slitmask, slits_left, slits_right, spat_flexure=None): msgs.info("Generating a tilts image from the fit parameters") # Check the optional inputs - _flexure = np.zeros_like((slits_left.shape[1], 2)) if spat_flexure is None else spat_flexure + _spat_flexure = np.zeros((slits_left.shape[1], 2)) if spat_flexure is None else spat_flexure # Setup the output image final_tilts = np.zeros_like(slitmask).astype(float) @@ -166,7 +171,7 @@ def fit2tiltimg(self, slitmask, slits_left, slits_right, spat_flexure=None): thismask_science = (slitmask == slit_spat) _spec_eval, _spat_eval = tracewave.fit2tilts_prepareSlit(slitmask.shape, slits_left[:, slit_idx], slits_right[:, slit_idx], - thismask_science, spat_flexure[slit_idx, :]) + thismask_science, _spat_flexure[slit_idx, :]) # Calculate the tilts final_tilts[thismask_science] = tracewave.fit2tilts(final_tilts.shape, coeff_out, self.func2d, spec_eval=_spec_eval, spat_eval=_spat_eval) @@ -331,7 +336,7 @@ def __init__(self, mstilt, slits, spectrograph, par, wavepar, det=1, qa_path=Non self.slits = slits self.det = det self.qa_path = qa_path - self.spat_flexure = spat_flexure + self.spat_flexure = spat_flexure if spat_flexure is not None else np.zeros((slits.nslits, 2), dtype=float) # Get the non-linear count level # TODO: This is currently hacked to deal with Mosaics @@ -794,10 +799,8 @@ def run(self, doqa=True, debug=False, show=False): _spec_eval, _spat_eval = tracewave.fit2tilts_prepareSlit(self.slitmask_science.shape, slits_left[:, slit_idx], slits_right[:, slit_idx], thismask_science, self.spat_flexure[slit_idx, :]) - self.tilts[thismask_science] = tracewave.fit2tilts(self.slitmask_science.shape, coeff_out, self.par['func2d'], + self.final_tilts[thismask_science] = tracewave.fit2tilts(self.slitmask_science.shape, coeff_out, self.par['func2d'], spec_eval=_spec_eval, spat_eval=_spat_eval) - # Save to final image - self.final_tilts[thismask_science] = self.tilts[thismask_science] if show: viewer, ch = display.show_image(self.mstilt.image * (self.slitmask > -1), chname='tilts') From c86977d578d16357fe7b7af5b8572a02e12f4854 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 18 Aug 2024 12:47:41 +0100 Subject: [PATCH 18/54] fix spatial flexure in spectrographs --- pypeit/spectrographs/gemini_gnirs.py | 2 +- pypeit/spectrographs/gtc_osiris.py | 2 +- pypeit/spectrographs/keck_lris.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pypeit/spectrographs/gemini_gnirs.py b/pypeit/spectrographs/gemini_gnirs.py index 52e346661b..5871bbd8a5 100644 --- a/pypeit/spectrographs/gemini_gnirs.py +++ b/pypeit/spectrographs/gemini_gnirs.py @@ -606,7 +606,7 @@ def default_pypeit_par(cls): par['scienceframe']['process']['objlim'] = 1.5 par['scienceframe']['process']['use_illumflat'] = False # illumflat is applied when building the relative scale image in reduce.py, so should be applied to scienceframe too. par['scienceframe']['process']['use_specillum'] = False # apply relative spectral illumination - par['scienceframe']['process']['spat_flexure_correct'] = False # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames + par['scienceframe']['process']['spat_flexure_correct'] = "none" # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames par['scienceframe']['process']['use_biasimage'] = False par['scienceframe']['process']['use_darkimage'] = False par['calibrations']['flatfield']['slit_illum_finecorr'] = False diff --git a/pypeit/spectrographs/gtc_osiris.py b/pypeit/spectrographs/gtc_osiris.py index 78bc81fa21..80eb64a5d4 100644 --- a/pypeit/spectrographs/gtc_osiris.py +++ b/pypeit/spectrographs/gtc_osiris.py @@ -460,7 +460,7 @@ def default_pypeit_par(cls): par['scienceframe']['process']['objlim'] = 1.5 par['scienceframe']['process']['use_illumflat'] = False # illumflat is applied when building the relative scale image in reduce.py, so should be applied to scienceframe too. par['scienceframe']['process']['use_specillum'] = False # apply relative spectral illumination - par['scienceframe']['process']['spat_flexure_correct'] = False # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames + par['scienceframe']['process']['spat_flexure_correct'] = "none" # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames par['scienceframe']['process']['use_biasimage'] = False par['scienceframe']['process']['use_darkimage'] = False par['calibrations']['flatfield']['slit_illum_finecorr'] = False diff --git a/pypeit/spectrographs/keck_lris.py b/pypeit/spectrographs/keck_lris.py index ffc42afb87..c1d8f04a5e 100644 --- a/pypeit/spectrographs/keck_lris.py +++ b/pypeit/spectrographs/keck_lris.py @@ -94,8 +94,8 @@ def default_pypeit_par(cls): # Always correct for spatial flexure on science images # TODO -- Decide whether to make the following defaults # May not want to do them for LongSlit - par['scienceframe']['process']['spat_flexure_correct'] = True - par['calibrations']['standardframe']['process']['spat_flexure_correct'] = True + par['scienceframe']['process']['spat_flexure_correct'] = "detector" + par['calibrations']['standardframe']['process']['spat_flexure_correct'] = "detector" par['scienceframe']['exprng'] = [61, None] From 0183a91021f7e064063a026ea64f4ad0f833c41f Mon Sep 17 00:00:00 2001 From: rcooke Date: Sun, 18 Aug 2024 12:50:47 +0100 Subject: [PATCH 19/54] allow fits.gz --- pypeit/spectrographs/gemini_gmos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypeit/spectrographs/gemini_gmos.py b/pypeit/spectrographs/gemini_gmos.py index c9ecd8d993..354f95c914 100644 --- a/pypeit/spectrographs/gemini_gmos.py +++ b/pypeit/spectrographs/gemini_gmos.py @@ -108,7 +108,7 @@ class GeminiGMOSSpectrograph(spectrograph.Spectrograph): """ ndet = 3 url = 'http://www.gemini.edu/instrumentation/gmos' - allowed_extensions = ['.fits', '.fits.bz2'] + allowed_extensions = ['.fits', '.fits.bz2', 'fits.gz'] def __init__(self): super().__init__() @@ -1046,7 +1046,7 @@ class GeminiGMOSNSpectrograph(GeminiGMOSSpectrograph): telescope = telescopes.GeminiNTelescopePar() camera = 'GMOS-N' header_name = 'GMOS-N' - allowed_extensions = ['.fits', '.fits.bz2'] + allowed_extensions = ['.fits', '.fits.bz2', '.fits.gz'] class GeminiGMOSNHamSpectrograph(GeminiGMOSNSpectrograph): From 70d2925e8b8f8520c56b14f6eb635838af2eae00 Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 19 Aug 2024 11:43:40 +0100 Subject: [PATCH 20/54] fix tests --- pypeit/pypeit.py | 4 ++-- pypeit/slittrace.py | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index f9e9621117..80a0e9ec6f 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -592,7 +592,7 @@ def reduce_exposure(self, frames, bg_frames=None, std_outfile=None): # slitmask stuff if len(calibrated_det) > 0 and self.par['reduce']['slitmask']['assign_obj']: # get object positions from slitmask design and slitmask offsets for all the detectors - spat_flexure = np.array([ss.spat_flexure for ss in sciImg_list]) + spat_flexure = [ss.spat_flexure for ss in sciImg_list] # Grab platescale with binning bin_spec, bin_spat = parse.parse_binning(self.binning) platescale = np.array([ss.detector.platescale*bin_spat for ss in sciImg_list]) @@ -808,7 +808,7 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): sciImg = bkg_redux_sciimg.sub(bgimg) # Flexure - spat_flexure = np.zeros((self.caliBrate.slits.nslits,1)) # No spatial flexure, unless we find it below + spat_flexure = np.zeros((self.caliBrate.slits.nslits, 2)) # No spatial flexure, unless we find it below # use the flexure correction in the "shift" column manual_flexure = self.fitstbl[frames[0]]['shift'] if (self.objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_correct'] != "none") or \ diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 62d19380b8..f69029e734 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -598,9 +598,12 @@ def select_edges(self, initial=False, spat_flexure=None): left, right = self.left_init, self.right_init # Add in spatial flexure? + self.left_flexure = left.copy() + self.right_flexure = right.copy() if spat_flexure is not None: - self.left_flexure = left + spat_flexure[:, 0] - self.right_flexure = right + spat_flexure[:, 1] + for slit in range(self.nslits): + self.left_flexure[:,slit] += spat_flexure[slit, 0] + self.right_flexure[:,slit] += spat_flexure[slit, 1] left, right = self.left_flexure, self.right_flexure # Return @@ -1609,9 +1612,9 @@ def get_maskdef_objpos_offset_alldets(sobjs, calib_slits, spat_flexure, platesca List of SpecObj that have been found and traced calib_slits (:obj:`list`): List of `SlitTraceSet` with information on the traced slit edges - spat_flexure (`numpy.ndarray`_, optional): - If provided, this is the shift, in spatial pixels, to - apply to each slit. This is used to correct for spatial + spat_flexure (:obj:`list`, optional): + If provided, this is a list of the shifts, in spatial pixels, + to apply to each slit. This is used to correct for spatial flexure. The shape of the array should be (nslits, 2), where the first column is the shift to apply to the left edge of each slit and the second column is the From a8c69400f84d64d31bd19e4aa148e7615a8b7aa3 Mon Sep 17 00:00:00 2001 From: rcooke Date: Tue, 20 Aug 2024 11:19:55 +0100 Subject: [PATCH 21/54] update docstring --- pypeit/core/skysub.py | 10 ++++++---- pypeit/pypeit.py | 8 ++++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/pypeit/core/skysub.py b/pypeit/core/skysub.py index b2ec8c5511..8005fccc0c 100644 --- a/pypeit/core/skysub.py +++ b/pypeit/core/skysub.py @@ -1593,10 +1593,12 @@ def generate_mask(pypeline, skyreg, slits, slits_left, slits_right, spat_flexure A 2D array containing the pixel coordinates of the left slit edges slits_right : `numpy.ndarray`_ A 2D array containing the pixel coordinates of the right slit edges - resolution: int, optional - The percentage regions will be scaled to the specified resolution. The - resolution should probably correspond to the number of spatial pixels - on the slit. + spat_flexure (`numpy.ndarray`_, optional): + If provided, this is the shift, in spatial pixels, to apply to each slit. + This is used to correct for spatial flexure. The shape of the array should + be (nslits, 2), where the first column is the shift to apply to the left + edge of each slit and the second column is the shift to apply to the + right edge of each slit. Returns ------- diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index 80a0e9ec6f..01f10896a3 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -901,8 +901,12 @@ def load_skyregions(self, initial_slits=False, scifile=None, frame=None, spat_fl frame : :obj:`int`, optional The index of the frame used to construct the calibration key. Only used if ``user_regions = user``. - spat_flexure : :obj:`float`, None, optional - The spatial flexure (measured in pixels) of the science frame relative to the trace frame. + spat_flexure (`numpy.ndarray`_, optional): + If provided, this is the shift, in spatial pixels, to apply to each slit. + This is used to correct for spatial flexure. The shape of the array should + be (nslits, 2), where the first column is the shift to apply to the left + edge of each slit and the second column is the shift to apply to the + right edge of each slit. Returns ------- From e4d5ed3fbd15c1c765b63bcb57695fe4da7e9ef8 Mon Sep 17 00:00:00 2001 From: rcooke Date: Tue, 20 Aug 2024 13:04:01 +0100 Subject: [PATCH 22/54] spatial flexure for skymask --- pypeit/core/skysub.py | 10 +++++++++- pypeit/slittrace.py | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pypeit/core/skysub.py b/pypeit/core/skysub.py index 8005fccc0c..967739a8bb 100644 --- a/pypeit/core/skysub.py +++ b/pypeit/core/skysub.py @@ -1614,7 +1614,11 @@ def generate_mask(pypeline, skyreg, slits, slits_left, slits_right, spat_flexure # regardless of which slit the sky regions falls in) left_edg, righ_edg = np.zeros((slits.nspec, 0)), np.zeros((slits.nspec, 0)) spec_min, spec_max = np.array([]), np.array([]) + # Create a dummy spatial flexure that determines the spatial flexure at the edge of each sky region + skyreg_spat_flexure = np.zeros((0, 2)) for sl in range(slits.nslits): + this_spat_flexure_left = spat_flexure[sl, 0] + this_spat_flexure_diff = spat_flexure[sl, 1] - spat_flexure[sl, 0] # Calculate the slit width diff = slits_right[:, sl] - slits_left[:, sl] # Break up the slit into `resolution` subpixels @@ -1632,6 +1636,10 @@ def generate_mask(pypeline, skyreg, slits, slits_left, slits_right, spat_flexure nreg += 1 spec_min = np.append(spec_min, slits.specmin[sl]) spec_max = np.append(spec_max, slits.specmax[sl]) + # Determine the spatial flexure at the edge of the sky regions -- linearly interpolate from the left edge + flex_left = this_spat_flexure_left + this_spat_flexure_diff * wl[rr]/(resolution-1.0) + flex_right = this_spat_flexure_left + this_spat_flexure_diff * wr[rr]/(resolution-1.0) + skyreg_spat_flexure = np.append(skyreg_spat_flexure, np.array([[flex_left, flex_right]]), axis=0) # Check if no regions were added if left_edg.shape[1] == 0: @@ -1644,7 +1652,7 @@ def generate_mask(pypeline, skyreg, slits, slits_left, slits_right, spat_flexure specmax=spec_max, binspec=slits.binspec, binspat=slits.binspat, pad=0) # Generate the mask, and return - return (slitreg.slit_img(use_spatial=False, spat_flexure=spat_flexure) >= 0).astype(bool) + return (slitreg.slit_img(use_spatial=False, spat_flexure=skyreg_spat_flexure) >= 0).astype(bool) diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index f69029e734..19d1062e7a 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -685,6 +685,7 @@ def slit_img(self, pad=None, slitidx=None, initial=False, spat_flexure=None, exc spat = np.arange(self.nspat) spec = np.arange(self.nspec) + # Get the edges left, right, _ = self.select_edges(initial=initial, spat_flexure=spat_flexure) # Choose the slits to use From 6dfb0cdc47643c1884e13742d203fbee494ce6d2 Mon Sep 17 00:00:00 2001 From: rcooke Date: Tue, 20 Aug 2024 14:45:05 +0100 Subject: [PATCH 23/54] update release notes for spatial flexure --- doc/releases/1.16.1dev.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/releases/1.16.1dev.rst b/doc/releases/1.16.1dev.rst index ed857afacb..23394a75b3 100644 --- a/doc/releases/1.16.1dev.rst +++ b/doc/releases/1.16.1dev.rst @@ -23,9 +23,12 @@ Functionality/Performance Improvements and Additions This is done by setting the parameter ``use_std_trace`` in FindObjPar. - Now PypeIt can handle the case where "Standard star trace does not match the number of orders in the echelle data" both in `run_pypeit` and in `pypeit_coadd_1dspec`. +- The spatial flexure can now take a constant value for every slit, independent values for each slit, + or independent values for each slit edge. Instrument-specific Updates --------------------------- + - Added support for the (decommissioned) AAT/UHRF instrument Script Changes @@ -52,6 +55,8 @@ Script Changes Datamodel Changes ----------------- +- Spatial flexure is now stored as a 2D numpy array. + Under-the-hood Improvements --------------------------- From 0588d2e2af1e5ee8154bd1b889f69a3f036aa3ed Mon Sep 17 00:00:00 2001 From: rcooke Date: Tue, 20 Aug 2024 14:50:59 +0100 Subject: [PATCH 24/54] fix for when spat_flexure is None --- pypeit/core/skysub.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pypeit/core/skysub.py b/pypeit/core/skysub.py index 1ff67e1290..1734be1dc4 100644 --- a/pypeit/core/skysub.py +++ b/pypeit/core/skysub.py @@ -1605,6 +1605,8 @@ def generate_mask(pypeline, skyreg, slits, slits_left, slits_right, spat_flexure mask : `numpy.ndarray`_ Boolean mask containing sky regions """ + # Check the input + _spat_flexure = np.zeros((slits.nslits, 2)) if spat_flexure is None else spat_flexure # Grab the resolution that was used to generate skyreg resolution = skyreg[0].size # Using the left/right slit edge traces, generate a series of traces that mark the @@ -1617,8 +1619,8 @@ def generate_mask(pypeline, skyreg, slits, slits_left, slits_right, spat_flexure # Create a dummy spatial flexure that determines the spatial flexure at the edge of each sky region skyreg_spat_flexure = np.zeros((0, 2)) for sl in range(slits.nslits): - this_spat_flexure_left = spat_flexure[sl, 0] - this_spat_flexure_diff = spat_flexure[sl, 1] - spat_flexure[sl, 0] + this_spat_flexure_left = _spat_flexure[sl, 0] + this_spat_flexure_diff = _spat_flexure[sl, 1] - _spat_flexure[sl, 0] # Calculate the slit width diff = slits_right[:, sl] - slits_left[:, sl] # Break up the slit into `resolution` subpixels From 6d687c46f09739b56b318fc8e72c70321efcdfad Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 21 Aug 2024 13:34:07 +0100 Subject: [PATCH 25/54] fix tests --- pypeit/spectrographs/keck_kcwi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/spectrographs/keck_kcwi.py b/pypeit/spectrographs/keck_kcwi.py index c4e03be5ce..92ccac143a 100644 --- a/pypeit/spectrographs/keck_kcwi.py +++ b/pypeit/spectrographs/keck_kcwi.py @@ -317,7 +317,7 @@ def default_pypeit_par(cls): # Illumination corrections par['scienceframe']['process']['use_illumflat'] = True # illumflat is applied when building the relative scale image in reduce.py, so should be applied to scienceframe too. par['scienceframe']['process']['use_specillum'] = True # apply relative spectral illumination - par['scienceframe']['process']['spat_flexure_correct'] = "detector" # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames + par['scienceframe']['process']['spat_flexure_correct'] = "none" # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames par['scienceframe']['process']['use_biasimage'] = True # Need to use bias frames for KCWI, because the bias level varies monotonically with spatial and spectral direction par['scienceframe']['process']['use_darkimage'] = False From f2d31fd25c752426703ce64a0e590ef97032d48e Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 2 Sep 2024 22:11:05 +0100 Subject: [PATCH 26/54] update for edge specific spatial flexure --- pypeit/core/flexure.py | 24 ++++++++++++++++++------ pypeit/scripts/chk_flexure.py | 3 +-- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/pypeit/core/flexure.py b/pypeit/core/flexure.py index a3bf85a503..9a6c9f13c4 100644 --- a/pypeit/core/flexure.py +++ b/pypeit/core/flexure.py @@ -1354,7 +1354,7 @@ def sky_em_residuals(wave:np.ndarray, flux:np.ndarray, wline = [line-noff,line+noff] mw = (wave > wline[0]) & (wave < wline[1]) & good_ivar - # Reuire minimum number + # Require minimum number if np.sum(mw) <= nfit_min: continue @@ -1422,7 +1422,7 @@ def flexure_diagnostic(file, file_type='spec2d', flexure_type='spec', chk_versio If False, throw a warning only. Default is False. Returns: - :obj:`astropy.table.Table` or :obj:`float`: + :obj:`astropy.table.Table` or :obj:`float` or None: - If file_type is 'spec2d' and flexure_type is 'spec', return a table with the spectral flexure - If file_type is 'spec2d' and flexure_type is 'spat', return the spatial flexure - If file_type is 'spec1d', return a table with the spectral flexure @@ -1451,11 +1451,23 @@ def flexure_diagnostic(file, file_type='spec2d', flexure_type='spec', chk_versio return_flex = spec_flex # get and print the spatial flexure if flexure_type == 'spat': - spat_flex = allspec2D[det].sci_spat_flexure - # print the value - print(f'Spat shift: {spat_flex}') + spat_flexure = allspec2D[det].sci_spat_flexure + if np.all(spat_flexure == spat_flexure[0, 0]): + # print the value + print(f'Spat shift: {spat_flexure}') + elif np.array_equal(spat_flexure[:,0],spat_flexure[:,1]): + # print the value of each slit + for ii in range(spat_flexure.shape[0]): + print(f' Slit {ii+1} shift: {spat_flexure[ii,0]}') + else: + # print the value for the edge of each slit + for ii in range(spat_flexure.shape[0]): + print(' Slit {0:2d} --- left edge shift: {1:f}'.format(ii+1, spat_flexure[ii,0])) + print(' --- right edge shift: {0:f}'.format(spat_flexure[ii,1])) # return the value - return_flex = spat_flex + # TODO :: This return_flex is in a for loop, so the return will be the last value. + # :: Also, the return_flex is not used in the code. So, perhaps this can be removed? + return_flex = spat_flexure elif file_type == 'spec1d': # no spat flexure in spec1d file if flexure_type == 'spat': diff --git a/pypeit/scripts/chk_flexure.py b/pypeit/scripts/chk_flexure.py index d384dc8e29..55526da0fe 100644 --- a/pypeit/scripts/chk_flexure.py +++ b/pypeit/scripts/chk_flexure.py @@ -26,7 +26,6 @@ def get_parser(cls, width=None): @staticmethod def main(args): - from IPython import embed from astropy.io import fits from pypeit import msgs from pypeit.core import flexure @@ -36,7 +35,7 @@ def main(args): # Loop over the input files for in_file in args.input_file: - msgs.info(f'Checking fluxure for file: {in_file}') + msgs.info(f'Checking flexure for file: {in_file}') # What kind of file are we?? hdul = fits.open(in_file) From fe676ae004ca1d252097fe5c44a9f50960368911 Mon Sep 17 00:00:00 2001 From: rcooke Date: Mon, 2 Sep 2024 22:12:20 +0100 Subject: [PATCH 27/54] update for edge specific spatial flexure --- pypeit/core/flexure.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pypeit/core/flexure.py b/pypeit/core/flexure.py index 9a6c9f13c4..9b1151d2c5 100644 --- a/pypeit/core/flexure.py +++ b/pypeit/core/flexure.py @@ -1454,16 +1454,16 @@ def flexure_diagnostic(file, file_type='spec2d', flexure_type='spec', chk_versio spat_flexure = allspec2D[det].sci_spat_flexure if np.all(spat_flexure == spat_flexure[0, 0]): # print the value - print(f'Spat shift: {spat_flexure}') + print(f'Spatial shift: {spat_flexure}') elif np.array_equal(spat_flexure[:,0],spat_flexure[:,1]): # print the value of each slit for ii in range(spat_flexure.shape[0]): - print(f' Slit {ii+1} shift: {spat_flexure[ii,0]}') + print(f' Slit {ii+1} spatial shift: {spat_flexure[ii,0]}') else: # print the value for the edge of each slit for ii in range(spat_flexure.shape[0]): - print(' Slit {0:2d} --- left edge shift: {1:f}'.format(ii+1, spat_flexure[ii,0])) - print(' --- right edge shift: {0:f}'.format(spat_flexure[ii,1])) + print(' Slit {0:2d} -- left edge spatial shift: {1:f}'.format(ii+1, spat_flexure[ii,0])) + print(' -- right edge spatial shift: {0:f}'.format(spat_flexure[ii,1])) # return the value # TODO :: This return_flex is in a for loop, so the return will be the last value. # :: Also, the return_flex is not used in the code. So, perhaps this can be removed? From 0fe9901a01a4b2288128355e9c7a23574ca5cb99 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 4 Sep 2024 20:49:50 +0100 Subject: [PATCH 28/54] UVES_popler updates --- doc/coadd1d.rst | 7 +------ doc/releases/1.16.1dev.rst | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/doc/coadd1d.rst b/doc/coadd1d.rst index 0a84d5b736..0c4f2b8da4 100644 --- a/doc/coadd1d.rst +++ b/doc/coadd1d.rst @@ -317,12 +317,7 @@ Here is an example of a coadded spectrum using UVES_popler: UVES_popler was originally written to coadd ESO/UVES echelle spectra that were reduced by the ESO pipeline, and it has been recently modified -to support the reduction of PypeIt longslit and echelle data. If you -require some support on how to use UVES_popler with PypeIt data, please -join the `PypeIt Users Slack `__ and -ask for help in the #coadd1d channel. All are welcome to join using -`this invitation link `__. - +to support the reduction of PypeIt longslit and echelle data. For details on how to use the tool, please refer to the `UVES_popler documentation `__. To get you started with reading in PypeIt :doc:`out_spec1D` files, diff --git a/doc/releases/1.16.1dev.rst b/doc/releases/1.16.1dev.rst index 9efed234ff..cc82174157 100644 --- a/doc/releases/1.16.1dev.rst +++ b/doc/releases/1.16.1dev.rst @@ -17,7 +17,6 @@ Dependency Changes Functionality/Performance Improvements and Additions ---------------------------------------------------- -- Added support for data coaddition with the UVES_popler GUI tool. - Added the possibility to decide if the extracted standard star spectrum should be used as a crutch for tracing the object in the science frame (before it was done as default). @@ -88,6 +87,7 @@ Datamodel Changes ----------------- - Spatial flexure is now stored as a 2D numpy array. +- Added support for data coaddition with the UVES_popler GUI tool. Under-the-hood Improvements --------------------------- From 6bfa9faf23da04736b95c3701e14bfaa7b791836 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 11 Sep 2024 19:08:35 +0100 Subject: [PATCH 29/54] updated users --- presentations/py/users.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/presentations/py/users.py b/presentations/py/users.py index 73902997e4..a690f10385 100644 --- a/presentations/py/users.py +++ b/presentations/py/users.py @@ -21,9 +21,9 @@ def set_fontsize(ax, fsz): ax.get_xticklabels() + ax.get_yticklabels()): item.set_fontsize(fsz) -user_dates = ["2021-03-11", "2022-04-29", "2022-11-07", "2022-12-06", "2023-06-08", "2023-06-29", "2023-07-11", "2023-09-03", "2023-10-13", "2023-12-01", "2023-12-15", "2024-02-22", "2024-03-21", "2024-04-09", "2024-05-02", "2024-05-19", "2024-06-06", "2024-06-10", "2024-08-20"] +user_dates = ["2021-03-11", "2022-04-29", "2022-11-07", "2022-12-06", "2023-06-08", "2023-06-29", "2023-07-11", "2023-09-03", "2023-10-13", "2023-12-01", "2023-12-15", "2024-02-22", "2024-03-21", "2024-04-09", "2024-05-02", "2024-05-19", "2024-06-06", "2024-06-10", "2024-08-20", "2024-09-11"] user_dates = numpy.array([numpy.datetime64(date) for date in user_dates]) -user_number = numpy.array([125, 293, 390, 394, 477, 487, 506, 518, 531, 544, 551, 568, 579, 588, 596, 603, 616, 620, 643]) +user_number = numpy.array([125, 293, 390, 394, 477, 487, 506, 518, 531, 544, 551, 568, 579, 588, 596, 603, 616, 620, 643, 655]) user_pred_dates = numpy.array([numpy.datetime64(date) for date in ["2024-06-10", "2024-12-31", "2025-12-31", "2026-12-31", From 13daa62b0ae81fbbd283dc20d9f88b183ab90b5a Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 11 Sep 2024 21:41:29 +0100 Subject: [PATCH 30/54] corrected spatial flexure in flatfield --- deprecated/find_objects.py | 2 +- deprecated/reduce.py | 2 +- pypeit/coadd2d.py | 2 +- pypeit/coadd3d.py | 2 +- pypeit/flatfield.py | 46 ++++++++++++++++++-------------------- pypeit/slittrace.py | 8 +++---- 6 files changed, 30 insertions(+), 32 deletions(-) diff --git a/deprecated/find_objects.py b/deprecated/find_objects.py index 819ccac820..23b93749ac 100644 --- a/deprecated/find_objects.py +++ b/deprecated/find_objects.py @@ -290,7 +290,7 @@ def illum_profile_spectral(self, global_sky, skymask=None): # relative spectral sensitivity is calculated at a given wavelength for all slits simultaneously. scaleImg = flatfield.illum_profile_spectral(self.sciImg.image.copy(), self.waveimg, self.slits, slit_illum_ref_idx=sl_ref, model=global_sky, gpmask=gpm, - skymask=skymask, trim=trim, flexure=self.spat_flexure_shift, + skymask=skymask, trim=trim, spat_flexure=self.spat_flexure_shift, smooth_npix=smooth_npix) # Now apply the correction to the science frame self.apply_relative_scale(scaleImg) diff --git a/deprecated/reduce.py b/deprecated/reduce.py index fa5a803ec2..610e281575 100644 --- a/deprecated/reduce.py +++ b/deprecated/reduce.py @@ -1701,7 +1701,7 @@ def illum_profile_spectral(self, global_sky, skymask=None): gpm = self.sciImg.select_flag(invert=True) scaleImg = flatfield.illum_profile_spectral(self.sciImg.image.copy(), self.waveimg, self.slits, slit_illum_ref_idx=ref_idx, model=global_sky, gpmask=gpm, - skymask=skymask, trim=trim, flexure=self.spat_flexure_shift) + skymask=skymask, trim=trim, spat_flexure=self.spat_flexure_shift) # Now apply the correction to the science frame self.apply_relative_scale(scaleImg) diff --git a/pypeit/coadd2d.py b/pypeit/coadd2d.py index 9f7ceaf5b6..42941f22f5 100644 --- a/pypeit/coadd2d.py +++ b/pypeit/coadd2d.py @@ -838,7 +838,7 @@ def reduce(self, pseudo_dict, show=False, clear_ginga=True, show_peaks=False, sh platescale = sciImage.detector.platescale * self.spat_samp_fact # Assign slitmask design information to detected objects - slits.assign_maskinfo(sobjs_obj, platescale, None, TOLER=parcopy['reduce']['slitmask']['obj_toler']) + slits.assign_maskinfo(sobjs_obj, platescale, None, tolerance=parcopy['reduce']['slitmask']['obj_toler']) if parcopy['reduce']['slitmask']['extract_missing_objs'] is True: # Set the FWHM for the extraction of missing objects diff --git a/pypeit/coadd3d.py b/pypeit/coadd3d.py index 6473aa6aa6..3a7aefe88b 100644 --- a/pypeit/coadd3d.py +++ b/pypeit/coadd3d.py @@ -846,7 +846,7 @@ def add_grating_corr(self, flatfile, waveimg, slits, spat_flexure=None): scale_model = flatfield.illum_profile_spectral(flatframe, waveimg, slits, slit_illum_ref_idx=self.flatpar['slit_illum_ref_idx'], model=None, trim=self.flatpar['slit_trim'], - flexure=spat_flexure, + spat_flexure=spat_flexure, smooth_npix=self.flatpar['slit_illum_smooth_npix']) else: msgs.info("Using relative spectral illumination from FlatImages") diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index 709f1a1cb1..dc0edfb4e1 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -856,10 +856,6 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): npoly = self.flatpar['twod_fit_npoly'] saturated_slits = self.flatpar['saturated_slits'] - # Build wavelength image -- not always used, but for convenience done here - # TODO :: This was deleted in a recent PR -- figure out why? I think it was moved to the init method - if self.waveimg is None: self.build_waveimg() - # Setup images nspec, nspat = self.rawflatimg.image.shape rawflat = self.rawflatimg.image @@ -1487,8 +1483,8 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, onslit_tweak_trim = self.slits.slit_img(pad=-slit_trim, slitidx=slit_idx, initial=False) == slit_spat # Setup slitimg = (slit_spat + 1) * onslit_tweak.astype(int) - 1 # Need to +1 and -1 so that slitimg=-1 when off the slit - # TODO :: need to fix the 0.0 value on the next line, and the same one a few lines below. - left, right, msk = self.slits.select_edges(spat_flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0) + spat_flexure = self.wavetilts.spat_flexure if self.wavetilts is not None else np.zeros((self.slits.nslits, 2)) + left, right, msk = self.slits.select_edges(spat_flexure=spat_flexure) this_left = left[:, slit_idx] this_right = right[:, slit_idx] slitlen = int(np.median(this_right - this_left)) @@ -1498,7 +1494,7 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, this_wave = self.waveimg[this_slit] xpos_img = self.slits.spatial_coordinate_image(slitidx=slit_idx, slitid_img=slitimg, - spat_flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0) + spat_flexure=spat_flexure) # Generate the trimmed versions for fitting this_slit_trim = np.where(onslit_tweak_trim & self.rawflatimg.select_flag(invert=True)) this_wave_trim = self.waveimg[this_slit_trim] @@ -1595,11 +1591,11 @@ def extract_structure(self, rawflat_orig, slit_trim=3): # Now fit the spectral profile # TODO: Should this be *any* flag, or just BPM? gpm = self.rawflatimg.select_flag(flag='BPM', invert=True) - # TODO :: Need to fix the flexure 0.0 value below, and also rename this argument to spat_flexure for consistency + spat_flexure = self.wavetilts.spat_flexure if self.wavetilts is not None else np.zeros((self.slits.nslits, 2)) scale_model = illum_profile_spectral(rawflat, self.waveimg, self.slits, slit_illum_ref_idx=self.flatpar['slit_illum_ref_idx'], model=None, gpmask=gpm, skymask=None, trim=self.flatpar['slit_trim'], - flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0, + spat_flexure=spat_flexure, smooth_npix=self.flatpar['slit_illum_smooth_npix']) # Trim the edges by a few pixels to avoid edge effects onslits_trim = gpm & (self.slits.slit_img(pad=-slit_trim, initial=False) != -1) @@ -1648,8 +1644,7 @@ def spectral_illumination(self, gpm=None, debug=False): # check if the waveimg is available if self.waveimg is None: msgs.warn("Cannot perform the spectral illumination without the wavelength image.") - # TODO :: Should this really be None? Probably it should be ones. - return None + return np.ones_like(self.rawflatimg.image) msgs.info('Performing a joint fit to the flat-field response') # Grab some parameters trim = self.flatpar['slit_trim'] @@ -1660,13 +1655,11 @@ def spectral_illumination(self, gpm=None, debug=False): gpm = self.rawflatimg.select_flag(flag='BPM', invert=True) # Obtain relative spectral illumination - # TODO :: fix the 0.0 value for the flexure argument. Also, rename flexure to spat_flexure for consistency + spat_flexure = self.wavetilts.spat_flexure if self.wavetilts is not None else np.zeros((self.slits.nslits, 2)) return illum_profile_spectral(rawflat, self.waveimg, self.slits, slit_illum_ref_idx=self.flatpar['slit_illum_ref_idx'], - model=None, gpmask=gpm, skymask=None, trim=trim, - flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0, - smooth_npix=self.flatpar['slit_illum_smooth_npix'], - debug=debug) + model=None, gpmask=gpm, skymask=None, trim=trim, spat_flexure=spat_flexure, + smooth_npix=self.flatpar['slit_illum_smooth_npix'], debug=debug) def tweak_slit_edges(self, left, right, spat_coo, norm_flat, method='threshold', thresh=0.93, maxfrac=0.1, debug=False): @@ -2164,7 +2157,7 @@ def show_flats(image_list, wcs_match=True, slits=None, waveimg=None): # TODO :: This could possibly be moved to core.flat def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_npix=None, polydeg=None, - model=None, gpmask=None, skymask=None, trim=3, flexure=None, maxiter=5, debug=False): + model=None, gpmask=None, skymask=None, trim=3, spat_flexure=None, maxiter=5, debug=False): """ Determine the relative spectral illumination of all slits. Currently only used for image slicer IFUs. @@ -2184,17 +2177,22 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ polydeg : int, optional Degree of polynomial to be used for determining relative spectral sensitivity. If None, coadd.smooth_weights will be used, with the smoothing length set to smooth_npix. - model : `numpy.ndarray`_, None + model : `numpy.ndarray`_, optional A model of the rawimg data. If None, rawimg will be used. - gpmask : `numpy.ndarray`_, None + gpmask : `numpy.ndarray`_, optional Good pixel mask - skymask : `numpy.ndarray`_, None + skymask : `numpy.ndarray`_, optional Sky mask trim : int Number of pixels to trim from the edges of the slit when deriving the spectral illumination - flexure : float, None - Spatial flexure + spat_flexure : `numpy.ndarray`_, optional + If provided, this is the shift, in spatial pixels, to + apply to each slit. This is used to correct for spatial + flexure. The shape of the array should be (nslits, 2), + where the first column is the shift to apply to the + left edge of each slit and the second column is the + shift to apply to the right edge of each slit. maxiter : :obj:`int` Maximum number of iterations to perform debug : :obj:`bool` @@ -2215,8 +2213,8 @@ def illum_profile_spectral(rawimg, waveimg, slits, slit_illum_ref_idx=0, smooth_ gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool) modelimg = model if (model is not None) else rawimg.copy() # Setup the slits - slitid_img = slits.slit_img(pad=0, spat_flexure=flexure) - slitid_img_trim = slits.slit_img(pad=-trim, spat_flexure=flexure) + slitid_img = slits.slit_img(pad=0, spat_flexure=spat_flexure) + slitid_img_trim = slits.slit_img(pad=-trim, spat_flexure=spat_flexure) scaleImg = np.ones_like(rawimg) modelimg_copy = modelimg.copy() # Obtain the minimum and maximum wavelength of all slits diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index d36d38444e..d01821c0d9 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -1011,7 +1011,7 @@ def mask_add_missing_obj(self, sobjs, spat_flexure, fwhm, boxcar_rad): # Return return sobjs - def assign_maskinfo(self, sobjs, plate_scale, spat_flexure, TOLER=1.): + def assign_maskinfo(self, sobjs, plate_scale, spat_flexure, tolerance=1.): """ Assign RA, DEC, Name to objects. Modified in place. @@ -1030,7 +1030,7 @@ def assign_maskinfo(self, sobjs, plate_scale, spat_flexure, TOLER=1.): shift to apply to the right edge of each slit. det_buffer (:obj:`int`): Minimum separation between detector edges and a slit edge. - TOLER (:obj:`float`, optional): + tolerance (:obj:`float`, optional): Matching tolerance in arcsec. Returns: @@ -1124,7 +1124,7 @@ def assign_maskinfo(self, sobjs, plate_scale, spat_flexure, TOLER=1.): obj_fwhm = cut_sobjs[ipeak].FWHM*plate_scale else: obj_fwhm = 0. - in_toler = np.abs(separ*plate_scale) < (TOLER + cc_rms + obj_fwhm/2) + in_toler = np.abs(separ*plate_scale) < (tolerance + cc_rms + obj_fwhm / 2) if np.any(in_toler): # Find positive peakflux peak_flux = cut_sobjs[idx].smash_peakflux[in_toler] @@ -1780,7 +1780,7 @@ def assign_addobjs_alldets(sobjs, calib_slits, spat_flexure, platescale, slitmas if calib_slits[i].maskdef_designtab is not None: # Assign slitmask design information to detected objects sobjs = calib_slits[i].assign_maskinfo(sobjs, platescale[i], spat_flexure[i], - TOLER=slitmask_par['obj_toler']) + tolerance=slitmask_par['obj_toler']) if slitmask_par['extract_missing_objs']: # Set the FWHM for the extraction of missing objects From e4f42d1080645fc211c27c2f3371f342557d5175 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Thu, 19 Sep 2024 17:31:55 -0700 Subject: [PATCH 31/54] doc update --- doc/coadd1d.rst | 2 +- doc/help/pypeit_sensfunc.rst | 26 +++++------ doc/include/class_datamodel_pypeitimage.rst | 42 ++++++++--------- doc/include/class_datamodel_spec2dobj.rst | 52 ++++++++++----------- doc/include/class_datamodel_specobj.rst | 4 +- doc/include/class_datamodel_wavetilts.rst | 34 +++++++------- doc/include/datamodel_spec2dobj.rst | 52 ++++++++++----------- doc/include/datamodel_specobj.rst | 4 +- doc/include/datamodel_wavetilts.rst | 2 +- doc/include/imgproc_defaults_table.rst | 2 +- doc/include/spectrographs_table.rst | 2 +- doc/pypeit_par.rst | 24 +++++----- pypeit/core/skysub.py | 2 +- 13 files changed, 126 insertions(+), 122 deletions(-) diff --git a/doc/coadd1d.rst b/doc/coadd1d.rst index 0c4f2b8da4..6520efe6aa 100644 --- a/doc/coadd1d.rst +++ b/doc/coadd1d.rst @@ -312,7 +312,7 @@ This tool is developed by Michael Murphy and is available at `this link `__. Here is an example of a coadded spectrum using UVES_popler: -.. image:: ../figures/uves_popler.png +.. image:: figures/uves_popler.png :scale: 60% UVES_popler was originally written to coadd ESO/UVES echelle spectra diff --git a/doc/help/pypeit_sensfunc.rst b/doc/help/pypeit_sensfunc.rst index 701944f327..26ac6639bb 100644 --- a/doc/help/pypeit_sensfunc.rst +++ b/doc/help/pypeit_sensfunc.rst @@ -2,7 +2,7 @@ $ pypeit_sensfunc -h usage: pypeit_sensfunc [-h] [--extr {OPT,BOX}] [--algorithm {UVIS,IR}] - [--multi MULTI] [-o OUTFILE] [-s SENS_FILE] [-f FLATFILE] + [--multi MULTI] [-o OUTFILE] [-s SENS_FILE] [-f] [--debug] [--par_outfile PAR_OUTFILE] [-v VERBOSITY] spec1dfile @@ -71,20 +71,20 @@ in the filename. -s SENS_FILE, --sens_file SENS_FILE Configuration file with sensitivity function parameters - -f FLATFILE, --flatfile FLATFILE - Use a flat calibration file to compute the blaze - function when generating the sensitivity function. This - is helpful to account for small scale undulations in the - sensitivity function. Note that it is not possible to - set --flatfile and simultaneously use a .sens file with - the --sens_file option. If you are using a .sens file, - set the flatfile there via e.g.: + -f, --use_flat Use the extracted spectrum of the flatfield calibration + to estimate the blaze function when generating the + sensitivity function. This is helpful to account for + small scale undulations in the sensitivity function. The + spec1dfile must contain the extracted flatfield response + in order to use this option. This spectrum is extracted + by default, unless you did not compute a pixelflat + frame. Note that it is not possible to set --use_flat + and simultaneously use a .sens file with the --sens_file + option. If you are using a .sens file, set the use_flat + flag with the argument: [sensfunc] - flatfile = Calibrations/Flat_A_0_DET01.fits - - Where Flat_A_0_DET01.fits is the flat file in your - Calibrations directory + use_flat = True --debug show debug plots? --par_outfile PAR_OUTFILE Name of output file to save the parameters used by the diff --git a/doc/include/class_datamodel_pypeitimage.rst b/doc/include/class_datamodel_pypeitimage.rst index 9008ab7bc3..be505a3a30 100644 --- a/doc/include/class_datamodel_pypeitimage.rst +++ b/doc/include/class_datamodel_pypeitimage.rst @@ -1,24 +1,24 @@ **Version**: 1.3.0 -================ =================================================================================================== ================= ======================================================================================================================================================================================== -Attribute Type Array Type Description -================ =================================================================================================== ================= ======================================================================================================================================================================================== -``PYP_SPEC`` str PypeIt spectrograph name -``amp_img`` `numpy.ndarray`_ `numpy.integer`_ Provides the amplifier that contributed to each pixel. If this is a detector mosaic, this must be used in combination with ``det_img`` to select pixels for a given detector amplifier. -``base_var`` `numpy.ndarray`_ `numpy.floating`_ Base-level image variance, excluding count shot-noise -``det_img`` `numpy.ndarray`_ `numpy.integer`_ If a detector mosaic, this image provides the detector that contributed to each pixel. -``detector`` :class:`~pypeit.images.detector_container.DetectorContainer`, :class:`~pypeit.images.mosaic.Mosaic` The detector (see :class:`~pypeit.images.detector_container.DetectorContainer`) or mosaic (see :class:`~pypeit.images.mosaic.Mosaic`) parameters -``exptime`` int, float Effective exposure time (s) -``filename`` str Filename for the image -``fullmask`` :class:`~pypeit.images.imagebitmask.ImageBitMaskArray` Image mask -``image`` `numpy.ndarray`_ `numpy.floating`_ Primary image data -``img_scale`` `numpy.ndarray`_ `numpy.floating`_ Image count scaling applied (e.g., 1/flat-field) -``ivar`` `numpy.ndarray`_ `numpy.floating`_ Inverse variance image -``nimg`` `numpy.ndarray`_ `numpy.integer`_ If a combination of multiple images, this is the number of images that contributed to each pixel -``noise_floor`` float Noise floor included in variance -``rn2img`` `numpy.ndarray`_ `numpy.floating`_ Read noise squared image -``shot_noise`` bool Shot-noise included in variance -``spat_flexure`` float Shift, in spatial pixels, between this image and SlitTrace -``units`` str (Unscaled) Pixel units (e- or ADU) -================ =================================================================================================== ================= ======================================================================================================================================================================================== +================ =================================================================================================== ================= ================================================================================================================================================================================================================================ +Attribute Type Array Type Description +================ =================================================================================================== ================= ================================================================================================================================================================================================================================ +``PYP_SPEC`` str PypeIt spectrograph name +``amp_img`` `numpy.ndarray`_ `numpy.integer`_ Provides the amplifier that contributed to each pixel. If this is a detector mosaic, this must be used in combination with ``det_img`` to select pixels for a given detector amplifier. +``base_var`` `numpy.ndarray`_ `numpy.floating`_ Base-level image variance, excluding count shot-noise +``det_img`` `numpy.ndarray`_ `numpy.integer`_ If a detector mosaic, this image provides the detector that contributed to each pixel. +``detector`` :class:`~pypeit.images.detector_container.DetectorContainer`, :class:`~pypeit.images.mosaic.Mosaic` The detector (see :class:`~pypeit.images.detector_container.DetectorContainer`) or mosaic (see :class:`~pypeit.images.mosaic.Mosaic`) parameters +``exptime`` int, float Effective exposure time (s) +``filename`` str Filename for the image +``fullmask`` :class:`~pypeit.images.imagebitmask.ImageBitMaskArray` Image mask +``image`` `numpy.ndarray`_ `numpy.floating`_ Primary image data +``img_scale`` `numpy.ndarray`_ `numpy.floating`_ Image count scaling applied (e.g., 1/flat-field) +``ivar`` `numpy.ndarray`_ `numpy.floating`_ Inverse variance image +``nimg`` `numpy.ndarray`_ `numpy.integer`_ If a combination of multiple images, this is the number of images that contributed to each pixel +``noise_floor`` float Noise floor included in variance +``rn2img`` `numpy.ndarray`_ `numpy.floating`_ Read noise squared image +``shot_noise`` bool Shot-noise included in variance +``spat_flexure`` `numpy.ndarray`_ `numpy.floating`_ Shift, in spatial pixels, between this image and SlitTrace. Shape is (nslits, 2), wherespat_flexure[i,0] is the spatial shift of the left edge of slit i and spat_flexure[i,1] is the spatial shift of the right edge of slit i. +``units`` str (Unscaled) Pixel units (e- or ADU) +================ =================================================================================================== ================= ================================================================================================================================================================================================================================ diff --git a/doc/include/class_datamodel_spec2dobj.rst b/doc/include/class_datamodel_spec2dobj.rst index b68107e926..c0803b9f40 100644 --- a/doc/include/class_datamodel_spec2dobj.rst +++ b/doc/include/class_datamodel_spec2dobj.rst @@ -1,28 +1,28 @@ -**Version**: 1.1.1 +**Version**: 1.1.2 -====================== =================================================================================================== ================= ================================================================================================================================================================================ -Attribute Type Array Type Description -====================== =================================================================================================== ================= ================================================================================================================================================================================ -``bkg_redux_skymodel`` `numpy.ndarray`_ `numpy.floating`_ 2D sky model image without the background subtraction (float32) -``bpmmask`` :class:`~pypeit.images.imagebitmask.ImageBitMaskArray` 2D bad-pixel mask for the image -``det`` int Detector index -``detector`` :class:`~pypeit.images.detector_container.DetectorContainer`, :class:`~pypeit.images.mosaic.Mosaic` Detector or Mosaic metadata -``ivarmodel`` `numpy.ndarray`_ `numpy.floating`_ 2D ivar model image (float32) -``ivarraw`` `numpy.ndarray`_ `numpy.floating`_ 2D processed inverse variance image (float32) -``maskdef_designtab`` `astropy.table.table.Table`_ Table with slitmask design and object info -``med_chis`` `numpy.ndarray`_ `numpy.floating`_ Median of the chi image for each slit/order -``objmodel`` `numpy.ndarray`_ `numpy.floating`_ 2D object model image (float32) -``scaleimg`` `numpy.ndarray`_ `numpy.floating`_ 2D multiplicative scale image [or a single scalar as an array] that has been applied to the science image (float32) -``sci_spat_flexure`` float Shift, in spatial pixels, between this image and SlitTrace -``sci_spec_flexure`` `astropy.table.table.Table`_ Global shift of the spectrum to correct for spectralflexure (pixels). This is based on the sky spectrum atthe center of each slit -``sciimg`` `numpy.ndarray`_ `numpy.floating`_ 2D processed science image (float32) -``skymodel`` `numpy.ndarray`_ `numpy.floating`_ 2D sky model image (float32) -``slits`` :class:`~pypeit.slittrace.SlitTraceSet` SlitTraceSet defining the slits -``std_chis`` `numpy.ndarray`_ `numpy.floating`_ std of the chi image for each slit/order -``tilts`` `numpy.ndarray`_ `numpy.floating`_ 2D tilts image (float64) -``vel_corr`` float Relativistic velocity correction for wavelengths -``vel_type`` str Type of reference frame correction (if any). Options are listed in the routine: WavelengthSolutionPar.valid_reference_frames() Current list: observed, heliocentric, barycentric -``waveimg`` `numpy.ndarray`_ `numpy.floating`_ 2D wavelength image in vacuum (float64) -``wavesol`` `astropy.table.table.Table`_ Table with WaveCalib diagnostic info -====================== =================================================================================================== ================= ================================================================================================================================================================================ +====================== =================================================================================================== ================= ================================================================================================================================================================================================================================= +Attribute Type Array Type Description +====================== =================================================================================================== ================= ================================================================================================================================================================================================================================= +``bkg_redux_skymodel`` `numpy.ndarray`_ `numpy.floating`_ 2D sky model image without the background subtraction (float32) +``bpmmask`` :class:`~pypeit.images.imagebitmask.ImageBitMaskArray` 2D bad-pixel mask for the image +``det`` int Detector index +``detector`` :class:`~pypeit.images.detector_container.DetectorContainer`, :class:`~pypeit.images.mosaic.Mosaic` Detector or Mosaic metadata +``ivarmodel`` `numpy.ndarray`_ `numpy.floating`_ 2D ivar model image (float32) +``ivarraw`` `numpy.ndarray`_ `numpy.floating`_ 2D processed inverse variance image (float32) +``maskdef_designtab`` `astropy.table.table.Table`_ Table with slitmask design and object info +``med_chis`` `numpy.ndarray`_ `numpy.floating`_ Median of the chi image for each slit/order +``objmodel`` `numpy.ndarray`_ `numpy.floating`_ 2D object model image (float32) +``scaleimg`` `numpy.ndarray`_ `numpy.floating`_ 2D multiplicative scale image [or a single scalar as an array] that has been applied to the science image (float32) +``sci_spat_flexure`` `numpy.ndarray`_ `numpy.floating`_ Shift, in spatial pixels, between this image and SlitTrace. Shape is (nslits, 2), where spat_flexure[i,0] is the spatial shift of the left edge of slit i and spat_flexure[i,1] is the spatial shift of the right edge of slit i. +``sci_spec_flexure`` `astropy.table.table.Table`_ Global shift of the spectrum to correct for spectral flexure (pixels). This is based on the sky spectrum at the center of each slit +``sciimg`` `numpy.ndarray`_ `numpy.floating`_ 2D processed science image (float32) +``skymodel`` `numpy.ndarray`_ `numpy.floating`_ 2D sky model image (float32) +``slits`` :class:`~pypeit.slittrace.SlitTraceSet` SlitTraceSet defining the slits +``std_chis`` `numpy.ndarray`_ `numpy.floating`_ std of the chi image for each slit/order +``tilts`` `numpy.ndarray`_ `numpy.floating`_ 2D tilts image (float64) +``vel_corr`` float Relativistic velocity correction for wavelengths +``vel_type`` str Type of reference frame correction (if any). Options are listed in the routine: WavelengthSolutionPar.valid_reference_frames() Current list: observed, heliocentric, barycentric +``waveimg`` `numpy.ndarray`_ `numpy.floating`_ 2D wavelength image in vacuum (float64) +``wavesol`` `astropy.table.table.Table`_ Table with WaveCalib diagnostic info +====================== =================================================================================================== ================= ================================================================================================================================================================================================================================= diff --git a/doc/include/class_datamodel_specobj.rst b/doc/include/class_datamodel_specobj.rst index 11492ddf32..9c6afca3d2 100644 --- a/doc/include/class_datamodel_specobj.rst +++ b/doc/include/class_datamodel_specobj.rst @@ -1,5 +1,5 @@ -**Version**: 1.1.10 +**Version**: 1.1.11 ======================= =================================================================================================== ===================== ==================================================================================================================================================================================== Attribute Type Array Type Description @@ -14,6 +14,7 @@ Attribute Type ``BOX_FLAM`` `numpy.ndarray`_ float Boxcar flux (erg/s/cm^2/Ang) ``BOX_FLAM_IVAR`` `numpy.ndarray`_ float Boxcar flux inverse variance (1e-17 erg/s/cm^2/Ang)^-2 ``BOX_FLAM_SIG`` `numpy.ndarray`_ float Boxcar flux uncertainty (1e-17 erg/s/cm^2/Ang) +``BOX_FLAT`` `numpy.ndarray`_ float Boxcar extracted flatfield spectrum, normalized to the peak value. ``BOX_FRAC_USE`` `numpy.ndarray`_ float Fraction of pixels in the object profile subimage used for this extraction ``BOX_FWHM`` `numpy.ndarray`_ float Spectral FWHM (in Angstroms) at every pixel of the boxcar extracted flux. ``BOX_MASK`` `numpy.ndarray`_ `numpy.bool`_ Mask for boxcar extracted flux. True=good @@ -51,6 +52,7 @@ Attribute Type ``OPT_FLAM`` `numpy.ndarray`_ float Optimal flux (1e-17 erg/s/cm^2/Ang) ``OPT_FLAM_IVAR`` `numpy.ndarray`_ float Optimal flux inverse variance (1e-17 erg/s/cm^2/Ang)^-2 ``OPT_FLAM_SIG`` `numpy.ndarray`_ float Optimal flux uncertainty (1e-17 erg/s/cm^2/Ang) +``OPT_FLAT`` `numpy.ndarray`_ float Optimally extracted flatfield spectrum, normalised to the peak value. ``OPT_FRAC_USE`` `numpy.ndarray`_ float Fraction of pixels in the object profile subimage used for this extraction ``OPT_FWHM`` `numpy.ndarray`_ float Spectral FWHM (in Angstroms) at every pixel of the optimally extracted flux. ``OPT_MASK`` `numpy.ndarray`_ `numpy.bool`_ Mask for optimally extracted flux. True=good diff --git a/doc/include/class_datamodel_wavetilts.rst b/doc/include/class_datamodel_wavetilts.rst index f071e3f17d..ada33ea306 100644 --- a/doc/include/class_datamodel_wavetilts.rst +++ b/doc/include/class_datamodel_wavetilts.rst @@ -1,19 +1,19 @@ -**Version**: 1.2.0 +**Version**: 1.2.1 -==================== ============================ ================= =============================================================================================================================================================== -Attribute Type Array Type Description -==================== ============================ ================= =============================================================================================================================================================== -``PYP_SPEC`` str PypeIt spectrograph name -``bpmtilts`` `numpy.ndarray`_ `numpy.integer`_ Bad pixel mask for tilt solutions. Keys are taken from SlitTraceSetBitmask -``coeffs`` `numpy.ndarray`_ `numpy.floating`_ 2D coefficents for the fit on the initial slits. One set per slit/order (3D array). -``func2d`` str Function used for the 2D fit -``nslit`` int Total number of slits. This can include masked slits -``slits_filename`` str Path to SlitTraceSet file. This helps to find the Slits calibration file when running pypeit_chk_tilts() -``spat_flexure`` float Flexure shift from the input TiltImage -``spat_id`` `numpy.ndarray`_ `numpy.integer`_ Slit spat_id -``spat_order`` `numpy.ndarray`_ `numpy.integer`_ Order for spatial fit (nslit) -``spec_order`` `numpy.ndarray`_ `numpy.integer`_ Order for spectral fit (nslit) -``tilt_traces`` `astropy.table.table.Table`_ Table with the positions of the traced and fitted tilts for all the slits. see :func:`~pypeit.wavetilts.BuildWaveTilts.make_tbl_tilt_traces` for more details. -``tiltimg_filename`` str Path to Tiltimg file. This helps to find Tiltimg file when running pypeit_chk_tilts() -==================== ============================ ================= =============================================================================================================================================================== +==================== ============================ ================= ================================================================================================================================================================================================================================================ +Attribute Type Array Type Description +==================== ============================ ================= ================================================================================================================================================================================================================================================ +``PYP_SPEC`` str PypeIt spectrograph name +``bpmtilts`` `numpy.ndarray`_ `numpy.integer`_ Bad pixel mask for tilt solutions. Keys are taken from SlitTraceSetBitmask +``coeffs`` `numpy.ndarray`_ `numpy.floating`_ 2D coefficents for the fit on the initial slits. One set per slit/order (3D array). +``func2d`` str Function used for the 2D fit +``nslit`` int Total number of slits. This can include masked slits +``slits_filename`` str Path to SlitTraceSet file. This helps to find the Slits calibration file when running pypeit_chk_tilts() +``spat_flexure`` `numpy.ndarray`_ `numpy.floating`_ Spatial flexure shift, in spatial pixels, between TiltImage and SlitTrace. Shape is (nslits, 2), where spat_flexure[i,0] is the spatial shift of the left edge of slit i and spat_flexure[i,1] is the spatial shift of the right edge of slit i. +``spat_id`` `numpy.ndarray`_ `numpy.integer`_ Slit spat_id +``spat_order`` `numpy.ndarray`_ `numpy.integer`_ Order for spatial fit (nslit) +``spec_order`` `numpy.ndarray`_ `numpy.integer`_ Order for spectral fit (nslit) +``tilt_traces`` `astropy.table.table.Table`_ Table with the positions of the traced and fitted tilts for all the slits. see :func:`~pypeit.wavetilts.BuildWaveTilts.make_tbl_tilt_traces` for more details. +``tiltimg_filename`` str Path to Tiltimg file. This helps to find Tiltimg file when running pypeit_chk_tilts() +==================== ============================ ================= ================================================================================================================================================================================================================================================ diff --git a/doc/include/datamodel_spec2dobj.rst b/doc/include/datamodel_spec2dobj.rst index 01417f2214..02ce84d137 100644 --- a/doc/include/datamodel_spec2dobj.rst +++ b/doc/include/datamodel_spec2dobj.rst @@ -1,29 +1,29 @@ -Version: 1.1.1 +Version: 1.1.2 -====================== ========================= ========== ================================================================================================================================================================================ -Obj Key Obj Type Array Type Description -====================== ========================= ========== ================================================================================================================================================================================ -``bkg_redux_skymodel`` ndarray floating 2D sky model image without the background subtraction (float32) -``bpmmask`` ImageBitMaskArray 2D bad-pixel mask for the image -``det`` int Detector index -``detector`` DetectorContainer, Mosaic Detector or Mosaic metadata -``ivarmodel`` ndarray floating 2D ivar model image (float32) -``ivarraw`` ndarray floating 2D processed inverse variance image (float32) -``maskdef_designtab`` Table Table with slitmask design and object info -``med_chis`` ndarray floating Median of the chi image for each slit/order -``objmodel`` ndarray floating 2D object model image (float32) -``scaleimg`` ndarray floating 2D multiplicative scale image [or a single scalar as an array] that has been applied to the science image (float32) -``sci_spat_flexure`` float Shift, in spatial pixels, between this image and SlitTrace -``sci_spec_flexure`` Table Global shift of the spectrum to correct for spectralflexure (pixels). This is based on the sky spectrum atthe center of each slit -``sciimg`` ndarray floating 2D processed science image (float32) -``skymodel`` ndarray floating 2D sky model image (float32) -``slits`` SlitTraceSet SlitTraceSet defining the slits -``std_chis`` ndarray floating std of the chi image for each slit/order -``tilts`` ndarray floating 2D tilts image (float64) -``vel_corr`` float Relativistic velocity correction for wavelengths -``vel_type`` str Type of reference frame correction (if any). Options are listed in the routine: WavelengthSolutionPar.valid_reference_frames() Current list: observed, heliocentric, barycentric -``waveimg`` ndarray floating 2D wavelength image in vacuum (float64) -``wavesol`` Table Table with WaveCalib diagnostic info -====================== ========================= ========== ================================================================================================================================================================================ +====================== ========================= ========== ================================================================================================================================================================================================================================= +Obj Key Obj Type Array Type Description +====================== ========================= ========== ================================================================================================================================================================================================================================= +``bkg_redux_skymodel`` ndarray floating 2D sky model image without the background subtraction (float32) +``bpmmask`` ImageBitMaskArray 2D bad-pixel mask for the image +``det`` int Detector index +``detector`` DetectorContainer, Mosaic Detector or Mosaic metadata +``ivarmodel`` ndarray floating 2D ivar model image (float32) +``ivarraw`` ndarray floating 2D processed inverse variance image (float32) +``maskdef_designtab`` Table Table with slitmask design and object info +``med_chis`` ndarray floating Median of the chi image for each slit/order +``objmodel`` ndarray floating 2D object model image (float32) +``scaleimg`` ndarray floating 2D multiplicative scale image [or a single scalar as an array] that has been applied to the science image (float32) +``sci_spat_flexure`` ndarray floating Shift, in spatial pixels, between this image and SlitTrace. Shape is (nslits, 2), where spat_flexure[i,0] is the spatial shift of the left edge of slit i and spat_flexure[i,1] is the spatial shift of the right edge of slit i. +``sci_spec_flexure`` Table Global shift of the spectrum to correct for spectral flexure (pixels). This is based on the sky spectrum at the center of each slit +``sciimg`` ndarray floating 2D processed science image (float32) +``skymodel`` ndarray floating 2D sky model image (float32) +``slits`` SlitTraceSet SlitTraceSet defining the slits +``std_chis`` ndarray floating std of the chi image for each slit/order +``tilts`` ndarray floating 2D tilts image (float64) +``vel_corr`` float Relativistic velocity correction for wavelengths +``vel_type`` str Type of reference frame correction (if any). Options are listed in the routine: WavelengthSolutionPar.valid_reference_frames() Current list: observed, heliocentric, barycentric +``waveimg`` ndarray floating 2D wavelength image in vacuum (float64) +``wavesol`` Table Table with WaveCalib diagnostic info +====================== ========================= ========== ================================================================================================================================================================================================================================= diff --git a/doc/include/datamodel_specobj.rst b/doc/include/datamodel_specobj.rst index deb7cb2c19..7de4f07a1a 100644 --- a/doc/include/datamodel_specobj.rst +++ b/doc/include/datamodel_specobj.rst @@ -1,6 +1,6 @@ -Version: 1.1.10 +Version: 1.1.11 ======================= ========================= ================= ==================================================================================================================================================================================== Obj Key Obj Type Array Type Description @@ -15,6 +15,7 @@ Obj Key Obj Type Array Type Descripti ``BOX_FLAM`` ndarray float Boxcar flux (erg/s/cm^2/Ang) ``BOX_FLAM_IVAR`` ndarray float Boxcar flux inverse variance (1e-17 erg/s/cm^2/Ang)^-2 ``BOX_FLAM_SIG`` ndarray float Boxcar flux uncertainty (1e-17 erg/s/cm^2/Ang) +``BOX_FLAT`` ndarray float Boxcar extracted flatfield spectrum, normalized to the peak value. ``BOX_FRAC_USE`` ndarray float Fraction of pixels in the object profile subimage used for this extraction ``BOX_FWHM`` ndarray float Spectral FWHM (in Angstroms) at every pixel of the boxcar extracted flux. ``BOX_MASK`` ndarray bool Mask for boxcar extracted flux. True=good @@ -52,6 +53,7 @@ Obj Key Obj Type Array Type Descripti ``OPT_FLAM`` ndarray float Optimal flux (1e-17 erg/s/cm^2/Ang) ``OPT_FLAM_IVAR`` ndarray float Optimal flux inverse variance (1e-17 erg/s/cm^2/Ang)^-2 ``OPT_FLAM_SIG`` ndarray float Optimal flux uncertainty (1e-17 erg/s/cm^2/Ang) +``OPT_FLAT`` ndarray float Optimally extracted flatfield spectrum, normalised to the peak value. ``OPT_FRAC_USE`` ndarray float Fraction of pixels in the object profile subimage used for this extraction ``OPT_FWHM`` ndarray float Spectral FWHM (in Angstroms) at every pixel of the optimally extracted flux. ``OPT_MASK`` ndarray bool Mask for optimally extracted flux. True=good diff --git a/doc/include/datamodel_wavetilts.rst b/doc/include/datamodel_wavetilts.rst index 6e9f122e53..4a95b30aa0 100644 --- a/doc/include/datamodel_wavetilts.rst +++ b/doc/include/datamodel_wavetilts.rst @@ -1,5 +1,5 @@ -Version 1.2.0 +Version 1.2.1 =========== ============================== ========= ===================================================== HDU Name HDU Type Data Type Description diff --git a/doc/include/imgproc_defaults_table.rst b/doc/include/imgproc_defaults_table.rst index 2fc2ffd84d..190a256e99 100644 --- a/doc/include/imgproc_defaults_table.rst +++ b/doc/include/imgproc_defaults_table.rst @@ -9,7 +9,7 @@ Parameter Default ``bias`` ``dark`` ``trace`` ``arc`` ``orient`` ``True`` ``use_biasimage`` ``True`` ``False`` ``use_darkimage`` ``False`` -``spat_flexure_correct`` ``False`` +``spat_flexure_correct`` ``none`` ``use_pixelflat`` ``True`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``use_illumflat`` ``True`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``use_specillum`` ``False`` diff --git a/doc/include/spectrographs_table.rst b/doc/include/spectrographs_table.rst index 99bef11a21..2b877fac53 100644 --- a/doc/include/spectrographs_table.rst +++ b/doc/include/spectrographs_table.rst @@ -20,7 +20,7 @@ keck_deimos :class:`~pypeit.spectrographs.keck_deimos.KeckDEIMOSSp keck_esi :class:`~pypeit.spectrographs.keck_esi.KeckESISpectrograph` KECK ESI Echelle True False keck_hires :class:`~pypeit.spectrographs.keck_hires.KECKHIRESSpectrograph` KECK HIRES `Link `__ Echelle False False Post detector upgrade (~ August 2004). See :doc:`keck_hires` keck_kcrm :class:`~pypeit.spectrographs.keck_kcwi.KeckKCRMSpectrograph` KECK KCRM `Link `__ SlicerIFU True False Supported setups: RL, RM1, RM2, RH3; see :doc:`keck_kcwi` -keck_kcwi :class:`~pypeit.spectrographs.keck_kcwi.KeckKCWISpectrograph` KECK KCWI `Link `__ SlicerIFU True False Supported setups: BL, BM, BH2, BH3; see :doc:`keck_kcwi` +keck_kcwi :class:`~pypeit.spectrographs.keck_kcwi.KeckKCWISpectrograph` KECK KCWI `Link `__ SlicerIFU True False Supported setups: BL, BM, BH2, BH3; see :doc:`keck_kcwi` keck_lris_blue :class:`~pypeit.spectrographs.keck_lris.KeckLRISBSpectrograph` KECK LRISb `Link `__ MultiSlit True False Blue camera; Current FITS file format; used from May 2009, see :doc:`lris` keck_lris_blue_orig :class:`~pypeit.spectrographs.keck_lris.KeckLRISBOrigSpectrograph` KECK LRISb `Link `__ MultiSlit True False Blue camera; Original FITS file format; used until April 2009; see :doc:`lris` keck_lris_red :class:`~pypeit.spectrographs.keck_lris.KeckLRISRSpectrograph` KECK LRISr `Link `__ MultiSlit True True Red camera; Current FITS file format; LBNL detector, 2kx4k; used from May 2009, see :doc:`lris` diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index ee49c46478..68b77bca42 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -895,7 +895,7 @@ Key Type Opti ``shot_noise`` bool .. True Use the bias- and dark-subtracted image to calculate and include electron count shot noise in the image processing error budget ``sigclip`` int, float .. 4.5 Sigma level for rejection in LA cosmics routine ``sigfrac`` int, float .. 0.3 Fraction for the lower clipping threshold in LA cosmics routine. -``spat_flexure_correct`` bool .. False Correct slits, illumination flat, etc. for flexure +``spat_flexure_correct`` str ``none``, ``detector``, ``slit``, ``edge`` ``none`` Correct slits, illumination flat, etc. for spatial flexure. Options are: none, detector, slit, edge"none" means no correction is performed. "detector" means that a single shift is applied to all slits. "slit" means that each slit is shifted independently."edge" means that each slit edge is shifted independently. ``spat_flexure_maxlag`` int .. 20 Maximum of possible spatial flexure correction, in pixels ``subtract_continuum`` bool .. False Subtract off the continuum level from an image. This parameter should only be set to True to combine arcs with multiple different lamps. For all other cases, this parameter should probably be False. ``subtract_scattlight`` bool .. False Subtract off the scattered light from an image. This parameter should only be set to True for spectrographs that have dedicated methods to subtract scattered light. For all other cases, this parameter should be False. @@ -948,7 +948,6 @@ Key Type Options ``extr`` str .. ``OPT`` Extraction method to use for the sensitivity function. Options are: 'OPT' (optimal extraction), 'BOX' (boxcar extraction). Default is 'OPT'. ``extrap_blu`` float .. 0.1 Fraction of minimum wavelength coverage to grow the wavelength coverage of the sensitivitity function in the blue direction (`i.e.`, if the standard star spectrum cuts off at ``wave_min``) the sensfunc will be extrapolated to cover down to (1.0 - ``extrap_blu``) * ``wave_min`` ``extrap_red`` float .. 0.1 Fraction of maximum wavelength coverage to grow the wavelength coverage of the sensitivitity function in the red direction (`i.e.`, if the standard star spectrumcuts off at ``wave_max``) the sensfunc will be extrapolated to cover up to (1.0 + ``extrap_red``) * ``wave_max`` -``flatfile`` str .. .. Flat field file to be used if the sensitivity function model will utilize the blaze function computed from a flat field file in the Calibrations directory, e.g.Calibrations/Flat_A_0_DET01.fits ``hydrogen_mask_wid`` float .. 10.0 Mask width from line center for hydrogen recombination lines in Angstroms (total mask width is 2x this value). ``mask_helium_lines`` bool .. False Mask certain ``HeII`` recombination lines prominent in O-type stars in the sensitivity function fit A region equal to 0.5 * ``hydrogen_mask_wid`` on either side of the line center is masked. ``mask_hydrogen_lines`` bool .. True Mask hydrogen Balmer, Paschen, Brackett, and Pfund recombination lines in the sensitivity function fit. A region equal to ``hydrogen_mask_wid`` on either side of the line center is masked. @@ -959,6 +958,7 @@ Key Type Options ``star_mag`` float .. .. Magnitude of the standard star (for near-IR mainly) ``star_ra`` float .. .. RA of the standard star. This will override values in the header (`i.e.`, if they are wrong or absent) ``star_type`` str .. .. Spectral type of the standard star (for near-IR mainly) +``use_flat`` bool .. False If True, the flatfield spectrum will be used when computing the sensitivity function. ======================= ============================================== ================ =========================== ============================================================================================================================================================================================================================================================================================================================================================================================ @@ -3492,7 +3492,7 @@ Alterations to the default parameters are: [[[process]]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = True + spat_flexure_correct = detector [[flatfield]] slit_illum_finecorr = False [[wavelengths]] @@ -3513,7 +3513,7 @@ Alterations to the default parameters are: [[process]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = True + spat_flexure_correct = detector [flexure] spec_method = boxcar [sensfunc] @@ -3604,7 +3604,7 @@ Alterations to the default parameters are: [[[process]]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = True + spat_flexure_correct = detector [[flatfield]] slit_illum_finecorr = False [[wavelengths]] @@ -3625,7 +3625,7 @@ Alterations to the default parameters are: [[process]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = True + spat_flexure_correct = detector [flexure] spec_method = boxcar [sensfunc] @@ -3716,7 +3716,7 @@ Alterations to the default parameters are: [[[process]]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = True + spat_flexure_correct = detector [[flatfield]] slit_illum_finecorr = False [[wavelengths]] @@ -3743,7 +3743,7 @@ Alterations to the default parameters are: sigclip = 5.0 objlim = 5.0 noise_floor = 0.01 - spat_flexure_correct = True + spat_flexure_correct = detector [reduce] [[skysub]] bspline_spacing = 0.8 @@ -3839,7 +3839,7 @@ Alterations to the default parameters are: [[[process]]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = True + spat_flexure_correct = detector [[flatfield]] slit_illum_finecorr = False [[wavelengths]] @@ -3866,7 +3866,7 @@ Alterations to the default parameters are: sigclip = 5.0 objlim = 5.0 noise_floor = 0.01 - spat_flexure_correct = True + spat_flexure_correct = detector [reduce] [[skysub]] bspline_spacing = 0.8 @@ -3962,7 +3962,7 @@ Alterations to the default parameters are: [[[process]]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = True + spat_flexure_correct = detector [[flatfield]] slit_illum_finecorr = False [[wavelengths]] @@ -3989,7 +3989,7 @@ Alterations to the default parameters are: sigclip = 5.0 objlim = 5.0 noise_floor = 0.01 - spat_flexure_correct = True + spat_flexure_correct = detector [reduce] [[skysub]] bspline_spacing = 0.8 diff --git a/pypeit/core/skysub.py b/pypeit/core/skysub.py index 65a3b404b0..8bf1734e72 100644 --- a/pypeit/core/skysub.py +++ b/pypeit/core/skysub.py @@ -1067,7 +1067,7 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, model_noise=True, debug_bkpts=False, show_profile=False, show_resids=False, show_fwhm=False, adderr=0.01, base_var=None, count_scale=None, no_local_sky:bool=False): - """ + r""" Perform local sky subtraction, profile fitting, and optimal extraction slit by slit. Objects are sky/subtracted extracted in order of the highest average (across all orders) S/N ratio object first, and then for a given From 25a0737a16a77e4a25405fc821ebfe0f9812135a Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 20 Sep 2024 19:49:01 +0100 Subject: [PATCH 32/54] PR cleanup --- pypeit/core/flexure.py | 4 ++-- pypeit/core/tracewave.py | 39 +++++++----------------------------- pypeit/flatfield.py | 6 ++++-- pypeit/images/pypeitimage.py | 2 +- pypeit/images/rawimage.py | 3 --- pypeit/wavetilts.py | 6 ++---- 6 files changed, 16 insertions(+), 44 deletions(-) diff --git a/pypeit/core/flexure.py b/pypeit/core/flexure.py index 600577de39..6e6c23a8e7 100644 --- a/pypeit/core/flexure.py +++ b/pypeit/core/flexure.py @@ -1425,11 +1425,11 @@ def flexure_diagnostic(file, file_type='spec2d', flexure_type='spec', chk_versio failed. If False, throw a warning only. Default is False. Returns: - :obj:`astropy.table.Table`, :obj:`float`, None: If file_type is 'spec2d' and + :obj:`astropy.table.Table`, :obj:`float`: If file_type is 'spec2d' and flexure_type is 'spec', return a table with the spectral flexure. If file_type is 'spec2d' and flexure_type is 'spat', return the spatial flexure. If file_type is 'spec1d', return a table with the spectral - flexure. + flexure. If the file_type is neither 'spec2d' nor 'spec1d', return None. """ # value to return diff --git a/pypeit/core/tracewave.py b/pypeit/core/tracewave.py index 2c8f7c6f8f..e6e0ad325c 100644 --- a/pypeit/core/tracewave.py +++ b/pypeit/core/tracewave.py @@ -851,14 +851,12 @@ def fit_tilts(trc_tilt_dict, thismask, slit_cen, spat_order=3, spec_order=4, max # msgs.info("RMS/FWHM: {}".format(rms_real/fwhm)) -def fit2tilts_prepareSlit(img_shape, slit_left, slit_right, thismask_science, spat_flexure=None): +def fit2tilts_prepareSlit(slit_left, slit_right, thismask_science, spat_flexure=None): """ Prepare the slit for the fit2tilts function Parameters ---------- - img_shape : :obj:`tuple` - Shape of the science image slit_left : `numpy.ndarray`_ Left slit edge slit_right : `numpy.ndarray`_ @@ -876,6 +874,8 @@ def fit2tilts_prepareSlit(img_shape, slit_left, slit_right, thismask_science, sp Tuple containing the spectral and spatial coordinates of the slit, including spatial flexure. These variables are to be used in the fit2tilts function. """ + # Get the image shape + img_shape = thismask_science.shape # Check the spatial flexure input if spat_flexure is not None and len(spat_flexure) != 2: msgs.error('Spatial flexure must be a two element array') @@ -900,7 +900,7 @@ def fit2tilts_prepareSlit(img_shape, slit_left, slit_right, thismask_science, sp return _spec_eval, _spat_eval -def fit2tilts(shape, coeff2, func2d, spec_eval=None, spat_eval=None, spat_shift=None): +def fit2tilts(shape, coeff2, func2d, spec_eval=None, spat_eval=None): """ Evaluate the wavelength tilt model over the full image. @@ -922,22 +922,14 @@ def fit2tilts(shape, coeff2, func2d, spec_eval=None, spat_eval=None, spat_shift= spec_eval : `numpy.ndarray`_, optional 1D array indicating how spectral pixel locations move across the image. If spec_eval is provided, spat_eval must also be provided. - spat_shift : float, `numpy.ndarray`_, optional - Spatial shift to be added to image pixels before evaluation - If you are accounting for flexure, then you probably wish to - input -1*flexure_shift into this parameter. Note that this - should either be a float or a 1D array with two elements - (and both of these elements must be equal). Returns ------- tilts : `numpy.ndarray`_, float Image indicating how spectral pixel locations move across the image. """ - # Determine which mode to run this function + # Determine if coordinates have been supplied that include the spatial flexure. if spec_eval is not None and spat_eval is not None: - if spat_shift is not None: - msgs.warn('spat_shift is ignored when spec_eval and spat_eval are provided.') _spec_eval = spec_eval _spat_eval = spat_eval else: @@ -945,25 +937,8 @@ def fit2tilts(shape, coeff2, func2d, spec_eval=None, spat_eval=None, spat_shift= if (spec_eval is None and spat_eval is not None) or (spec_eval is not None and spat_eval is None): msgs.warn('Both spec_eval and spat_eval must be provided.' + msgs.newline() + 'Only one variable provided, so a new (full) grid will be generated.') - # Check the flexure - if spat_shift is None: - _spat_shift = 0. - elif isinstance(spat_shift, (int, float)): - _spat_shift = spat_shift - elif isinstance(spat_shift, np.ndarray): - if spat_shift.size != 2: - msgs.error('spat_shift must be a 2-element array.') - elif spat_shift[0] != spat_shift[1]: - msgs.error('The two elements of spat_shift must be equal.' + msgs.newline() + - 'To include different spatial shifts at the two ends of the slit, ' + msgs.newline() + - 'you must provide the variables spec_eval and spat_eval instead of spat_shift.') - else: - _spat_shift = spat_shift[0] - else: - msgs.error('spat_shift must be either None, a float, or a 1D array with two identical elements.' + msgs.newline() + - 'To include different spatial shifts at the two ends of the slit, ' + msgs.newline() + - 'you must provide the variables spec_eval and spat_eval instead of spat_shift.') - + msgs.warn("Assuming no spatial flexure.") + _spat_shift = 0.0 # Setup the evaluation grid nspec, nspat = shape xnspecmin1 = float(nspec - 1) diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index dc0edfb4e1..0113506b58 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -999,10 +999,12 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): if self.slitless: tilts = np.tile(np.arange(rawflat.shape[0]) / (rawflat.shape[0]-1), (rawflat.shape[1], 1)).T else: - # TODO -- JFH Confirm the sign of this shift is correct! _flexure = np.zeros(2) if self.wavetilts.spat_flexure is None else self.wavetilts.spat_flexure[slit_idx,:] + _spec_eval, _spat_eval = tracewave.fit2tilts_prepareSlit(self.slits.left_init[:, slit_idx], + self.slits.right_init[:, slit_idx], + onslit_init, _flexure) tilts = tracewave.fit2tilts(rawflat.shape, self.wavetilts['coeffs'][:,:,slit_idx], - self.wavetilts['func2d'], spat_shift=-1*_flexure) + self.wavetilts['func2d'], spec_eval=_spec_eval, spat_eval=_spat_eval) # Convert the tilt image to an image with the spectral pixel index spec_coo = tilts * (nspec-1) diff --git a/pypeit/images/pypeitimage.py b/pypeit/images/pypeitimage.py index 6a62901c3c..a39e951552 100644 --- a/pypeit/images/pypeitimage.py +++ b/pypeit/images/pypeitimage.py @@ -94,7 +94,7 @@ class PypeItImage(datamodel.DataContainer): # TODO These docs are confusing. The __init__ method needs to be documented just as it is for # every other class that we have written in PypeIt, i.e. the arguments all need to be documented. They are not # documented here and instead we have the odd Args documentation above. - version = '1.3.0' + version = '1.3.1' """Datamodel version number""" datamodel = {'PYP_SPEC': dict(otype=str, descr='PypeIt spectrograph name'), diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 6d6dc20e3b..37c1a4ab25 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -675,9 +675,6 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl maxlag=self.par['spat_flexure_maxlag']) \ if self.par['spat_flexure_correct'] != "none" else None - # self.spat_flexure_shift = self.spatial_flexure_shift(slits, maxlag=self.par['spat_flexure_maxlag']) \ - # if self.par['spat_flexure_correct'] else None - # - Subtract scattered light... this needs to be done before flatfielding. if self.par['subtract_scattlight']: self.subtract_scattlight(scattlight, slits) diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index 47dd6f0886..818f5cef72 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -169,8 +169,7 @@ def fit2tiltimg(self, slitmask, slits_left, slits_right, spat_flexure=None): coeff_out = self.coeffs[:self.spec_order[slit_idx]+1, :self.spat_order[slit_idx]+1, slit_idx] # Extract the spectral and spatial coordinates for this slit thismask_science = (slitmask == slit_spat) - _spec_eval, _spat_eval = tracewave.fit2tilts_prepareSlit(slitmask.shape, - slits_left[:, slit_idx], slits_right[:, slit_idx], + _spec_eval, _spat_eval = tracewave.fit2tilts_prepareSlit(slits_left[:, slit_idx], slits_right[:, slit_idx], thismask_science, _spat_flexure[slit_idx, :]) # Calculate the tilts final_tilts[thismask_science] = tracewave.fit2tilts(final_tilts.shape, coeff_out, self.func2d, @@ -796,8 +795,7 @@ def run(self, doqa=True, debug=False, show=False): # which corresonds to the same binning as the science # images, trace images, and pixelflats etc. thismask_science = self.slitmask_science == slit_spat - _spec_eval, _spat_eval = tracewave.fit2tilts_prepareSlit(self.slitmask_science.shape, - slits_left[:, slit_idx], slits_right[:, slit_idx], + _spec_eval, _spat_eval = tracewave.fit2tilts_prepareSlit(slits_left[:, slit_idx], slits_right[:, slit_idx], thismask_science, self.spat_flexure[slit_idx, :]) self.final_tilts[thismask_science] = tracewave.fit2tilts(self.slitmask_science.shape, coeff_out, self.par['func2d'], spec_eval=_spec_eval, spat_eval=_spat_eval) From 73d0afbc002ffa94de79fc6aff967cef736877c0 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 20 Sep 2024 20:03:55 +0100 Subject: [PATCH 33/54] fix tilt shape --- pypeit/flatfield.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index 0113506b58..ae94ef053b 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -1003,7 +1003,8 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): _spec_eval, _spat_eval = tracewave.fit2tilts_prepareSlit(self.slits.left_init[:, slit_idx], self.slits.right_init[:, slit_idx], onslit_init, _flexure) - tilts = tracewave.fit2tilts(rawflat.shape, self.wavetilts['coeffs'][:,:,slit_idx], + tilts = np.zeros(rawflat.shape, dtype=float) + tilts[onslit_init] = tracewave.fit2tilts(rawflat.shape, self.wavetilts['coeffs'][:,:,slit_idx], self.wavetilts['func2d'], spec_eval=_spec_eval, spat_eval=_spat_eval) # Convert the tilt image to an image with the spectral pixel index spec_coo = tilts * (nspec-1) From 78120d8f69c088be20be74e023c4621285249302 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 20 Sep 2024 20:19:55 +0100 Subject: [PATCH 34/54] update users --- presentations/py/users.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/presentations/py/users.py b/presentations/py/users.py index a690f10385..23cea32e4a 100644 --- a/presentations/py/users.py +++ b/presentations/py/users.py @@ -21,9 +21,9 @@ def set_fontsize(ax, fsz): ax.get_xticklabels() + ax.get_yticklabels()): item.set_fontsize(fsz) -user_dates = ["2021-03-11", "2022-04-29", "2022-11-07", "2022-12-06", "2023-06-08", "2023-06-29", "2023-07-11", "2023-09-03", "2023-10-13", "2023-12-01", "2023-12-15", "2024-02-22", "2024-03-21", "2024-04-09", "2024-05-02", "2024-05-19", "2024-06-06", "2024-06-10", "2024-08-20", "2024-09-11"] +user_dates = ["2021-03-11", "2022-04-29", "2022-11-07", "2022-12-06", "2023-06-08", "2023-06-29", "2023-07-11", "2023-09-03", "2023-10-13", "2023-12-01", "2023-12-15", "2024-02-22", "2024-03-21", "2024-04-09", "2024-05-02", "2024-05-19", "2024-06-06", "2024-06-10", "2024-08-20", "2024-09-11", "2024-09-20"] user_dates = numpy.array([numpy.datetime64(date) for date in user_dates]) -user_number = numpy.array([125, 293, 390, 394, 477, 487, 506, 518, 531, 544, 551, 568, 579, 588, 596, 603, 616, 620, 643, 655]) +user_number = numpy.array([125, 293, 390, 394, 477, 487, 506, 518, 531, 544, 551, 568, 579, 588, 596, 603, 616, 620, 643, 655, 671]) user_pred_dates = numpy.array([numpy.datetime64(date) for date in ["2024-06-10", "2024-12-31", "2025-12-31", "2026-12-31", From 4f5fa099162026e7fa085e6b6c9b083411d29da5 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 2 Oct 2024 11:26:10 +0100 Subject: [PATCH 35/54] merge with order_sync --- pypeit/core/flexure.py | 90 ------------------------------------------ pypeit/spec2dobj.py | 18 +++++++-- 2 files changed, 14 insertions(+), 94 deletions(-) diff --git a/pypeit/core/flexure.py b/pypeit/core/flexure.py index f54f55025e..1cae726fc8 100644 --- a/pypeit/core/flexure.py +++ b/pypeit/core/flexure.py @@ -17,7 +17,6 @@ from astropy import stats from astropy import units from astropy.io import ascii -from astropy.table import Table import scipy.signal import scipy.optimize as opt from scipy import interpolate @@ -1407,95 +1406,6 @@ def sky_em_residuals(wave:np.ndarray, flux:np.ndarray, return dwave[m], diff[m], diff_err[m], los[m], los_err[m] -def flexure_diagnostic(file, file_type='spec2d', flexure_type='spec', chk_version=False): - """ - Print the spectral or spatial flexure of a spec2d or spec1d file - - Args: - file (:obj:`str`, `Path`_): - Filename of the spec2d or spec1d file to check - file_type (:obj:`str`, optional): - Type of the file to check. Options are 'spec2d' or 'spec1d'. Default - is 'spec2d'. - flexure_type (:obj:`str`, optional): - Type of flexure to check. Options are 'spec' or 'spat'. Default is - 'spec'. - chk_version (:obj:`bool`, optional): - If True, raise an error if the datamodel version or type check - failed. If False, throw a warning only. Default is False. - - Returns: - :obj:`astropy.table.Table`, :obj:`float`: If file_type is 'spec2d' and - flexure_type is 'spec', return a table with the spectral flexure. If - file_type is 'spec2d' and flexure_type is 'spat', return the spatial - flexure. If file_type is 'spec1d', return a table with the spectral - flexure. If the file_type is neither 'spec2d' nor 'spec1d', return None. - """ - - # value to return - return_flex = None - - if file_type == 'spec2d': - # load the spec2d file - allspec2D = spec2dobj.AllSpec2DObj.from_fits(file, chk_version=chk_version) - # Loop on Detectors - for det in allspec2D.detectors: - print('') - print('=' * 50 + f'{det:^7}' + '=' * 51) - # get and print the spectral flexure - if flexure_type == 'spec': - spec_flex = allspec2D[det].sci_spec_flexure - spec_flex.rename_column('sci_spec_flexure', 'global_spec_shift') - if np.all(spec_flex['global_spec_shift'] != None): - spec_flex['global_spec_shift'].format = '0.3f' - # print the table - spec_flex.pprint_all() - # return the table - return_flex = spec_flex - # get and print the spatial flexure - if flexure_type == 'spat': - spat_flexure = allspec2D[det].sci_spat_flexure - if np.all(spat_flexure == spat_flexure[0, 0]): - # print the value - print(f'Spatial shift: {spat_flexure}') - elif np.array_equal(spat_flexure[:,0],spat_flexure[:,1]): - # print the value of each slit - for ii in range(spat_flexure.shape[0]): - print(f' Slit {ii+1} spatial shift: {spat_flexure[ii,0]}') - else: - # print the value for the edge of each slit - for ii in range(spat_flexure.shape[0]): - print(' Slit {0:2d} -- left edge spatial shift: {1:f}'.format(ii+1, spat_flexure[ii,0])) - print(' -- right edge spatial shift: {0:f}'.format(spat_flexure[ii,1])) - # return the value - # TODO :: This return_flex is in a for loop, so the return will be the last value. - # :: Also, the return_flex is not used in the code. So, perhaps this can be removed? - return_flex = spat_flexure - elif file_type == 'spec1d': - # no spat flexure in spec1d file - if flexure_type == 'spat': - msgs.error("Spat flexure not available in the spec1d file, try with a spec2d file") - # load the spec1d file - sobjs = specobjs.SpecObjs.from_fitsfile(file, chk_version=chk_version) - spec_flex = Table() - spec_flex['NAME'] = sobjs.NAME - spec_flex['global_spec_shift'] = sobjs.FLEX_SHIFT_GLOBAL - if np.all(spec_flex['global_spec_shift'] != None): - spec_flex['global_spec_shift'].format = '0.3f' - spec_flex['local_spec_shift'] = sobjs.FLEX_SHIFT_LOCAL - if np.all(spec_flex['local_spec_shift'] != None): - spec_flex['local_spec_shift'].format = '0.3f' - spec_flex['total_spec_shift'] = sobjs.FLEX_SHIFT_TOTAL - if np.all(spec_flex['total_spec_shift'] != None): - spec_flex['total_spec_shift'].format = '0.3f' - # print the table - spec_flex.pprint_all() - # return the table - return_flex = spec_flex - - return return_flex - - # TODO -- Consider separating the methods from the DataContainer as per calibrations class MultiSlitFlexure(DataContainer): """ diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index f44d1ac863..da076bc983 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -768,11 +768,21 @@ def flexure_diagnostics(self, flexure_type='spat'): return_flex[det] = spec_flex # get and print the spatial flexure if flexure_type == 'spat': - spat_flex = self[det].sci_spat_flexure - # print the value - print(f'Spat shift: {spat_flex}') + spat_flexure = self[det].sci_spat_flexure + if np.all(spat_flexure == spat_flexure[0, 0]): + # print the value + print(f'Spatial shift: {spat_flexure}') + elif np.array_equal(spat_flexure[:,0],spat_flexure[:,1]): + # print the value of each slit + for ii in range(spat_flexure.shape[0]): + print(f' Slit {ii+1} spatial shift: {spat_flexure[ii,0]}') + else: + # print the value for the edge of each slit + for ii in range(spat_flexure.shape[0]): + print(' Slit {0:2d} -- left edge spatial shift: {1:f}'.format(ii+1, spat_flexure[ii,0])) + print(' -- right edge spatial shift: {0:f}'.format(spat_flexure[ii,1])) # return the value - return_flex[det] = spat_flex + return_flex[det] = spat_flexure return return_flex From 7755dee0f047661b3b02da3ee51a110b0cc9ff04 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 2 Oct 2024 13:05:30 +0100 Subject: [PATCH 36/54] review comments addressed --- deprecated/flat.py | 2 +- pypeit/calibrations.py | 2 +- pypeit/core/tracewave.py | 18 ++++++++++++++---- pypeit/flatfield.py | 7 ++++--- pypeit/images/pypeitimage.py | 12 +++++++++--- pypeit/images/rawimage.py | 10 +++++----- pypeit/par/pypeitpar.py | 18 +++++++++--------- pypeit/pypeit.py | 6 +++--- pypeit/spectrographs/gemini_gnirs.py | 2 +- pypeit/spectrographs/gtc_osiris.py | 2 +- pypeit/spectrographs/keck_kcwi.py | 2 +- pypeit/spectrographs/keck_lris.py | 4 ++-- pypeit/wavetilts.py | 6 +++--- 13 files changed, 54 insertions(+), 37 deletions(-) diff --git a/deprecated/flat.py b/deprecated/flat.py index f59334af70..65e737494a 100644 --- a/deprecated/flat.py +++ b/deprecated/flat.py @@ -226,7 +226,7 @@ def fit_flat(flat, tilts_dict, tslits_dict_in, slit, inmask = None, slitmask_pad = pixels.tslits2mask(tslits_dict_in, pad = pad) thismask = (slitmask_pad == slit) # mask enclosing the wider slit bounadries # Create a tilts image using this padded thismask, rather than using the original thismask_in slit pixels - tilts = tracewave.fit2tilts(shape, tilts_dict['coeffs'], tilts_dict['func2d']) + tilts = tracewave.fit2tilts(tilts_dict['coeffs'], tilts_dict['func2d'], shape=shape) piximg = tilts * (nspec-1) pixvec = np.arange(nspec) diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index c89b2ce7d8..167cefddca 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -1173,7 +1173,7 @@ def get_tilts(self): return self.wavetilts # Get flexure - _spat_flexure = self.mstilt.spat_flexure if self.par['tiltframe']['process']['spat_flexure_correct'] != "none" \ + _spat_flexure = self.mstilt.spat_flexure if self.par['tiltframe']['process']['spat_flexure_method'] != "skip" \ else None # Build diff --git a/pypeit/core/tracewave.py b/pypeit/core/tracewave.py index e6e0ad325c..9f68ac2419 100644 --- a/pypeit/core/tracewave.py +++ b/pypeit/core/tracewave.py @@ -900,18 +900,20 @@ def fit2tilts_prepareSlit(slit_left, slit_right, thismask_science, spat_flexure= return _spec_eval, _spat_eval -def fit2tilts(shape, coeff2, func2d, spec_eval=None, spat_eval=None): +def fit2tilts(coeff2, func2d, shape=None, spec_eval=None, spat_eval=None): """ - Evaluate the wavelength tilt model over the full image. + Evaluate the wavelength tilt model over the full image. Note that this function + requires either shape or both spec_eval and spat_eval to be provided. If all three + are provided, spec_eval and spat_eval will be used. Parameters ---------- - shape : tuple of ints, - shape of image coeff2 : `numpy.ndarray`_, float result of griddata tilt fit func2d : str the 2d function used to fit the tilts + shape : tuple of ints, optional + Shape of image. Only used if spat_eval and spec_eval are not provided. spat_eval : `numpy.ndarray`_, optional 1D array indicating how spatial pixel locations move across the image. If spat_eval is provided, spec_eval must also be provided. @@ -937,6 +939,14 @@ def fit2tilts(shape, coeff2, func2d, spec_eval=None, spat_eval=None): if (spec_eval is None and spat_eval is not None) or (spec_eval is not None and spat_eval is None): msgs.warn('Both spec_eval and spat_eval must be provided.' + msgs.newline() + 'Only one variable provided, so a new (full) grid will be generated.') + # Print a warning if neither are provided + if spec_eval is None and spat_eval is None: + msgs.warn('No spatial and spectral coordinates provided.' + msgs.newline() + + 'A new (full) grid will be generated.') + # Print a warning is shape is not provided + if shape is None: + msgs.error('No shape provided for the image.' + msgs.newline() + + 'You must provide either `shape` or both `spat_eval` and `spec_eval`.') msgs.warn("Assuming no spatial flexure.") _spat_shift = 0.0 # Setup the evaluation grid diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index ae94ef053b..03de5e41b9 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -711,7 +711,7 @@ def build_waveimg(self): msgs.error("Wavelength calib or tilts are not available. Cannot generate wavelength image.") else: spat_flexure = self.wavetilts.spat_flexure - left, right, msk = self.slits.select_edges(spat_flexure=spat_flexure) + left, right, msk = self.slits.select_edges(initial=True, spat_flexure=spat_flexure) slitmask = self.slits.slit_img(initial=True, spat_flexure=spat_flexure) tilts = self.wavetilts.fit2tiltimg(slitmask, left, right, spat_flexure=spat_flexure) # Save to class attribute for inclusion in the Flat calibration frame @@ -1004,8 +1004,9 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): self.slits.right_init[:, slit_idx], onslit_init, _flexure) tilts = np.zeros(rawflat.shape, dtype=float) - tilts[onslit_init] = tracewave.fit2tilts(rawflat.shape, self.wavetilts['coeffs'][:,:,slit_idx], - self.wavetilts['func2d'], spec_eval=_spec_eval, spat_eval=_spat_eval) + tilts[onslit_init] = tracewave.fit2tilts(self.wavetilts['coeffs'][:,:,slit_idx], + self.wavetilts['func2d'], + spec_eval=_spec_eval, spat_eval=_spat_eval) # Convert the tilt image to an image with the spectral pixel index spec_coo = tilts * (nspec-1) diff --git a/pypeit/images/pypeitimage.py b/pypeit/images/pypeitimage.py index a39e951552..d69f174b22 100644 --- a/pypeit/images/pypeitimage.py +++ b/pypeit/images/pypeitimage.py @@ -785,10 +785,16 @@ def sub(self, other): # Spatial flexure spat_flexure = self.spat_flexure if other.spat_flexure is not None and spat_flexure is not None \ - and np.array_equal(other.spat_flexure, spat_flexure): - msgs.warn(f'Spatial flexure different for images being subtracted. Adopting ' \ + and not np.array_equal(other.spat_flexure, spat_flexure): + msgs.warn(f'Spatial flexure different for images being subtracted. Adopting ' f'the maximum spatial flexure of each individual edge.') - spat_flexure = np.maximum(spat_flexure, other.spat_flexure) + # Loop through all slit edges and find the largest flexure + for ii in range(spat_flexure.shape[0]): + # Assign the largest flexure (irrespective of sign) for each edge + if np.abs(other.spat_flexure[ii,0]) > np.abs(spat_flexure[ii,0]): + spat_flexure[ii,0] = other.spat_flexure[ii,0] + if np.abs(other.spat_flexure[ii,1]) > np.abs(spat_flexure[ii,1]): + spat_flexure[ii,1] = other.spat_flexure[ii,1] # Create a copy of the detector, if it is defined, to be used when # creating the new pypeit image below diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 37c1a4ab25..41907a8fca 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -241,7 +241,7 @@ def use_slits(self): """ if self.par is None: return False - return (self.par['spat_flexure_correct'] != "none") or (self.use_flat and self.par['use_illumflat']) + return (self.par['spat_flexure_method'] != "skip") or (self.use_flat and self.par['use_illumflat']) def apply_gain(self, force=False): """ @@ -506,7 +506,7 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl Bias image for bias subtraction. slits (:class:`~pypeit.slittrace.SlitTraceSet`, optional): Used to calculate spatial flexure between the image and the - slits, if requested via the ``spat_flexure_correct`` parameter + slits, if requested via the ``spat_flexure_method`` parameter in :attr:`par`; see :func:`~pypeit.core.flexure.spat_flexure_shift`. Also used to construct the slit illumination profile, if requested via the @@ -546,7 +546,7 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl msgs.error('No dark available for dark subtraction!') if self.par['subtract_scattlight'] and scattlight is None: msgs.error('Scattered light subtraction requested, but scattered light model not provided.') - if (self.par['spat_flexure_correct'] != "none") and slits is None: + if (self.par['spat_flexure_method'] != "skip") and slits is None: msgs.error('Spatial flexure correction requested but no slits provided.') if self.use_flat and flatimages is None: msgs.error('Flat-field corrections requested but no flat-field images generated ' @@ -671,9 +671,9 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl # bias and dark subtraction) and before field flattening. Also the # function checks that the slits exist if running the spatial flexure # correction, so no need to do it again here. - self.spat_flexure_shift = self.spatial_flexure_shift(slits, method=self.par['spat_flexure_correct'], + self.spat_flexure_shift = self.spatial_flexure_shift(slits, method=self.par['spat_flexure_method'], maxlag=self.par['spat_flexure_maxlag']) \ - if self.par['spat_flexure_correct'] != "none" else None + if self.par['spat_flexure_method'] != "skip" else None # - Subtract scattered light... this needs to be done before flatfielding. if self.par['subtract_scattlight']: diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index a7028441b9..1cef1022d3 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -222,7 +222,7 @@ def __init__(self, trim=None, apply_gain=None, orient=None, empirical_rn=None, shot_noise=None, noise_floor=None, use_pixelflat=None, use_illumflat=None, use_specillum=None, use_pattern=None, subtract_scattlight=None, scattlight=None, subtract_continuum=None, - spat_flexure_correct=None, spat_flexure_maxlag=None): + spat_flexure_method=None, spat_flexure_maxlag=None): # Grab the parameter names and values from the function # arguments @@ -356,12 +356,12 @@ def __init__(self, trim=None, apply_gain=None, orient=None, '``slit_illum_relative=True`` in the ``flatfield`` parameter set!' # Flexure - defaults['spat_flexure_correct'] = 'none' - options['spat_flexure_correct'] = ProcessImagesPar.valid_spatial_flexure() - dtypes['spat_flexure_correct'] = str - descr['spat_flexure_correct'] = 'Correct slits, illumination flat, etc. for spatial flexure. ' \ - 'Options are: {0}'.format(', '.join(options['spat_flexure_correct'])) + \ - '"none" means no correction is performed. ' \ + defaults['spat_flexure_method'] = 'skip' + options['spat_flexure_method'] = ProcessImagesPar.valid_spatial_flexure() + dtypes['spat_flexure_method'] = str + descr['spat_flexure_method'] = 'Correct slits, illumination flat, etc. for spatial flexure. ' \ + 'Options are: {0}'.format(', '.join(options['spat_flexure_method'])) + \ + '"skip" means no correction is performed. ' \ '"detector" means that a single shift is applied to all slits. ' \ '"slit" means that each slit is shifted independently.' \ '"edge" means that each slit edge is shifted independently.' @@ -467,7 +467,7 @@ def from_dict(cls, cfg): parkeys = ['trim', 'apply_gain', 'orient', 'use_biasimage', 'subtract_continuum', 'subtract_scattlight', 'scattlight', 'use_pattern', 'use_overscan', 'overscan_method', 'overscan_par', 'use_darkimage', 'dark_expscale', - 'spat_flexure_correct', 'spat_flexure_maxlag', 'use_illumflat', 'use_specillum', + 'spat_flexure_method', 'spat_flexure_maxlag', 'use_illumflat', 'use_specillum', 'empirical_rn', 'shot_noise', 'noise_floor', 'use_pixelflat', 'combine', 'scale_to_mean', 'correct_nonlinear', 'satpix', #'calib_setup_and_bit', 'n_lohi', 'mask_cr', 'lamaxiter', 'grow', 'clip', 'comb_sigrej', 'rmcompact', @@ -504,7 +504,7 @@ def valid_spatial_flexure(): """ Return the valid methods for combining frames. """ - return ['none', 'detector', 'slit', 'edge'] + return ['skip', 'detector', 'slit', 'edge'] @staticmethod def valid_saturation_handling(): diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index b8edbe7a17..824924dbd2 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -814,11 +814,11 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): spat_flexure = np.zeros((self.caliBrate.slits.nslits, 2)) # No spatial flexure, unless we find it below # use the flexure correction in the "shift" column manual_flexure = self.fitstbl[frames[0]]['shift'] - if (self.objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_correct'] != "none") or \ - (self.objtype == 'standard' and self.par['calibrations']['standardframe']['process']['spat_flexure_correct'] != "none") or \ + if (self.objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_method'] != "skip") or \ + (self.objtype == 'standard' and self.par['calibrations']['standardframe']['process']['spat_flexure_method'] != "skip") or \ manual_flexure: if (manual_flexure != self.fitstbl.MASKED_VALUE) and np.issubdtype(self.fitstbl[frames[0]]["shift"], np.integer): - msgs.info(f'Implementing manual spatial flexure of {manual_flexure}') + msgs.info(f'Implementing manual spatial flexure of {manual_flexure} pixels') spat_flexure = np.full((self.caliBrate.slits.nslits, 2), np.float64(manual_flexure)) sciImg.spat_flexure = spat_flexure else: diff --git a/pypeit/spectrographs/gemini_gnirs.py b/pypeit/spectrographs/gemini_gnirs.py index 5871bbd8a5..60a7bf8577 100644 --- a/pypeit/spectrographs/gemini_gnirs.py +++ b/pypeit/spectrographs/gemini_gnirs.py @@ -606,7 +606,7 @@ def default_pypeit_par(cls): par['scienceframe']['process']['objlim'] = 1.5 par['scienceframe']['process']['use_illumflat'] = False # illumflat is applied when building the relative scale image in reduce.py, so should be applied to scienceframe too. par['scienceframe']['process']['use_specillum'] = False # apply relative spectral illumination - par['scienceframe']['process']['spat_flexure_correct'] = "none" # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames + par['scienceframe']['process']['spat_flexure_method'] = "skip" # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames par['scienceframe']['process']['use_biasimage'] = False par['scienceframe']['process']['use_darkimage'] = False par['calibrations']['flatfield']['slit_illum_finecorr'] = False diff --git a/pypeit/spectrographs/gtc_osiris.py b/pypeit/spectrographs/gtc_osiris.py index 80eb64a5d4..922fa72546 100644 --- a/pypeit/spectrographs/gtc_osiris.py +++ b/pypeit/spectrographs/gtc_osiris.py @@ -460,7 +460,7 @@ def default_pypeit_par(cls): par['scienceframe']['process']['objlim'] = 1.5 par['scienceframe']['process']['use_illumflat'] = False # illumflat is applied when building the relative scale image in reduce.py, so should be applied to scienceframe too. par['scienceframe']['process']['use_specillum'] = False # apply relative spectral illumination - par['scienceframe']['process']['spat_flexure_correct'] = "none" # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames + par['scienceframe']['process']['spat_flexure_method'] = "skip" # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames par['scienceframe']['process']['use_biasimage'] = False par['scienceframe']['process']['use_darkimage'] = False par['calibrations']['flatfield']['slit_illum_finecorr'] = False diff --git a/pypeit/spectrographs/keck_kcwi.py b/pypeit/spectrographs/keck_kcwi.py index 218fb99a69..1ead2754b6 100644 --- a/pypeit/spectrographs/keck_kcwi.py +++ b/pypeit/spectrographs/keck_kcwi.py @@ -319,7 +319,7 @@ def default_pypeit_par(cls): # Illumination corrections par['scienceframe']['process']['use_illumflat'] = True # illumflat is applied when building the relative scale image in reduce.py, so should be applied to scienceframe too. par['scienceframe']['process']['use_specillum'] = True # apply relative spectral illumination - par['scienceframe']['process']['spat_flexure_correct'] = "none" # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames + par['scienceframe']['process']['spat_flexure_method'] = "skip" # don't correct for spatial flexure - varying spatial illumination profile could throw this correction off. Also, there's no way to do astrometric correction if we can't correct for spatial flexure of the contbars frames par['scienceframe']['process']['use_biasimage'] = True # Need to use bias frames for KCWI, because the bias level varies monotonically with spatial and spectral direction par['scienceframe']['process']['use_darkimage'] = False diff --git a/pypeit/spectrographs/keck_lris.py b/pypeit/spectrographs/keck_lris.py index 07848a8294..2b63aa1c7e 100644 --- a/pypeit/spectrographs/keck_lris.py +++ b/pypeit/spectrographs/keck_lris.py @@ -102,8 +102,8 @@ def default_pypeit_par(cls): # Always correct for spatial flexure on science images # TODO -- Decide whether to make the following defaults # May not want to do them for LongSlit - par['scienceframe']['process']['spat_flexure_correct'] = "detector" - par['calibrations']['standardframe']['process']['spat_flexure_correct'] = "detector" + par['scienceframe']['process']['spat_flexure_method'] = "detector" + par['calibrations']['standardframe']['process']['spat_flexure_method'] = "detector" par['scienceframe']['exprng'] = [61, None] diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index 818f5cef72..cd89b9d589 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -172,7 +172,7 @@ def fit2tiltimg(self, slitmask, slits_left, slits_right, spat_flexure=None): _spec_eval, _spat_eval = tracewave.fit2tilts_prepareSlit(slits_left[:, slit_idx], slits_right[:, slit_idx], thismask_science, _spat_flexure[slit_idx, :]) # Calculate the tilts - final_tilts[thismask_science] = tracewave.fit2tilts(final_tilts.shape, coeff_out, self.func2d, + final_tilts[thismask_science] = tracewave.fit2tilts(coeff_out, self.func2d, spec_eval=_spec_eval, spat_eval=_spat_eval) # Return return final_tilts @@ -797,8 +797,8 @@ def run(self, doqa=True, debug=False, show=False): thismask_science = self.slitmask_science == slit_spat _spec_eval, _spat_eval = tracewave.fit2tilts_prepareSlit(slits_left[:, slit_idx], slits_right[:, slit_idx], thismask_science, self.spat_flexure[slit_idx, :]) - self.final_tilts[thismask_science] = tracewave.fit2tilts(self.slitmask_science.shape, coeff_out, self.par['func2d'], - spec_eval=_spec_eval, spat_eval=_spat_eval) + self.final_tilts[thismask_science] = tracewave.fit2tilts(coeff_out, self.par['func2d'], + spec_eval=_spec_eval, spat_eval=_spat_eval) if show: viewer, ch = display.show_image(self.mstilt.image * (self.slitmask > -1), chname='tilts') From 77cf107080f154491072e5ce75cc618ffba5d36a Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 2 Oct 2024 13:16:48 +0100 Subject: [PATCH 37/54] updated flexure docs --- doc/calibrations/flexure.rst | 22 ++- doc/calibrations/image_proc.rst | 4 +- doc/help/run_pypeit.rst | 2 +- doc/include/class_datamodel_pypeitimage.rst | 2 +- doc/include/datamodel_arcimage.rst | 2 +- doc/include/datamodel_biasimage.rst | 2 +- doc/include/datamodel_darkimage.rst | 2 +- doc/include/datamodel_tiltimage.rst | 2 +- doc/include/gemini_gnirs_echelle_A.pypeit.rst | 4 +- doc/include/imgproc_defaults_table.rst | 38 ++--- doc/include/keck_deimos.sorted.rst | 6 +- doc/include/keck_deimos_A.pypeit.rst | 4 +- doc/include/keck_nires_A.pypeit.rst | 4 +- doc/include/shane_kast_blue_A.pypeit.rst | 4 +- doc/pypeit_par.rst | 160 +++++++++--------- doc/scripts/build_imgproc_defaults.py | 2 +- 16 files changed, 135 insertions(+), 125 deletions(-) diff --git a/doc/calibrations/flexure.rst b/doc/calibrations/flexure.rst index c75e3c31de..a53f8b4627 100644 --- a/doc/calibrations/flexure.rst +++ b/doc/calibrations/flexure.rst @@ -26,12 +26,22 @@ Spatial Flexure The code has a simple yet relatively robust method to cross-correlate the slits against any input image to determine a rigid, spatial offset. This algorithm is performed for any frame type by setting -``spat_flexure_correct = True`` in the ``process`` block -of :ref:`processimagespar`. +``spat_flexure_method = `` in the ``process`` block +of :ref:`processimagespar`, where you should replace ```` with one of the +following valid methods: +``skip`` - this will skip the spatial flexure correction +``detector`` - this will calculate one spatial flexure value for all slits on the detector +``slit`` - this will calculate the spatial flexure for each slit separately +``edge`` - this will calculate the spatial flexure for each slit edge separately We have made our own determinations for which instruments to enable this as the default. Inspect the :ref:`instr_par` list to see if your instrument is included -(search for the value of ``spat_flexure_correct``). +(search for the value of ``spat_flexure_method``). We recommend that you use the +``detector`` method for most instruments, unless you have good reason to believe +that the spatial flexure is different for different slits or slit edges. Also note +that some spectrographs post-process the spatial flexure values when using the ``slit`` +or ``edge`` methods (e.g. perform a linear fit to the spatial flexure values) to improve +the result. Consult the documentation for your instrument to see if this is the case. Depending on what frame types you choose to correct, the code will behave somewhat differently. Here we describe @@ -49,11 +59,11 @@ add this to your PypeIt file: [scienceframe] [[process]] - spat_flexure_correct = True + spat_flexure_method = detector [calibrations] [[standardframe]] [[[process]]] - spat_flexure_correct = True + spat_flexure_method = detector This will: @@ -75,7 +85,7 @@ following to your :doc:`../pypeit_file`: [calibrations] [[tiltframe]] [[[process]]] - spat_flexure_correct = True + spat_flexure_method = detector This will: diff --git a/doc/calibrations/image_proc.rst b/doc/calibrations/image_proc.rst index b80caecf80..b6785573e0 100644 --- a/doc/calibrations/image_proc.rst +++ b/doc/calibrations/image_proc.rst @@ -288,8 +288,8 @@ Spatial Flexure Shift A spatial shift in the slit positions due to instrument flexure is calculated using :func:`~pypeit.core.flexure.spat_flexure_shift` if the -``spat_flexure_correct`` parameter is true. See :ref:`flexure` for additional -discussion. +``spat_flexure_method`` parameter is set to an option other than ``skip``. +See :ref:`flexure` for additional discussion. Flat-fielding ------------- diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index 77d8e84bbc..24424bd4c4 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -4,7 +4,7 @@ usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c] pypeit_file - ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.16.1.dev468+g832ee84e4 + ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.15.1.dev38+g075fdecaa.d20240212 ## ## Available spectrographs include: ## aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, diff --git a/doc/include/class_datamodel_pypeitimage.rst b/doc/include/class_datamodel_pypeitimage.rst index be505a3a30..ecc344bfab 100644 --- a/doc/include/class_datamodel_pypeitimage.rst +++ b/doc/include/class_datamodel_pypeitimage.rst @@ -1,5 +1,5 @@ -**Version**: 1.3.0 +**Version**: 1.3.1 ================ =================================================================================================== ================= ================================================================================================================================================================================================================================ Attribute Type Array Type Description diff --git a/doc/include/datamodel_arcimage.rst b/doc/include/datamodel_arcimage.rst index 513d279776..221c8887a8 100644 --- a/doc/include/datamodel_arcimage.rst +++ b/doc/include/datamodel_arcimage.rst @@ -1,5 +1,5 @@ -Version 1.3.0 +Version 1.3.1 ================ ============================== ========= ================================================================================================================================================ HDU Name HDU Type Data Type Description diff --git a/doc/include/datamodel_biasimage.rst b/doc/include/datamodel_biasimage.rst index fcc54ad6c9..a9710f5d4b 100644 --- a/doc/include/datamodel_biasimage.rst +++ b/doc/include/datamodel_biasimage.rst @@ -1,5 +1,5 @@ -Version 1.3.0 +Version 1.3.1 ================= ============================== ========= ================================================================================================================================================ HDU Name HDU Type Data Type Description diff --git a/doc/include/datamodel_darkimage.rst b/doc/include/datamodel_darkimage.rst index 787919fc75..e56057a0fb 100644 --- a/doc/include/datamodel_darkimage.rst +++ b/doc/include/datamodel_darkimage.rst @@ -1,5 +1,5 @@ -Version 1.3.0 +Version 1.3.1 ================= ============================== ========= ================================================================================================================================================ HDU Name HDU Type Data Type Description diff --git a/doc/include/datamodel_tiltimage.rst b/doc/include/datamodel_tiltimage.rst index 55189a8428..abf90e427f 100644 --- a/doc/include/datamodel_tiltimage.rst +++ b/doc/include/datamodel_tiltimage.rst @@ -1,5 +1,5 @@ -Version 1.3.0 +Version 1.3.1 ================= ============================== ========= ================================================================================================================================================ HDU Name HDU Type Data Type Description diff --git a/doc/include/gemini_gnirs_echelle_A.pypeit.rst b/doc/include/gemini_gnirs_echelle_A.pypeit.rst index 996d0ea8ad..9f67b44af5 100644 --- a/doc/include/gemini_gnirs_echelle_A.pypeit.rst +++ b/doc/include/gemini_gnirs_echelle_A.pypeit.rst @@ -1,7 +1,7 @@ .. code-block:: console - # Auto-generated PypeIt input file using PypeIt version: 1.16.0 - # UTC 2024-06-11T08:31:55 + # Auto-generated PypeIt input file using PypeIt version: 1.15.1.dev38+g075fdecaa.d20240212 + # UTC 2024-10-02T12:11:46.067+00:00 # User-defined execution parameters [rdx] diff --git a/doc/include/imgproc_defaults_table.rst b/doc/include/imgproc_defaults_table.rst index 190a256e99..bbeb19e0f6 100644 --- a/doc/include/imgproc_defaults_table.rst +++ b/doc/include/imgproc_defaults_table.rst @@ -1,19 +1,19 @@ -======================== ========= ========= ========= ========= ========= ========= ========= ============= ============= ======== ============ =========== -Parameter Default ``bias`` ``dark`` ``trace`` ``arc`` ``tilt`` ``align`` ``pixelflat`` ``illumflat`` ``sky`` ``standard`` ``science`` -======================== ========= ========= ========= ========= ========= ========= ========= ============= ============= ======== ============ =========== -``apply_gain`` ``True`` -``use_pattern`` ``False`` -``empirical_rn`` ``False`` -``use_overscan`` ``True`` -``trim`` ``True`` -``orient`` ``True`` -``use_biasimage`` ``True`` ``False`` -``use_darkimage`` ``False`` -``spat_flexure_correct`` ``none`` -``use_pixelflat`` ``True`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` -``use_illumflat`` ``True`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` -``use_specillum`` ``False`` -``shot_noise`` ``True`` ``False`` -``noise_floor`` ``0.0`` ``0.01`` ``0.01`` ``0.01`` -``mask_cr`` ``False`` ``True`` ``True`` ``True`` ``True`` -======================== ========= ========= ========= ========= ========= ========= ========= ============= ============= ======== ============ =========== +======================= ========= ========= ========= ========= ========= ========= ========= ============= ============= ======== ============ =========== +Parameter Default ``bias`` ``dark`` ``trace`` ``arc`` ``tilt`` ``align`` ``pixelflat`` ``illumflat`` ``sky`` ``standard`` ``science`` +======================= ========= ========= ========= ========= ========= ========= ========= ============= ============= ======== ============ =========== +``apply_gain`` ``True`` +``use_pattern`` ``False`` +``empirical_rn`` ``False`` +``use_overscan`` ``True`` +``trim`` ``True`` +``orient`` ``True`` +``use_biasimage`` ``True`` ``False`` +``use_darkimage`` ``False`` +``spat_flexure_method`` ``skip`` +``use_pixelflat`` ``True`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` +``use_illumflat`` ``True`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` ``False`` +``use_specillum`` ``False`` +``shot_noise`` ``True`` ``False`` +``noise_floor`` ``0.0`` ``0.01`` ``0.01`` ``0.01`` +``mask_cr`` ``False`` ``True`` ``True`` ``True`` ``True`` +======================= ========= ========= ========= ========= ========= ========= ========= ============= ============= ======== ============ =========== diff --git a/doc/include/keck_deimos.sorted.rst b/doc/include/keck_deimos.sorted.rst index 828519122f..655eedb3a6 100644 --- a/doc/include/keck_deimos.sorted.rst +++ b/doc/include/keck_deimos.sorted.rst @@ -14,13 +14,13 @@ d0527_0030.fits.gz | arc,tilt | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.077631 | 1.41291034 | 1.0 | 8099.98291016 | SINGLE:B | OG550 | Kr Xe Ar Ne | 2017-05-27 | 01:51:53.87 | 30 | 0 d0527_0031.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.07851 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:53:10.93 | 31 | 0 DE.20170527.06790.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.07851 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:53:10.93 | 31 | 0 - DE.20170527.06864.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.079356 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:54:24.03 | 32 | 0 d0527_0032.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.079356 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:54:24.03 | 32 | 0 - d0527_0033.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.080211 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:55:36.93 | 33 | 0 + DE.20170527.06864.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.079356 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:54:24.03 | 32 | 0 DE.20170527.06936.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.080211 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:55:36.93 | 33 | 0 + d0527_0033.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.080211 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:55:36.93 | 33 | 0 DE.20170527.37601.fits | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.435131 | 1.03078874 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:26:41.61 | 80 | 0 - d0527_0081.fits.gz | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.449842 | 1.01267696 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:47:52.92 | 81 | 0 DE.20170527.38872.fits | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.449842 | 1.01267696 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:47:52.92 | 81 | 0 + d0527_0081.fits.gz | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.449842 | 1.01267696 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:47:52.92 | 81 | 0 DE.20170527.41775.fits | science | 261.0362916666666 | 19.028888888888886 | P261_OFF | 830G | LongMirr | 1,1 | 57900.483427 | 1.00093023 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 11:36:15.35 | 83 | 0 d0527_0083.fits.gz | science | 261.0362916666666 | 19.028888888888886 | P261_OFF | 830G | LongMirr | 1,1 | 57900.483427 | 1.00093023 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 11:36:15.35 | 83 | 0 DE.20170527.43045.fits | science | 261.0362916666666 | 19.028888888888886 | P261_OFF | 830G | LongMirr | 1,1 | 57900.498135 | 1.00838805 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 11:57:25.35 | 84 | 0 diff --git a/doc/include/keck_deimos_A.pypeit.rst b/doc/include/keck_deimos_A.pypeit.rst index ba4dae4c2b..a2a9ecca04 100644 --- a/doc/include/keck_deimos_A.pypeit.rst +++ b/doc/include/keck_deimos_A.pypeit.rst @@ -1,7 +1,7 @@ .. code-block:: console - # Auto-generated PypeIt input file using PypeIt version: 1.16.0 - # UTC 2024-06-11T08:31:55 + # Auto-generated PypeIt input file using PypeIt version: 1.15.1.dev38+g075fdecaa.d20240212 + # UTC 2024-10-02T12:11:45.568+00:00 # User-defined execution parameters [rdx] diff --git a/doc/include/keck_nires_A.pypeit.rst b/doc/include/keck_nires_A.pypeit.rst index d50c6ffb58..b5129b39c9 100644 --- a/doc/include/keck_nires_A.pypeit.rst +++ b/doc/include/keck_nires_A.pypeit.rst @@ -1,7 +1,7 @@ .. code-block:: console - # Auto-generated PypeIt input file using PypeIt version: 1.16.0 - # UTC 2024-06-11T08:31:55 + # Auto-generated PypeIt input file using PypeIt version: 1.15.1.dev38+g075fdecaa.d20240212 + # UTC 2024-10-02T12:11:46.443+00:00 # User-defined execution parameters [rdx] diff --git a/doc/include/shane_kast_blue_A.pypeit.rst b/doc/include/shane_kast_blue_A.pypeit.rst index 50fed58129..6c6900edd1 100644 --- a/doc/include/shane_kast_blue_A.pypeit.rst +++ b/doc/include/shane_kast_blue_A.pypeit.rst @@ -1,7 +1,7 @@ .. code-block:: console - # Auto-generated PypeIt input file using PypeIt version: 1.16.0 - # UTC 2024-06-11T08:31:55 + # Auto-generated PypeIt input file using PypeIt version: 1.15.1.dev38+g075fdecaa.d20240212 + # UTC 2024-10-02T12:11:39.981+00:00 # User-defined execution parameters [rdx] diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index d921969ecb..9d0316bcb2 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -592,21 +592,21 @@ Collate1DPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.Collate1DPar` -========================= =============== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -========================= =============== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== -``dry_run`` bool .. False If set, the script will display the matching File and Object Ids but will not flux, coadd or archive. -``exclude_serendip`` bool .. False Whether to exclude SERENDIP objects from collating. -``exclude_slit_trace_bm`` list, str .. A list of slit trace bitmask bits that should be excluded. -``flux`` bool .. False If set, the script will flux calibrate using archived sensfuncs before coadding. -``ignore_flux`` bool .. False If set, the script will only coadd non-fluxed spectra even if flux data is present. Otherwise fluxed spectra are coadded if all spec1ds have been fluxed calibrated. -``match_using`` str .. ``ra/dec`` Determines how 1D spectra are matched as being the same object. Must be either 'pixel' or 'ra/dec'. -``outdir`` str .. ``/Users/westfall/Work/packages/pypeit/doc`` The path where all coadded output files and report files will be placed. -``refframe`` str .. .. Perform reference frame correction prior to coadding. Options are: observed, heliocentric, barycentric -``spec1d_outdir`` str .. .. The path where all modified spec1d files are placed. These are only created if flux calibration or refframe correction are asked for. -``tolerance`` str, float, int .. 1.0 The tolerance used when comparing the coordinates of objects. If two objects are within this distance from each other, they are considered the same object. If match_using is 'ra/dec' (the default) this is an angular distance. The defaults units are arcseconds but other units supported by astropy.coordinates.Angle can be used (`e.g.`, '0.003d' or '0h1m30s'). If match_using is 'pixel' this is a float. -``wv_rms_thresh`` float .. .. If set, any objects with a wavelength RMS > this value are skipped, else all wavelength RMS values are accepted. -========================= =============== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== +========================= =============== ======= ===================================== ================================================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +========================= =============== ======= ===================================== ================================================================================================================================================================================================================================================================================================================================================================================================================== +``dry_run`` bool .. False If set, the script will display the matching File and Object Ids but will not flux, coadd or archive. +``exclude_serendip`` bool .. False Whether to exclude SERENDIP objects from collating. +``exclude_slit_trace_bm`` list, str .. A list of slit trace bitmask bits that should be excluded. +``flux`` bool .. False If set, the script will flux calibrate using archived sensfuncs before coadding. +``ignore_flux`` bool .. False If set, the script will only coadd non-fluxed spectra even if flux data is present. Otherwise fluxed spectra are coadded if all spec1ds have been fluxed calibrated. +``match_using`` str .. ``ra/dec`` Determines how 1D spectra are matched as being the same object. Must be either 'pixel' or 'ra/dec'. +``outdir`` str .. ``/Users/rcooke/Software/PypeIt/doc`` The path where all coadded output files and report files will be placed. +``refframe`` str .. .. Perform reference frame correction prior to coadding. Options are: observed, heliocentric, barycentric +``spec1d_outdir`` str .. .. The path where all modified spec1d files are placed. These are only created if flux calibration or refframe correction are asked for. +``tolerance`` str, float, int .. 1.0 The tolerance used when comparing the coordinates of objects. If two objects are within this distance from each other, they are considered the same object. If match_using is 'ra/dec' (the default) this is an angular distance. The defaults units are arcseconds but other units supported by astropy.coordinates.Angle can be used (`e.g.`, '0.003d' or '0h1m30s'). If match_using is 'pixel' this is a float. +``wv_rms_thresh`` float .. .. If set, any objects with a wavelength RMS > this value are skipped, else all wavelength RMS values are accepted. +========================= =============== ======= ===================================== ================================================================================================================================================================================================================================================================================================================================================================================================================== ---- @@ -659,22 +659,22 @@ ReduxPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.ReduxPar` -====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== -``calwin`` int, float .. 0 The window of time in hours to search for calibration frames for a science frame -``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that all files were created with the current version of PypeIt. If set to False, the code will attempt to read out-of-date files and keep going. Beware (!!) that this can lead to unforeseen bugs that either cause the code to crash or lead to erroneous results. I.e., you really need to know what you are doing if you set this to False! -``detnum`` int, list .. .. Restrict reduction to a list of detector indices. In case of mosaic reduction (currently only available for Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of tuples of the detector indices that are mosaiced together. E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]`` -``ignore_bad_headers`` bool .. False Ignore bad headers (NOT recommended unless you know it is safe). -``maskIDs`` str, int, list .. .. Restrict reduction to a set of slitmask IDs Example syntax -- ``maskIDs = 818006,818015`` This must be used with detnum (for now). -``qadir`` str .. ``QA`` Directory relative to calling directory to write quality assessment files. -``quicklook`` bool .. False Run a quick look reduction? This is usually good if you want to quickly reduce the data (usually at the telescope in real time) to get an initial estimate of the data quality. -``redux_path`` str .. ``/Users/westfall/Work/packages/pypeit/doc`` Path to folder for performing reductions. Default is the current working directory. -``scidir`` str .. ``Science`` Directory relative to calling directory to write science files. -``slitspatnum`` str, list .. .. Restrict reduction to a set of slit DET:SPAT values (closest slit is used). Example syntax -- slitspatnum = DET01:175,DET01:205 or MSC02:2234 If you are re-running the code, (i.e. modifying one slit) you *must* have the precise SPAT_ID index. -``sortroot`` str .. .. A filename given to output the details of the sorted files. If None, the default is the root name of the pypeit file. If off, no output is produced. -``spectrograph`` str .. .. Spectrograph that provided the data to be reduced. See :ref:`instruments` for valid options. -====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== +====================== ============== ======= ===================================== ========================================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +====================== ============== ======= ===================================== ========================================================================================================================================================================================================================================================================================================================================================================================================== +``calwin`` int, float .. 0 The window of time in hours to search for calibration frames for a science frame +``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that all files were created with the current version of PypeIt. If set to False, the code will attempt to read out-of-date files and keep going. Beware (!!) that this can lead to unforeseen bugs that either cause the code to crash or lead to erroneous results. I.e., you really need to know what you are doing if you set this to False! +``detnum`` int, list .. .. Restrict reduction to a list of detector indices. In case of mosaic reduction (currently only available for Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of tuples of the detector indices that are mosaiced together. E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]`` +``ignore_bad_headers`` bool .. False Ignore bad headers (NOT recommended unless you know it is safe). +``maskIDs`` str, int, list .. .. Restrict reduction to a set of slitmask IDs Example syntax -- ``maskIDs = 818006,818015`` This must be used with detnum (for now). +``qadir`` str .. ``QA`` Directory relative to calling directory to write quality assessment files. +``quicklook`` bool .. False Run a quick look reduction? This is usually good if you want to quickly reduce the data (usually at the telescope in real time) to get an initial estimate of the data quality. +``redux_path`` str .. ``/Users/rcooke/Software/PypeIt/doc`` Path to folder for performing reductions. Default is the current working directory. +``scidir`` str .. ``Science`` Directory relative to calling directory to write science files. +``slitspatnum`` str, list .. .. Restrict reduction to a set of slit DET:SPAT values (closest slit is used). Example syntax -- slitspatnum = DET01:175,DET01:205 or MSC02:2234 If you are re-running the code, (i.e. modifying one slit) you *must* have the precise SPAT_ID index. +``sortroot`` str .. .. A filename given to output the details of the sorted files. If None, the default is the root name of the pypeit file. If off, no output is produced. +``spectrograph`` str .. .. Spectrograph that provided the data to be reduced. See :ref:`instruments` for valid options. +====================== ============== ======= ===================================== ========================================================================================================================================================================================================================================================================================================================================================================================================== ---- @@ -874,45 +874,45 @@ ProcessImagesPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.ProcessImagesPar` -======================== ================================================ =================================================================== ============================= ======================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -======================== ================================================ =================================================================== ============================= ======================================================================================================================================================================================================================================================================================================================================================================================== -``apply_gain`` bool .. True Convert the ADUs to electrons using the detector gain -``clip`` bool .. True Perform sigma clipping when combining. Only used with combine=mean -``comb_sigrej`` float .. .. Sigma-clipping level for when clip=True; Use None for automatic limit (recommended). -``combine`` str ``median``, ``mean`` ``mean`` Method used to combine multiple frames. Options are: median, mean -``correct_nonlinear`` list .. .. Correct for non-linear response of the detector. If None, no correction is performed. If a list, then the list should be the non-linear correction parameter (alpha), where the functional form is given by Ct = Cm (1 + alpha x Cm), with Ct and Cm the true and measured counts. This parameter is usually hard-coded for a given spectrograph, and should otherwise be left as None. -``dark_expscale`` bool .. False If designated dark frames are used and have a different exposure time than the science frames, scale the counts by the by the ratio in the exposure times to adjust the dark counts for the difference in exposure time. WARNING: You should always take dark frames that have the same exposure time as your science frames, so use this option with care! -``empirical_rn`` bool .. False If True, use the standard deviation in the overscan region to measure an empirical readnoise to use in the noise model. -``grow`` int, float .. 1.5 Factor by which to expand regions with cosmic rays detected by the LA cosmics routine. -``lamaxiter`` int .. 1 Maximum number of iterations for LA cosmics routine. -``mask_cr`` bool .. False Identify CRs and mask them -``n_lohi`` list .. 0, 0 Number of pixels to reject at the lowest and highest ends of the distribution; i.e., n_lohi = low, high. Use None for no limit. -``noise_floor`` float .. 0.0 Impose a noise floor by adding the provided fraction of the bias- and dark-subtracted electron counts to the error budget. E.g., a value of 0.01 means that the S/N of the counts in the image will never be greater than 100. -``objlim`` int, float .. 3.0 Object detection limit in LA cosmics routine -``orient`` bool .. True Orient the raw image into the PypeIt frame -``overscan_method`` str ``chebyshev``, ``polynomial``, ``savgol``, ``median``, ``odd_even`` ``savgol`` Method used to fit the overscan. Options are: chebyshev, polynomial, savgol, median, odd_even Note: Method "polynomial" is identical to "chebyshev"; the former is deprecated and will be removed. -``overscan_par`` int, list .. 5, 65 Parameters for the overscan subtraction. For 'chebyshev' or 'polynomial', set overcan_par = order; for 'savgol', set overscan_par = order, window size ; for 'median', set overscan_par = None or omit the keyword. -``rmcompact`` bool .. True Remove compact detections in LA cosmics routine -``satpix`` str ``reject``, ``force``, ``nothing`` ``reject`` Handling of saturated pixels. Options are: reject, force, nothing -``scale_to_mean`` bool .. False If True, scale the input images to have the same mean before combining. -``scattlight`` :class:`~pypeit.par.pypeitpar.ScatteredLightPar` .. `ScatteredLightPar Keywords`_ Scattered light subtraction parameters. -``shot_noise`` bool .. True Use the bias- and dark-subtracted image to calculate and include electron count shot noise in the image processing error budget -``sigclip`` int, float .. 4.5 Sigma level for rejection in LA cosmics routine -``sigfrac`` int, float .. 0.3 Fraction for the lower clipping threshold in LA cosmics routine. -``spat_flexure_correct`` str ``none``, ``detector``, ``slit``, ``edge`` ``none`` Correct slits, illumination flat, etc. for spatial flexure. Options are: none, detector, slit, edge"none" means no correction is performed. "detector" means that a single shift is applied to all slits. "slit" means that each slit is shifted independently."edge" means that each slit edge is shifted independently. -``spat_flexure_maxlag`` int .. 20 Maximum of possible spatial flexure correction, in pixels -``subtract_continuum`` bool .. False Subtract off the continuum level from an image. This parameter should only be set to True to combine arcs with multiple different lamps. For all other cases, this parameter should probably be False. -``subtract_scattlight`` bool .. False Subtract off the scattered light from an image. This parameter should only be set to True for spectrographs that have dedicated methods to subtract scattered light. For all other cases, this parameter should be False. -``trim`` bool .. True Trim the image to the detector supplied region -``use_biasimage`` bool .. True Use a bias image. If True, one or more must be supplied in the PypeIt file. -``use_darkimage`` bool .. False Subtract off a dark image. If True, one or more darks must be provided. -``use_illumflat`` bool .. True Use the illumination flat to correct for the illumination profile of each slit. -``use_overscan`` bool .. True Subtract off the overscan. Detector *must* have one or code will crash. -``use_pattern`` bool .. False Subtract off a detector pattern. This pattern is assumed to be sinusoidal along one direction, with a frequency that is constant across the detector. -``use_pixelflat`` bool .. True Use the pixel flat to make pixel-level corrections. A pixelflat image must be provied. -``use_specillum`` bool .. False Use the relative spectral illumination profiles to correct the spectral illumination profile of each slit. This is primarily used for slicer IFUs. To use this, you must set ``slit_illum_relative=True`` in the ``flatfield`` parameter set! -======================== ================================================ =================================================================== ============================= ======================================================================================================================================================================================================================================================================================================================================================================================== +======================= ================================================ =================================================================== ============================= ======================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +======================= ================================================ =================================================================== ============================= ======================================================================================================================================================================================================================================================================================================================================================================================== +``apply_gain`` bool .. True Convert the ADUs to electrons using the detector gain +``clip`` bool .. True Perform sigma clipping when combining. Only used with combine=mean +``comb_sigrej`` float .. .. Sigma-clipping level for when clip=True; Use None for automatic limit (recommended). +``combine`` str ``median``, ``mean`` ``mean`` Method used to combine multiple frames. Options are: median, mean +``correct_nonlinear`` list .. .. Correct for non-linear response of the detector. If None, no correction is performed. If a list, then the list should be the non-linear correction parameter (alpha), where the functional form is given by Ct = Cm (1 + alpha x Cm), with Ct and Cm the true and measured counts. This parameter is usually hard-coded for a given spectrograph, and should otherwise be left as None. +``dark_expscale`` bool .. False If designated dark frames are used and have a different exposure time than the science frames, scale the counts by the by the ratio in the exposure times to adjust the dark counts for the difference in exposure time. WARNING: You should always take dark frames that have the same exposure time as your science frames, so use this option with care! +``empirical_rn`` bool .. False If True, use the standard deviation in the overscan region to measure an empirical readnoise to use in the noise model. +``grow`` int, float .. 1.5 Factor by which to expand regions with cosmic rays detected by the LA cosmics routine. +``lamaxiter`` int .. 1 Maximum number of iterations for LA cosmics routine. +``mask_cr`` bool .. False Identify CRs and mask them +``n_lohi`` list .. 0, 0 Number of pixels to reject at the lowest and highest ends of the distribution; i.e., n_lohi = low, high. Use None for no limit. +``noise_floor`` float .. 0.0 Impose a noise floor by adding the provided fraction of the bias- and dark-subtracted electron counts to the error budget. E.g., a value of 0.01 means that the S/N of the counts in the image will never be greater than 100. +``objlim`` int, float .. 3.0 Object detection limit in LA cosmics routine +``orient`` bool .. True Orient the raw image into the PypeIt frame +``overscan_method`` str ``chebyshev``, ``polynomial``, ``savgol``, ``median``, ``odd_even`` ``savgol`` Method used to fit the overscan. Options are: chebyshev, polynomial, savgol, median, odd_even Note: Method "polynomial" is identical to "chebyshev"; the former is deprecated and will be removed. +``overscan_par`` int, list .. 5, 65 Parameters for the overscan subtraction. For 'chebyshev' or 'polynomial', set overcan_par = order; for 'savgol', set overscan_par = order, window size ; for 'median', set overscan_par = None or omit the keyword. +``rmcompact`` bool .. True Remove compact detections in LA cosmics routine +``satpix`` str ``reject``, ``force``, ``nothing`` ``reject`` Handling of saturated pixels. Options are: reject, force, nothing +``scale_to_mean`` bool .. False If True, scale the input images to have the same mean before combining. +``scattlight`` :class:`~pypeit.par.pypeitpar.ScatteredLightPar` .. `ScatteredLightPar Keywords`_ Scattered light subtraction parameters. +``shot_noise`` bool .. True Use the bias- and dark-subtracted image to calculate and include electron count shot noise in the image processing error budget +``sigclip`` int, float .. 4.5 Sigma level for rejection in LA cosmics routine +``sigfrac`` int, float .. 0.3 Fraction for the lower clipping threshold in LA cosmics routine. +``spat_flexure_maxlag`` int .. 20 Maximum of possible spatial flexure correction, in pixels +``spat_flexure_method`` str ``skip``, ``detector``, ``slit``, ``edge`` ``skip`` Correct slits, illumination flat, etc. for spatial flexure. Options are: skip, detector, slit, edge"skip" means no correction is performed. "detector" means that a single shift is applied to all slits. "slit" means that each slit is shifted independently."edge" means that each slit edge is shifted independently. +``subtract_continuum`` bool .. False Subtract off the continuum level from an image. This parameter should only be set to True to combine arcs with multiple different lamps. For all other cases, this parameter should probably be False. +``subtract_scattlight`` bool .. False Subtract off the scattered light from an image. This parameter should only be set to True for spectrographs that have dedicated methods to subtract scattered light. For all other cases, this parameter should be False. +``trim`` bool .. True Trim the image to the detector supplied region +``use_biasimage`` bool .. True Use a bias image. If True, one or more must be supplied in the PypeIt file. +``use_darkimage`` bool .. False Subtract off a dark image. If True, one or more darks must be provided. +``use_illumflat`` bool .. True Use the illumination flat to correct for the illumination profile of each slit. +``use_overscan`` bool .. True Subtract off the overscan. Detector *must* have one or code will crash. +``use_pattern`` bool .. False Subtract off a detector pattern. This pattern is assumed to be sinusoidal along one direction, with a frequency that is constant across the detector. +``use_pixelflat`` bool .. True Use the pixel flat to make pixel-level corrections. A pixelflat image must be provied. +``use_specillum`` bool .. False Use the relative spectral illumination profiles to correct the spectral illumination profile of each slit. This is primarily used for slicer IFUs. To use this, you must set ``slit_illum_relative=True`` in the ``flatfield`` parameter set! +======================= ================================================ =================================================================== ============================= ======================================================================================================================================================================================================================================================================================================================================================================================== ---- @@ -3497,7 +3497,7 @@ Alterations to the default parameters are: [[[process]]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = detector + spat_flexure_method = detector [[flatfield]] slit_illum_finecorr = False [[wavelengths]] @@ -3518,7 +3518,7 @@ Alterations to the default parameters are: [[process]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = detector + spat_flexure_method = detector [flexure] spec_method = boxcar [sensfunc] @@ -3609,7 +3609,7 @@ Alterations to the default parameters are: [[[process]]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = detector + spat_flexure_method = detector [[flatfield]] slit_illum_finecorr = False [[wavelengths]] @@ -3630,7 +3630,7 @@ Alterations to the default parameters are: [[process]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = detector + spat_flexure_method = detector [flexure] spec_method = boxcar [sensfunc] @@ -3721,7 +3721,7 @@ Alterations to the default parameters are: [[[process]]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = detector + spat_flexure_method = detector [[flatfield]] slit_illum_finecorr = False [[wavelengths]] @@ -3748,7 +3748,7 @@ Alterations to the default parameters are: sigclip = 5.0 objlim = 5.0 noise_floor = 0.01 - spat_flexure_correct = detector + spat_flexure_method = detector [reduce] [[skysub]] bspline_spacing = 0.8 @@ -3844,7 +3844,7 @@ Alterations to the default parameters are: [[[process]]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = detector + spat_flexure_method = detector [[flatfield]] slit_illum_finecorr = False [[wavelengths]] @@ -3871,7 +3871,7 @@ Alterations to the default parameters are: sigclip = 5.0 objlim = 5.0 noise_floor = 0.01 - spat_flexure_correct = detector + spat_flexure_method = detector [reduce] [[skysub]] bspline_spacing = 0.8 @@ -3967,7 +3967,7 @@ Alterations to the default parameters are: [[[process]]] mask_cr = True noise_floor = 0.01 - spat_flexure_correct = detector + spat_flexure_method = detector [[flatfield]] slit_illum_finecorr = False [[wavelengths]] @@ -3994,7 +3994,7 @@ Alterations to the default parameters are: sigclip = 5.0 objlim = 5.0 noise_floor = 0.01 - spat_flexure_correct = detector + spat_flexure_method = detector [reduce] [[skysub]] bspline_spacing = 0.8 diff --git a/doc/scripts/build_imgproc_defaults.py b/doc/scripts/build_imgproc_defaults.py index 7d43f365e8..a4c5eadcf2 100644 --- a/doc/scripts/build_imgproc_defaults.py +++ b/doc/scripts/build_imgproc_defaults.py @@ -25,7 +25,7 @@ def write_imgproc_def_table(ofile, spec=None): 'orient', 'use_biasimage', 'use_darkimage', - 'spat_flexure_correct', + 'spat_flexure_method', 'use_pixelflat', 'use_illumflat', 'use_specillum', From 16847ef9df7f921a3f9dd98c9f01f653371ece11 Mon Sep 17 00:00:00 2001 From: rcooke Date: Wed, 2 Oct 2024 13:46:42 +0100 Subject: [PATCH 38/54] update for flexure refactor --- deprecated/ql_keck_lris.py | 2 +- deprecated/ql_multislit.py | 2 +- pypeit/extraction.py | 6 +++--- pypeit/find_objects.py | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/deprecated/ql_keck_lris.py b/deprecated/ql_keck_lris.py index a27f4bdf32..84d5474fe0 100644 --- a/deprecated/ql_keck_lris.py +++ b/deprecated/ql_keck_lris.py @@ -59,7 +59,7 @@ def config_lines(args): cfg_lines += [' refframe = observed'] cfg_lines += ['[scienceframe]'] cfg_lines += [' [[process]]'] - cfg_lines += [' spat_flexure_correct = False'] + cfg_lines += [' spat_flexure_method = skip'] if not args.mask_cr: cfg_lines += [' mask_cr = False'] cfg_lines += ['[reduce]'] diff --git a/deprecated/ql_multislit.py b/deprecated/ql_multislit.py index fb5780f138..00867887b3 100644 --- a/deprecated/ql_multislit.py +++ b/deprecated/ql_multislit.py @@ -75,7 +75,7 @@ def config_lines(args): cfg_lines += [' refframe = observed'] cfg_lines += ['[scienceframe]'] cfg_lines += [' [[process]]'] - cfg_lines += [' spat_flexure_correct = False'] + cfg_lines += [' spat_flexure_method = skip'] if not args.mask_cr: cfg_lines += [' mask_cr = False'] cfg_lines += ['[reduce]'] diff --git a/pypeit/extraction.py b/pypeit/extraction.py index 805af69aa5..8587cf1d00 100644 --- a/pypeit/extraction.py +++ b/pypeit/extraction.py @@ -162,8 +162,8 @@ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_ # frames. Is that okay for this usage? # Flexure self.spat_flexure_shift = None - if (objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_correct']) or \ - (objtype == 'standard' and self.par['calibrations']['standardframe']['process']['spat_flexure_correct']): + if (objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_method'] != "skip") or \ + (objtype == 'standard' and self.par['calibrations']['standardframe']['process']['spat_flexure_method'] != "skip"): self.spat_flexure_shift = self.sciImg.spat_flexure elif objtype == 'science_coadd2d': self.spat_flexure_shift = None @@ -216,7 +216,7 @@ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_ self.waveTilts = waveTilts self.waveTilts.is_synced(self.slits) # Deal with Flexure - if self.par['calibrations']['tiltframe']['process']['spat_flexure_correct']: + if self.par['calibrations']['tiltframe']['process']['spat_flexure_method'] != "skip": _spat_flexure = 0. if self.spat_flexure_shift is None else self.spat_flexure_shift # If they both shifted the same, there will be no reason to shift the tilts tilt_flexure_shift = _spat_flexure - self.waveTilts.spat_flexure diff --git a/pypeit/find_objects.py b/pypeit/find_objects.py index 156f303742..53d2558a16 100644 --- a/pypeit/find_objects.py +++ b/pypeit/find_objects.py @@ -158,8 +158,8 @@ def __init__(self, sciImg, slits, spectrograph, par, objtype, wv_calib=None, wav # frames. Is that okay for this usage? # Flexure self.spat_flexure_shift = None - if (objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_correct'] != "none") or \ - (objtype == 'standard' and self.par['calibrations']['standardframe']['process']['spat_flexure_correct'] != "none"): + if (objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_method'] != "skip") or \ + (objtype == 'standard' and self.par['calibrations']['standardframe']['process']['spat_flexure_method'] != "skip"): self.spat_flexure_shift = self.sciImg.spat_flexure elif objtype == 'science_coadd2d': self.spat_flexure_shift = None @@ -227,7 +227,7 @@ def __init__(self, sciImg, slits, spectrograph, par, objtype, wv_calib=None, wav self.waveTilts = waveTilts self.waveTilts.is_synced(self.slits) # Deal with Flexure - if self.par['calibrations']['tiltframe']['process']['spat_flexure_correct'] != "none": + if self.par['calibrations']['tiltframe']['process']['spat_flexure_method'] != "skip": _spat_flexure = np.zeros((slits.nslits, 2)) if self.spat_flexure_shift is None \ else self.spat_flexure_shift # If they both shifted the same, there will be no reason to shift the tilts From 2f3eb138134fe81f1d4d8cd8817ad01b3334bf10 Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 10 Oct 2024 09:35:46 +0100 Subject: [PATCH 39/54] masked flexure --- pypeit/inputfiles.py | 8 +-- pypeit/metadata.py | 15 +++--- pypeit/pypeit.py | 110 ++++++++++++++++++++-------------------- pypeit/scripts/setup.py | 2 +- 4 files changed, 67 insertions(+), 68 deletions(-) diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index 52a41f7c11..e7e1e96749 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -380,14 +380,14 @@ def _read_data_file_table(lines, preserve_comments): ## Recast each as "object" in case the user has mucked with the Table ## e.g. a mix of floats and None - ## Also handle Masked columns -- fill with '' for key in tbl.keys(): # Object tbl[key] = tbl[key].data.astype(object) - if isinstance(tbl[key], column.MaskedColumn): + # RJC -- Not sure why we need to fill masked columns with empty data. Let's just retain the masked values. + # if isinstance(tbl[key], column.MaskedColumn): # Fill with empty string - tbl[key].fill_value = '' - tbl[key] = tbl[key].filled() + # tbl[key].fill_value = np.ma.masked + # tbl[key] = tbl[key].filled() # Build the table -- Old code # Because we allow (even encourage!) the users to modify entries by hand, diff --git a/pypeit/metadata.py b/pypeit/metadata.py index f38ce78fd2..dfca3ada3f 100644 --- a/pypeit/metadata.py +++ b/pypeit/metadata.py @@ -665,10 +665,6 @@ def n_configs(self): 'unique_configurations first.') return len(list(self.configs.keys())) - @property - def MASKED_VALUE(self): - return -9999 - def unique_configurations(self, force=False, copy=False, rm_none=False): """ Return the unique instrument configurations. @@ -1544,7 +1540,7 @@ def set_combination_groups(self, assign_objects=True): if 'bkg_id' not in self.keys(): self['bkg_id'] = -1 if 'shift' not in self.keys(): - self['shift'] = self.MASKED_VALUE + self['shift'] = np.ma.masked # NOTE: Importantly, this if statement means that, if the user has # defined any non-negative combination IDs in their pypeit file, none of @@ -1580,7 +1576,8 @@ def set_user_added_columns(self): if 'manual' not in self.keys(): self['manual'] = '' if 'shift' not in self.keys(): - self['shift'] = self.MASKED_VALUE + # Instantiate a masked array + self['shift'] = np.ma.array(np.zeros(len(self)), mask=np.ones(len(self), dtype=bool)) def write_sorted(self, ofile, overwrite=True, ignore=None, write_bkg_pairs=False, write_manual=False): @@ -1665,7 +1662,7 @@ def write_sorted(self, ofile, overwrite=True, ignore=None, def write_pypeit(self, output_path=None, cfg_lines=None, write_bkg_pairs=False, write_manual=False, - write_shift = False, + write_shift=False, configs=None, config_subdir=True, version_override=None, date_override=None): """ @@ -1745,7 +1742,7 @@ def write_pypeit(self, output_path=None, cfg_lines=None, # Grab output columns output_cols = self.set_pypeit_cols(write_bkg_pairs=write_bkg_pairs, write_manual=write_manual, - write_shift = write_shift) + write_shift=write_shift) # Write the pypeit files ofiles = [None]*len(cfg_keys) @@ -1797,7 +1794,7 @@ def write_pypeit(self, output_path=None, cfg_lines=None, pypeItFile = inputfiles.PypeItFile(cfg_lines, paths, subtbl, setup_dict) # Write pypeItFile.write(ofiles[j], version_override=version_override, - date_override=date_override) + date_override=date_override) # Return return ofiles diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index 824924dbd2..5904ad1ccb 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -17,6 +17,7 @@ __UTC__ = datetime.UTC except AttributeError as e: from datetime import timezone + __UTC__ = timezone.utc from IPython import embed @@ -38,12 +39,12 @@ from pypeit import extraction from pypeit import spec2dobj from pypeit import specobjs -#from pypeit.spectrographs.util import load_spectrograph +# from pypeit.spectrographs.util import load_spectrograph from pypeit import slittrace from pypeit import utils from pypeit.history import History -#from pypeit.par import PypeItPar -#from pypeit.par.pypeitpar import ql_is_on +# from pypeit.par import PypeItPar +# from pypeit.par.pypeitpar import ql_is_on from pypeit.metadata import PypeItMetaData from pypeit.manual_extract import ManualExtractionObj from pypeit.core import skysub @@ -92,6 +93,7 @@ class PypeIt: fitstbl (:obj:`pypeit.metadata.PypeItMetaData`): holds the meta info """ + def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_calibs=False, logname=None, show=False, redux_path=None, calib_only=False): @@ -99,9 +101,9 @@ def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_calibs=False, self.logname = logname self.verbosity = verbosity self.pypeit_file = pypeit_file - + self.msgs_reset() - + # Load up PypeIt file self.pypeItFile = inputfiles.PypeItFile.from_file(pypeit_file) self.calib_only = calib_only @@ -126,14 +128,14 @@ def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_calibs=False, # Build the meta data # - Re-initilize based on the file data msgs.info('Compiling metadata') - self.fitstbl = PypeItMetaData(self.spectrograph, self.par, + self.fitstbl = PypeItMetaData(self.spectrograph, self.par, files=self.pypeItFile.filenames, - usrdata=self.pypeItFile.data, + usrdata=self.pypeItFile.data, strict=True) # - Interpret automated or user-provided data from the PypeIt # file self.fitstbl.finalize_usr_build( - self.pypeItFile.frametypes, + self.pypeItFile.frametypes, self.pypeItFile.setup_name) # Other Internals @@ -261,7 +263,7 @@ def get_std_outfile(self, standard_frames): # Prepare to load up standard? if std_frame is not None: std_outfile = self.spec_output_file(std_frame) \ - if isinstance(std_frame, (int,np.integer)) else None + if isinstance(std_frame, (int, np.integer)) else None if std_outfile is not None and not os.path.isfile(std_outfile): msgs.error('Could not find standard file: {0}'.format(std_outfile)) return std_outfile @@ -399,21 +401,21 @@ def reduce_all(self): std_outfile = self.get_std_outfile(frame_indx[is_standard]) # Reduce all the science frames; keep the basenames of the science # frames for use in flux calibration - science_basename = [None]*len(grp_science) + science_basename = [None] * len(grp_science) # Loop on unique comb_id u_combid = np.unique(self.fitstbl['comb_id'][grp_science]) - + for j, comb_id in enumerate(u_combid): # TODO: This was causing problems when multiple science frames # were provided to quicklook and the user chose *not* to stack # the frames. But this means it now won't skip processing the # B-A pair when the background image(s) are defined. Punting # for now... -# # Quicklook mode? -# if self.par['rdx']['quicklook'] and j > 0: -# msgs.warn('PypeIt executed in quicklook mode. Only reducing science frames ' -# 'in the first combination group!') -# break + # # Quicklook mode? + # if self.par['rdx']['quicklook'] and j > 0: + # msgs.warn('PypeIt executed in quicklook mode. Only reducing science frames ' + # 'in the first combination group!') + # break # frames = np.where(self.fitstbl['comb_id'] == comb_id)[0] # Find all frames whose comb_id matches the current frames bkg_id. @@ -424,7 +426,7 @@ def reduce_all(self): # syntax below would require that we could somehow list multiple # numbers for the bkg_id which is impossible without a comma # separated list -# bg_frames = np.where(self.fitstbl['bkg_id'] == comb_id)[0] + # bg_frames = np.where(self.fitstbl['bkg_id'] == comb_id)[0] if not self.outfile_exists(frames[0]) or self.overwrite: # Build history to document what contributd to the reduced @@ -515,11 +517,11 @@ def reduce_exposure(self, frames, bg_frames=None, std_outfile=None): # something other than the default of None. self.find_negative = (('science' in self.fitstbl['frametype'][bg_frames[0]]) | ('standard' in self.fitstbl['frametype'][bg_frames[0]])) \ - if self.par['reduce']['findobj']['find_negative'] is None else \ - self.par['reduce']['findobj']['find_negative'] + if self.par['reduce']['findobj']['find_negative'] is None else \ + self.par['reduce']['findobj']['find_negative'] else: self.bkg_redux = False - self.find_negative= False + self.find_negative = False # Container for all the Spec2DObj all_spec2d = spec2dobj.AllSpec2DObj() @@ -585,7 +587,7 @@ def reduce_exposure(self, frames, bg_frames=None, std_outfile=None): # global_sky, skymask and sciImg are needed in the extract loop initial_sky, sobjs_obj, sciImg, bkg_redux_sciimg, objFind = self.objfind_one( frames, self.det, bg_frames=bg_frames, std_outfile=std_outfile) - if len(sobjs_obj)>0: + if len(sobjs_obj) > 0: all_specobjs_objfind.add_sobj(sobjs_obj) initial_sky_list.append(initial_sky) sciImg_list.append(sciImg) @@ -598,7 +600,7 @@ def reduce_exposure(self, frames, bg_frames=None, std_outfile=None): spat_flexure = [ss.spat_flexure for ss in sciImg_list] # Grab platescale with binning bin_spec, bin_spat = parse.parse_binning(self.binning) - platescale = np.array([ss.detector.platescale*bin_spat for ss in sciImg_list]) + platescale = np.array([ss.detector.platescale * bin_spat for ss in sciImg_list]) # get the dither offset if available and if desired dither_off = None if self.par['reduce']['slitmask']['use_dither_offset']: @@ -635,18 +637,18 @@ def reduce_exposure(self, frames, bg_frames=None, std_outfile=None): # Extract all_spec2d[detname], tmp_sobjs \ - = self.extract_one(frames, self.det, sciImg_list[i], bkg_redux_sciimg_list[i], objFind_list[i], - initial_sky_list[i], all_specobjs_on_det) + = self.extract_one(frames, self.det, sciImg_list[i], bkg_redux_sciimg_list[i], objFind_list[i], + initial_sky_list[i], all_specobjs_on_det) # Hold em if tmp_sobjs.nobj > 0: all_specobjs_extract.add_sobj(tmp_sobjs) # Add calibration associations to the SpecObjs object all_specobjs_extract.calibs = calibrations.Calibrations.get_association( - self.fitstbl, self.spectrograph, self.calibrations_path, - self.fitstbl[frames[0]]['setup'], - self.fitstbl.find_frame_calib_groups(frames[0])[0], self.det, - must_exist=True, proc_only=True) + self.fitstbl, self.spectrograph, self.calibrations_path, + self.fitstbl[frames[0]]['setup'], + self.fitstbl.find_frame_calib_groups(frames[0])[0], self.det, + must_exist=True, proc_only=True) # JFH TODO write out the background frame? @@ -675,9 +677,9 @@ def get_sci_metadata(self, frame, det): # Set binning, obstime, basename, and objtype binning = self.fitstbl['binning'][frame] - obstime = self.fitstbl.construct_obstime(frame) + obstime = self.fitstbl.construct_obstime(frame) basename = self.fitstbl.construct_basename(frame, obstime=obstime) - types = self.fitstbl['frametype'][frame].split(',') + types = self.fitstbl['frametype'][frame].split(',') if 'science' in types: objtype_out = 'science' elif 'standard' in types: @@ -757,14 +759,14 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): """ # Grab some meta-data needed for the reduction from the fitstbl self.objtype, self.setup, self.obstime, self.basename, self.binning \ - = self.get_sci_metadata(frames[0], det) + = self.get_sci_metadata(frames[0], det) msgs.info("Object finding begins for {} on det={}".format(self.basename, det)) # Is this a standard star? self.std_redux = self.objtype == 'standard' frame_par = self.par['calibrations']['standardframe'] \ - if self.std_redux else self.par['scienceframe'] + if self.std_redux else self.par['scienceframe'] # Get the standard trace if need be if self.std_redux is False and std_outfile is not None: @@ -816,8 +818,9 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): manual_flexure = self.fitstbl[frames[0]]['shift'] if (self.objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_method'] != "skip") or \ (self.objtype == 'standard' and self.par['calibrations']['standardframe']['process']['spat_flexure_method'] != "skip") or \ - manual_flexure: - if (manual_flexure != self.fitstbl.MASKED_VALUE) and np.issubdtype(self.fitstbl[frames[0]]["shift"], np.integer): + not np.ma.is_masked(manual_flexure): + # First check for manual flexure + if not np.ma.is_masked(manual_flexure): msgs.info(f'Implementing manual spatial flexure of {manual_flexure} pixels') spat_flexure = np.full((self.caliBrate.slits.nslits, 2), np.float64(manual_flexure)) sciImg.spat_flexure = spat_flexure @@ -836,7 +839,8 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): else: # Print the flexure values for each slit separately for slit in range(spat_flexure.shape[0]): - msgs.info(f'Spatial flexure for slit {self.caliBrate.slits.spat_id[slit]} is: left={spat_flexure[slit, 0]} right={spat_flexure[slit, 1]}') + msgs.info( + f'Spatial flexure for slit {self.caliBrate.slits.spat_id[slit]} is: left={spat_flexure[slit, 0]} right={spat_flexure[slit, 1]}') # Build the initial sky mask initial_skymask = self.load_skyregions(initial_slits=self.spectrograph.pypeline != 'SlicerIFU', scifile=sciImg.files[0], frame=frames[0], spat_flexure=spat_flexure) @@ -925,9 +929,9 @@ def load_skyregions(self, initial_slits=False, scifile=None, frame=None, spat_fl if self.par['reduce']['skysub']['user_regions'] == 'user': # Build the file name calib_key = CalibFrame.construct_calib_key( - self.fitstbl['setup'][frame], - CalibFrame.ingest_calib_id(self.fitstbl['calib'][frame]), - self.spectrograph.get_det_name(self.det)) + self.fitstbl['setup'][frame], + CalibFrame.ingest_calib_id(self.fitstbl['calib'][frame]), + self.spectrograph.get_det_name(self.det)) regfile = buildimage.SkyRegions.construct_file_name(calib_key, calib_dir=self.calibrations_path, basename=io.remove_suffix(scifile)) @@ -948,7 +952,7 @@ def load_skyregions(self, initial_slits=False, scifile=None, frame=None, spat_fl slits_left, slits_right, _ \ = self.caliBrate.slits.select_edges(initial=initial_slits, spat_flexure=None) - maxslitlength = np.max(slits_right-slits_left) + maxslitlength = np.max(slits_right - slits_left) # Get the regions status, regions = skysub.read_userregions(skyregtxt, self.caliBrate.slits.nslits, maxslitlength) if status == 1: @@ -998,7 +1002,7 @@ def extract_one(self, frames, det, sciImg, bkg_redux_sciimg, objFind, initial_sk """ # Grab some meta-data needed for the reduction from the fitstbl self.objtype, self.setup, self.obstime, self.basename, self.binning \ - = self.get_sci_metadata(frames[0], det) + = self.get_sci_metadata(frames[0], det) # Is this a standard star? self.std_redux = 'standard' in self.objtype @@ -1026,7 +1030,7 @@ def extract_one(self, frames, det, sciImg, bkg_redux_sciimg, objFind, initial_sk skymask = objFind.create_skymask(sobjs_obj) if skymask is None else skymask # DO NOT reinit_bpm, nor update_crmask bkg_redux_global_sky = objFind.global_skysub(skymask=skymask, bkg_redux_sciimg=bkg_redux_sciimg, - reinit_bpm=False, update_crmask=False, show=self.show) + reinit_bpm=False, update_crmask=False, show=self.show) scaleImg = objFind.scaleimg @@ -1089,7 +1093,7 @@ def extract_one(self, frames, det, sciImg, bkg_redux_sciimg, objFind, initial_sk # Tack on wavelength RMS for sobj in sobjs: iwv = np.where(self.caliBrate.wv_calib.spat_ids == sobj.SLITID)[0][0] - sobj.WAVE_RMS =self.caliBrate.wv_calib.wv_fits[iwv].rms + sobj.WAVE_RMS = self.caliBrate.wv_calib.wv_fits[iwv].rms # Construct table of spectral flexure spec_flex_table = Table() @@ -1118,10 +1122,10 @@ def extract_one(self, frames, det, sciImg, bkg_redux_sciimg, objFind, initial_sk spec2DObj.process_steps = sciImg.process_steps spec2DObj.calibs = calibrations.Calibrations.get_association( - self.fitstbl, self.spectrograph, self.calibrations_path, - self.fitstbl[frames[0]]['setup'], - self.fitstbl.find_frame_calib_groups(frames[0])[0], det, - must_exist=True, proc_only=True) + self.fitstbl, self.spectrograph, self.calibrations_path, + self.fitstbl[frames[0]]['setup'], + self.fitstbl.find_frame_calib_groups(frames[0])[0], det, + must_exist=True, proc_only=True) # QA spec2DObj.gen_qa() @@ -1227,8 +1231,8 @@ def save_exposure(self, frame, all_spec2d, all_specobjs, basename, history=None) if self.par['rdx']['detnum'] is None: update_det = None elif isinstance(self.par['rdx']['detnum'], list): - update_det = [self.spectrograph.allowed_mosaics.index(d)+1 - if isinstance(d, tuple) else d for d in self.par['rdx']['detnum']] + update_det = [self.spectrograph.allowed_mosaics.index(d) + 1 + if isinstance(d, tuple) else d for d in self.par['rdx']['detnum']] else: update_det = self.par['rdx']['detnum'] @@ -1238,7 +1242,7 @@ def save_exposure(self, frame, all_spec2d, all_specobjs, basename, history=None) # Spectra outfile1d = os.path.join(self.science_path, 'spec1d_{:s}.fits'.format(basename)) # TODO - #embed(header='deal with the following for maskIDs; 713 of pypeit') + # embed(header='deal with the following for maskIDs; 713 of pypeit') all_specobjs.write_to_fits(subheader, outfile1d, update_det=update_det, slitspatnum=self.par['rdx']['slitspatnum'], @@ -1251,7 +1255,7 @@ def save_exposure(self, frame, all_spec2d, all_specobjs, basename, history=None) # option is to re-work write_info to also "append" sobjs = specobjs.SpecObjs.from_fitsfile(outfile1d, chk_version=False) sobjs.write_info(outfiletxt, self.spectrograph.pypeline) - #all_specobjs.write_info(outfiletxt, self.spectrograph.pypeline) + # all_specobjs.write_info(outfiletxt, self.spectrograph.pypeline) # 2D spectra outfile2d = os.path.join(self.science_path, 'spec2d_{:s}.fits'.format(basename)) @@ -1281,7 +1285,7 @@ def print_end_time(self): Print the elapsed time """ # Capture the end time and print it to user - msgs.info(utils.get_time_string(time.perf_counter()-self.tstart)) + msgs.info(utils.get_time_string(time.perf_counter() - self.tstart)) # TODO: Move this to fitstbl? def show_science(self): @@ -1289,10 +1293,8 @@ def show_science(self): Simple print of science frames """ indx = self.fitstbl.find_frames('science') - print(self.fitstbl[['target','ra','dec','exptime','dispname']][indx]) + print(self.fitstbl[['target', 'ra', 'dec', 'exptime', 'dispname']][indx]) def __repr__(self): # Generate sets string return '<{:s}: pypeit_file={}>'.format(self.__class__.__name__, self.pypeit_file) - - diff --git a/pypeit/scripts/setup.py b/pypeit/scripts/setup.py index f417caf172..67163e12ad 100644 --- a/pypeit/scripts/setup.py +++ b/pypeit/scripts/setup.py @@ -150,7 +150,7 @@ def main(args): pypeit_files = ps.fitstbl.write_pypeit(output_path=output_path, cfg_lines=ps.user_cfg, write_bkg_pairs=args.background, write_manual=args.manual_extraction, - write_shift = args.flexure, + write_shift=args.flexure, configs=configs, version_override=args.version_override, date_override=args.date_override) From 731bb3fb3da2e2da44505f4a49960c91410756ac Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 10 Oct 2024 20:15:40 +0100 Subject: [PATCH 40/54] manual flexure --- pypeit/images/rawimage.py | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 41907a8fca..4009b1fda3 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -401,7 +401,7 @@ def build_rn2img(self, units='e-', digitization=False): return np.array(rn2) def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, slits=None, dark=None, - mosaic=False, debug=False): + mosaic=False, manual_spat_flexure=None, debug=False): """ Process the data. @@ -671,9 +671,11 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl # bias and dark subtraction) and before field flattening. Also the # function checks that the slits exist if running the spatial flexure # correction, so no need to do it again here. - self.spat_flexure_shift = self.spatial_flexure_shift(slits, method=self.par['spat_flexure_method'], - maxlag=self.par['spat_flexure_maxlag']) \ - if self.par['spat_flexure_method'] != "skip" else None + self.spat_flexure_shift = None + if self.par['spat_flexure_method'] != "skip" or not np.ma.is_masked(manual_spat_flexure): + self.spat_flexure_shift = self.spatial_flexure_shift(slits, method=self.par['spat_flexure_method'], + manual_spat_flexure=manual_spat_flexure, + maxlag=self.par['spat_flexure_maxlag']) # - Subtract scattered light... this needs to be done before flatfielding. if self.par['subtract_scattlight']: @@ -766,7 +768,7 @@ def _squeeze(self): return _det, self.image, self.ivar, self.datasec_img, self.det_img, self.rn2img, \ self.base_var, self.img_scale, self.bpm - def spatial_flexure_shift(self, slits, force=False, method="detector", maxlag=20): + def spatial_flexure_shift(self, slits, force=False, manual_spat_flexure=np.ma.masked, method="detector", maxlag=20): """ Calculate a spatial shift in the edge traces due to flexure. @@ -779,6 +781,13 @@ def spatial_flexure_shift(self, slits, force=False, method="detector", maxlag=20 force (:obj:`bool`, optional): Force the image to be field flattened, even if the step log (:attr:`steps`) indicates that it already has been. + manual_spat_flexure (:obj:`float`, optional): + Manually set the spatial flexure shift. If provided, this + value is used instead of calculating the shift. The default + value is `np.ma.masked`, which means the shift is calculated + from the image data. The only way this value is used is if + the user sets the `shift` parameter in their pypeit file to + be a float. method (:obj:`str`, optional): Method to use to calculate the spatial flexure shift. Options are 'detector' (default), 'slit', and 'edge'. The 'detector' @@ -801,10 +810,26 @@ def spatial_flexure_shift(self, slits, force=False, method="detector", maxlag=20 if self.nimg > 1: msgs.error('CODING ERROR: Must use a single image (single detector or detector ' 'mosaic) to determine spatial flexure.') - self.spat_flexure_shift = flexure.spat_flexure_shift(self.image[0], slits, method=method, maxlag=maxlag) + + # First check for manual flexure + if not np.ma.is_masked(manual_spat_flexure): + msgs.info(f'Adopting a manual spatial flexure of {manual_spat_flexure} pixels') + spat_flexure = np.full((slits.nslits, 2), np.float64(manual_spat_flexure)) + else: + spat_flexure = flexure.spat_flexure_shift(self.image[0], slits, method=method, maxlag=maxlag) + + # Print the flexure values + if np.all(spat_flexure == spat_flexure[0, 0]): + msgs.info(f'Spatial flexure is: {spat_flexure[0, 0]}') + else: + # Print the flexure values for each slit separately + for slit in range(spat_flexure.shape[0]): + msgs.info( + f'Spatial flexure for slit {slits.spat_id[slit]} is: left={spat_flexure[slit, 0]} right={spat_flexure[slit, 1]}') + self.steps[step] = True # Return - return self.spat_flexure_shift + return spat_flexure def flatfield(self, flatimages, slits=None, force=False, debug=False): """ From 09c54973212240728c4bff1642e1f5c11f962192 Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 10 Oct 2024 20:16:28 +0100 Subject: [PATCH 41/54] mv manual flexure to rawimage --- pypeit/images/buildimage.py | 7 ++++--- pypeit/metadata.py | 23 +++++++++++++++++++++- pypeit/pypeit.py | 39 +++++++------------------------------ 3 files changed, 33 insertions(+), 36 deletions(-) diff --git a/pypeit/images/buildimage.py b/pypeit/images/buildimage.py index b09581ed56..760be750d8 100644 --- a/pypeit/images/buildimage.py +++ b/pypeit/images/buildimage.py @@ -160,7 +160,7 @@ def construct_file_name(cls, calib_key, calib_dir=None, basename=None): def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm=None, dark=None, scattlight=None, flatimages=None, maxiters=5, ignore_saturation=True, - slits=None, mosaic=None, calib_dir=None, setup=None, calib_id=None): + slits=None, mosaic=None, manual_spat_flexure=None, calib_dir=None, setup=None, calib_id=None): """ Perform basic image processing on a list of images and combine the results. All core processing steps for each image are handled by :class:`~pypeit.images.rawimage.RawImage` and @@ -254,13 +254,14 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm= rawImage_list = [] # Loop on the files - for ifile in file_list: + for ii, ifile in enumerate(file_list): # Load raw image rawImage = rawimage.RawImage(ifile, spectrograph, det) # Process rawImage_list.append(rawImage.process( frame_par['process'], scattlight=scattlight, bias=bias, - bpm=bpm, dark=dark, flatimages=flatimages, slits=slits, mosaic=mosaic)) + bpm=bpm, dark=dark, flatimages=flatimages, slits=slits, mosaic=mosaic, + manual_spat_flexure=manual_spat_flexure[ii])) # Do it combineImage = combineimage.CombineImage(rawImage_list, frame_par['process']) diff --git a/pypeit/metadata.py b/pypeit/metadata.py index dfca3ada3f..366466b6b9 100644 --- a/pypeit/metadata.py +++ b/pypeit/metadata.py @@ -1309,10 +1309,31 @@ def frame_paths(self, indx): Returns: list: List of the full paths of one or more frames. """ - if isinstance(indx, (int,np.integer)): + if isinstance(indx, (int, np.integer)): return os.path.join(self['directory'][indx], self['filename'][indx]) return [os.path.join(d,f) for d,f in zip(self['directory'][indx], self['filename'][indx])] + def get_shifts(self, indx): + """ + Return the shifts for the provided rows. + + Args: + indx (:obj:`int`, array-like): + One or more 0-indexed rows in the table with the frames + to return. Can be an array of indices or a boolean + array of the correct length. + + Returns: + `numpy.ndarray`_: Array with the shifts. + """ + # Make indx an array + _indx = np.atleast_1d(indx) + # Check if shifts are defined, if not, return a masked array + if 'shift' not in self.keys(): + return np.ma.array(np.zeros(_indx.shape), mask=np.ones(_indx.shape, dtype=bool)) + # Otherwise, return the shifts + return self['shift'][indx] + def set_frame_types(self, type_bits, merge=True): """ Set and return a Table with the frame types and bits. diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index 5904ad1ccb..08b5319443 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -776,6 +776,7 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): # Build Science image sci_files = self.fitstbl.frame_paths(frames) + manual_spat_flexure = self.fitstbl.get_shifts(frames) sciImg = buildimage.buildimage_fromlist( self.spectrograph, det, frame_par, sci_files, bias=self.caliBrate.msbias, bpm=self.caliBrate.msbpm, @@ -783,7 +784,8 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): scattlight=self.caliBrate.msscattlight, flatimages=self.caliBrate.flatimages, slits=self.caliBrate.slits, # For flexure correction - ignore_saturation=False) + ignore_saturation=False, + manual_spat_flexure=manual_spat_flexure) # get no bkg subtracted sciImg to generate a global sky model without bkg subtraction. # it's a dictionary with only `image` and `ivar` keys if bkg_redux=False, otherwise it's None @@ -795,6 +797,7 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): bkg_redux_sciimg = sciImg # Build the background image bg_file_list = self.fitstbl.frame_paths(bg_frames) + bg_manual_spat_flexure = self.fitstbl.get_shifts(bg_frames) # TODO I think we should create a separate self.par['bkgframe'] parameter set to hold the image # processing parameters for the background frames. This would allow the user to specify different # parameters for the background frames than for the science frames. @@ -805,45 +808,17 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): scattlight=self.caliBrate.msscattlight, flatimages=self.caliBrate.flatimages, slits=self.caliBrate.slits, - ignore_saturation=False) + ignore_saturation=False, + manual_spat_flexure=bg_manual_spat_flexure) # NOTE: If the spatial flexure exists for sciImg, the subtraction # function propagates that to the subtracted image, ignoring any # spatial flexure determined for the background image. sciImg = bkg_redux_sciimg.sub(bgimg) - # Flexure - spat_flexure = np.zeros((self.caliBrate.slits.nslits, 2)) # No spatial flexure, unless we find it below - # use the flexure correction in the "shift" column - manual_flexure = self.fitstbl[frames[0]]['shift'] - if (self.objtype == 'science' and self.par['scienceframe']['process']['spat_flexure_method'] != "skip") or \ - (self.objtype == 'standard' and self.par['calibrations']['standardframe']['process']['spat_flexure_method'] != "skip") or \ - not np.ma.is_masked(manual_flexure): - # First check for manual flexure - if not np.ma.is_masked(manual_flexure): - msgs.info(f'Implementing manual spatial flexure of {manual_flexure} pixels') - spat_flexure = np.full((self.caliBrate.slits.nslits, 2), np.float64(manual_flexure)) - sciImg.spat_flexure = spat_flexure - else: - if sciImg.spat_flexure is not None: - msgs.info(f'Using auto-computed spatial flexure') - spat_flexure = sciImg.spat_flexure - else: - msgs.info('Assuming no spatial flexure correction') - else: - msgs.info('Assuming no spatial flexure correction') - - # Print the flexure values - if np.all(spat_flexure == spat_flexure[0, 0]): - msgs.info(f'Spatial flexure is: {spat_flexure[0, 0]}') - else: - # Print the flexure values for each slit separately - for slit in range(spat_flexure.shape[0]): - msgs.info( - f'Spatial flexure for slit {self.caliBrate.slits.spat_id[slit]} is: left={spat_flexure[slit, 0]} right={spat_flexure[slit, 1]}') # Build the initial sky mask initial_skymask = self.load_skyregions(initial_slits=self.spectrograph.pypeline != 'SlicerIFU', - scifile=sciImg.files[0], frame=frames[0], spat_flexure=spat_flexure) + scifile=sciImg.files[0], frame=frames[0], spat_flexure=sciImg.spat_flexure) # Deal with manual extraction row = self.fitstbl[frames[0]] From 872dbec11fb653bae68724a2f8f3d34dbf5061f5 Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 10 Oct 2024 21:06:06 +0100 Subject: [PATCH 42/54] mv manual flexure to rawimage --- pypeit/calibrations.py | 95 +++++++++++++++++++++++++++++++----------- 1 file changed, 70 insertions(+), 25 deletions(-) diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index 167cefddca..ff495993aa 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -270,6 +270,9 @@ def find_calibrations(self, frametype, frameclass): raw_files : :obj:`list` The list of raw files in :attr:`fitstbl` with the provided frametype. + rows : :obj:`numpy.ndarray` + The indices of the raw files in :attr:`fitstbl` with the provided + frametype. cal_file : `Path`_ The path with/for the processed calibration frame calib_key : :obj:`str` @@ -297,12 +300,12 @@ def find_calibrations(self, frametype, frameclass): setup = self.fitstbl['setup'][self.frame] cal_file = frameclass.glob(self.calib_dir, setup, self.calib_ID, detname=detname) if cal_file is None or len(cal_file) > 1: - return [], None, None, setup, None, detname + return [], rows, None, None, setup, None, detname cal_file = cal_file[0] calib_key = frameclass.parse_key_dir(str(cal_file), from_filename=True)[0] calib_id = frameclass.parse_calib_key(calib_key)[1] - return [], cal_file, calib_key, setup, frameclass.ingest_calib_id(calib_id), detname + return [], rows, cal_file, calib_key, setup, frameclass.ingest_calib_id(calib_id), detname # Otherwise, use the metadata for the raw frames to set the name of # the processed calibration frame. @@ -312,7 +315,7 @@ def find_calibrations(self, frametype, frameclass): # Construct the expected calibration frame file name cal_file = Path(frameclass.construct_file_name(calib_key, calib_dir=self.calib_dir)) - return self.fitstbl.frame_paths(rows), cal_file, calib_key, setup, \ + return self.fitstbl.frame_paths(rows), rows, cal_file, calib_key, setup, \ frameclass.ingest_calib_id(calib_id), detname def set_config(self, frame, det, par=None): @@ -360,7 +363,7 @@ def get_arc(self): # Find the calibrations frame = {'type': 'arc', 'class': buildimage.ArcImage} - raw_files, cal_file, calib_key, setup, calib_id, detname \ + raw_files, raw_index, cal_file, calib_key, setup, calib_id, detname \ = self.find_calibrations(frame['type'], frame['class']) if len(raw_files) == 0 and cal_file is None: @@ -381,13 +384,17 @@ def get_arc(self): # Perform a check on the files self.check_calibrations(raw_files) + # If a manual spatial flexure is requested for any of the frames, collect it now + manual_spat_flexure = self.fitstbl.get_shifts(raw_index) + # Otherwise, create the processed file. msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.') self.msarc = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['arcframe'], raw_files, bias=self.msbias, bpm=self.msbpm, dark=self.msdark, calib_dir=self.calib_dir, - setup=setup, calib_id=calib_id) + setup=setup, calib_id=calib_id, + manual_spat_flexure=manual_spat_flexure) # Save the result self.msarc.to_file() # Return it @@ -406,7 +413,7 @@ def get_tiltimg(self): # Find the calibrations frame = {'type': 'tilt', 'class':buildimage.TiltImage} - raw_files, cal_file, calib_key, setup, calib_id, detname \ + raw_files, raw_index, cal_file, calib_key, setup, calib_id, detname \ = self.find_calibrations(frame['type'], frame['class']) if len(raw_files) == 0 and cal_file is None: @@ -427,6 +434,9 @@ def get_tiltimg(self): # Perform a check on the files self.check_calibrations(raw_files) + # If a manual spatial flexure is requested for any of the frames, collect it now + manual_spat_flexure = self.fitstbl.get_shifts(raw_index) + # Otherwise, create the processed file. msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.') self.mstilt = buildimage.buildimage_fromlist(self.spectrograph, self.det, @@ -434,7 +444,8 @@ def get_tiltimg(self): bias=self.msbias, bpm=self.msbpm, dark=self.msdark, slits=self.slits, calib_dir=self.calib_dir, setup=setup, - calib_id=calib_id) + calib_id=calib_id, + manual_spat_flexure=manual_spat_flexure) # Save the result self.mstilt.to_file() # Return it @@ -457,7 +468,7 @@ def get_align(self): # Find the calibrations frame = {'type': 'align', 'class': alignframe.Alignments} - raw_files, cal_file, calib_key, setup, calib_id, detname \ + raw_files, raw_index, cal_file, calib_key, setup, calib_id, detname \ = self.find_calibrations(frame['type'], frame['class']) if len(raw_files) == 0 and cal_file is None: @@ -479,13 +490,17 @@ def get_align(self): # Perform a check on the files self.check_calibrations(raw_files) + # If a manual spatial flexure is requested for any of the frames, collect it now + manual_spat_flexure = self.fitstbl.get_shifts(raw_index) + # Otherwise, create the processed file. msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.') msalign = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['alignframe'], raw_files, bias=self.msbias, bpm=self.msbpm, dark=self.msdark, calib_dir=self.calib_dir, - setup=setup, calib_id=calib_id) + setup=setup, calib_id=calib_id, + manual_spat_flexure=manual_spat_flexure) # Instantiate # TODO: From JFH: Do we need the bpm here? Check that this was in the previous code. @@ -511,7 +526,7 @@ def get_bias(self): # Find the calibrations frame = {'type': 'bias', 'class': buildimage.BiasImage} - raw_files, cal_file, calib_key, setup, calib_id, detname \ + raw_files, raw_index, cal_file, calib_key, setup, calib_id, detname \ = self.find_calibrations(frame['type'], frame['class']) if len(raw_files) == 0 and cal_file is None: @@ -553,7 +568,7 @@ def get_dark(self): # Find the calibrations frame = {'type': 'dark', 'class': buildimage.DarkImage} - raw_files, cal_file, calib_key, setup, calib_id, detname \ + raw_files, raw_index, cal_file, calib_key, setup, calib_id, detname \ = self.find_calibrations(frame['type'], frame['class']) if len(raw_files) == 0 and cal_file is None: @@ -632,7 +647,7 @@ def get_scattlight(self): # Prep frame = {'type': 'scattlight', 'class': scattlight.ScatteredLight} - raw_scattlight_files, cal_file, calib_key, setup, calib_id, detname = \ + raw_scattlight_files, raw_scattlight_index, cal_file, calib_key, setup, calib_id, detname = \ self.find_calibrations(frame['type'], frame['class']) scatt_idx = self.fitstbl.find_frames(frame['type'], calib_ID=self.calib_ID, index=True) @@ -659,13 +674,17 @@ def get_scattlight(self): # Perform a check on the files self.check_calibrations(raw_scattlight_files) + # If a manual spatial flexure is requested for any of the frames, collect it now + manual_spat_flexure = self.fitstbl.get_shifts(raw_scattlight_index) + binning = self.fitstbl[scatt_idx[0]]['binning'] dispname = self.fitstbl[scatt_idx[0]]['dispname'] scattlightImage = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['scattlightframe'], raw_scattlight_files, bias=self.msbias, bpm=self.msbpm, dark=self.msdark, calib_dir=self.calib_dir, - setup=setup, calib_id=calib_id) + setup=setup, calib_id=calib_id, + manual_spat_flexure=manual_spat_flexure) spatbin = parse.parse_binning(binning)[1] pad = self.par['scattlight_pad'] // spatbin @@ -747,7 +766,7 @@ def get_flats(self): # get illumination flat frames illum_frame = {'type': 'illumflat', 'class': flatfield.FlatImages} - raw_illum_files, illum_cal_file, illum_calib_key, illum_setup, illum_calib_id, detname \ + raw_illum_files, raw_illum_index, illum_cal_file, illum_calib_key, illum_setup, illum_calib_id, detname \ = self.find_calibrations(illum_frame['type'], illum_frame['class']) # get pixel flat frames @@ -756,11 +775,12 @@ def get_flats(self): = [], None, None, illum_setup, None, detname # read in the raw pixelflat frames only if the user has not provided a pixelflat_file if self.par['flatfield']['pixelflat_file'] is None: - raw_pixel_files, pixel_cal_file, pixel_calib_key, pixel_setup, pixel_calib_id, detname \ + raw_pixel_files, raw_pixel_index, pixel_cal_file, pixel_calib_key, pixel_setup, pixel_calib_id, detname \ = self.find_calibrations(pixel_frame['type'], pixel_frame['class']) # get lamp off flat frames raw_lampoff_files = self.fitstbl.find_frame_files('lampoffflats', calib_ID=self.calib_ID) + raw_lampoff_index = self.fitstbl.find_frames('lampoffflats', calib_ID=self.calib_ID, index=True) # Check if we have any calibration frames to work with if len(raw_pixel_files) == 0 and pixel_cal_file is None \ @@ -819,6 +839,9 @@ def get_flats(self): # Perform a check on the files self.check_calibrations(raw_pixel_files) + # If a manual spatial flexure is requested for any of the frames, collect it now + manual_spat_flexure = self.fitstbl.get_shifts(raw_pixel_index) + msgs.info('Creating pixel-flat calibration frame using files: ') for f in raw_pixel_files: msgs.prindent(f'{Path(f).name}') @@ -827,7 +850,8 @@ def get_flats(self): raw_pixel_files, dark=self.msdark, slits=self.slits, bias=self.msbias, bpm=self.msbpm, - scattlight=self.msscattlight) + scattlight=self.msscattlight, + manual_spat_flexure=manual_spat_flexure) if len(raw_lampoff_files) > 0: # Reset the BPM self.get_bpm(frame=raw_lampoff_files[0]) @@ -835,6 +859,9 @@ def get_flats(self): # Perform a check on the files self.check_calibrations(raw_lampoff_files) + # If a manual spatial flexure is requested for any of the frames, collect it now + manual_spat_flexure = self.fitstbl.get_shifts(raw_lampoff_index) + msgs.info('Subtracting lamp off flats using files: ') for f in raw_lampoff_files: msgs.prindent(f'{Path(f).name}') @@ -843,7 +870,8 @@ def get_flats(self): raw_lampoff_files, slits=self.slits, dark=self.msdark, bias=self.msbias, - bpm=self.msbpm, scattlight=self.msscattlight) + bpm=self.msbpm, scattlight=self.msscattlight, + manual_spat_flexure=manual_spat_flexure) pixel_flat = pixel_flat.sub(lampoff_flat) # Initialise the pixel flat @@ -865,6 +893,9 @@ def get_flats(self): # Perform a check on the files self.check_calibrations(raw_illum_files) + # If a manual spatial flexure is requested for any of the frames, collect it now + manual_spat_flexure = self.fitstbl.get_shifts(raw_illum_index) + msgs.info('Creating slit-illumination flat calibration frame using files: ') for f in raw_illum_files: msgs.prindent(f'{Path(f).name}') @@ -872,7 +903,8 @@ def get_flats(self): illum_flat = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['illumflatframe'], raw_illum_files, dark=self.msdark, bias=self.msbias, scattlight=self.msscattlight, - slits=self.slits, flatimages=self.flatimages, bpm=self.msbpm) + slits=self.slits, flatimages=self.flatimages, bpm=self.msbpm, + manual_spat_flexure=manual_spat_flexure) if len(raw_lampoff_files) > 0: msgs.info('Subtracting lamp off flats using files: ') for f in raw_lampoff_files: @@ -881,6 +913,9 @@ def get_flats(self): # Perform a check on the files self.check_calibrations(raw_lampoff_files) + # If a manual spatial flexure is requested for any of the frames, collect it now + manual_spat_flexure = self.fitstbl.get_shifts(raw_lampoff_index) + # Build the image lampoff_flat = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['lampoffflatsframe'], @@ -889,7 +924,8 @@ def get_flats(self): bias=self.msbias, slits=self.slits, scattlight=self.msscattlight, - bpm=self.msbpm) + bpm=self.msbpm, + manual_spat_flexure=manual_spat_flexure) illum_flat = illum_flat.sub(lampoff_flat) # Initialise the illum flat @@ -953,9 +989,10 @@ def get_slits(self): # Prep frame = {'type': 'trace', 'class': slittrace.SlitTraceSet} - raw_trace_files, cal_file, calib_key, setup, calib_id, detname \ - = self.find_calibrations(frame['type'], frame['class']) + raw_trace_files, raw_trace_index, cal_file, calib_key, setup, calib_id, detname \ + = self.find_calibrations(frame['type'], frame['class']) raw_lampoff_files = self.fitstbl.find_frame_files('lampoffflats', calib_ID=self.calib_ID) + raw_lampoff_index = self.fitstbl.find_frames('lampoffflats', calib_ID=self.calib_ID, index=True) if len(raw_trace_files) == 0 and cal_file is None: msgs.warn(f'No raw {frame["type"]} frames found and unable to identify a relevant ' @@ -998,12 +1035,16 @@ def get_slits(self): # Perform a check on the files self.check_calibrations(raw_trace_files) + # If a manual spatial flexure is requested for any of the frames, collect it now + manual_spat_flexure = self.fitstbl.get_shifts(raw_trace_index) + traceImage = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['traceframe'], raw_trace_files, bias=self.msbias, bpm=self.msbpm, scattlight=self.msscattlight, dark=self.msdark, calib_dir=self.calib_dir, - setup=setup, calib_id=calib_id) + setup=setup, calib_id=calib_id, + manual_spat_flexure=manual_spat_flexure) if len(raw_lampoff_files) > 0: msgs.info('Subtracting lamp off flats using files: ') for f in raw_lampoff_files: @@ -1015,11 +1056,15 @@ def get_slits(self): # Perform a check on the files self.check_calibrations(raw_lampoff_files) + # If a manual spatial flexure is requested for any of the frames, collect it now + manual_spat_flexure = self.fitstbl.get_shifts(raw_lampoff_index) + lampoff_flat = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['lampoffflatsframe'], raw_lampoff_files, dark=self.msdark, bias=self.msbias, scattlight=self.msscattlight, - bpm=self.msbpm) + bpm=self.msbpm, + manual_spat_flexure=manual_spat_flexure) traceImage = traceImage.sub(lampoff_flat) edges = edgetrace.EdgeTraceSet(traceImage, self.spectrograph, self.par['slitedges'], @@ -1076,7 +1121,7 @@ def get_wv_calib(self): # Find the calibrations frame = {'type': 'arc', 'class': wavecalib.WaveCalib} - raw_files, cal_file, calib_key, setup, calib_id, detname \ + raw_files, raw_index, cal_file, calib_key, setup, calib_id, detname \ = self.find_calibrations(frame['type'], frame['class']) if len(raw_files) == 0 and cal_file is None: @@ -1155,7 +1200,7 @@ def get_tilts(self): # Find the calibrations frame = {'type': 'tilt', 'class': wavetilts.WaveTilts} - raw_files, cal_file, calib_key, setup, calib_id, detname \ + raw_files, raw_index, cal_file, calib_key, setup, calib_id, detname \ = self.find_calibrations(frame['type'], frame['class']) if len(raw_files) == 0 and cal_file is None: From e8f1426564338f0201c13c6eef701590281582a8 Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 10 Oct 2024 21:36:09 +0100 Subject: [PATCH 43/54] flexure only for when slits is not None --- pypeit/images/buildimage.py | 3 ++- pypeit/images/rawimage.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pypeit/images/buildimage.py b/pypeit/images/buildimage.py index 760be750d8..acbb4a9c0b 100644 --- a/pypeit/images/buildimage.py +++ b/pypeit/images/buildimage.py @@ -247,6 +247,7 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm= # NOTE: This should not be necessary because FrameGroupPar explicitly # requires frametype to be valid msgs.error(f'{frame_par["frametype"]} is not a valid PypeIt frame type.') + manual_spatflex = manual_spat_flexure if manual_spat_flexure is not None else [np.ma.masked]*len(file_list) # Should the detectors be reformatted into a single image mosaic? if mosaic is None: @@ -261,7 +262,7 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm= rawImage_list.append(rawImage.process( frame_par['process'], scattlight=scattlight, bias=bias, bpm=bpm, dark=dark, flatimages=flatimages, slits=slits, mosaic=mosaic, - manual_spat_flexure=manual_spat_flexure[ii])) + manual_spat_flexure=manual_spatflex[ii])) # Do it combineImage = combineimage.CombineImage(rawImage_list, frame_par['process']) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 4009b1fda3..10cb29579e 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -672,7 +672,8 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl # function checks that the slits exist if running the spatial flexure # correction, so no need to do it again here. self.spat_flexure_shift = None - if self.par['spat_flexure_method'] != "skip" or not np.ma.is_masked(manual_spat_flexure): + if slits is not None and \ + (self.par['spat_flexure_method'] != "skip" or not np.ma.is_masked(manual_spat_flexure)): self.spat_flexure_shift = self.spatial_flexure_shift(slits, method=self.par['spat_flexure_method'], manual_spat_flexure=manual_spat_flexure, maxlag=self.par['spat_flexure_maxlag']) @@ -820,12 +821,12 @@ def spatial_flexure_shift(self, slits, force=False, manual_spat_flexure=np.ma.ma # Print the flexure values if np.all(spat_flexure == spat_flexure[0, 0]): - msgs.info(f'Spatial flexure is: {spat_flexure[0, 0]}') + msgs.info(f'Spatial flexure is: {spat_flexure[0, 0]} pixels') else: # Print the flexure values for each slit separately for slit in range(spat_flexure.shape[0]): msgs.info( - f'Spatial flexure for slit {slits.spat_id[slit]} is: left={spat_flexure[slit, 0]} right={spat_flexure[slit, 1]}') + f'Spatial flexure for slit {slits.spat_id[slit]} is: left={spat_flexure[slit, 0]} pixels; right={spat_flexure[slit, 1]} pixels') self.steps[step] = True # Return From e5112b1736693cc9adcef68ff3c931ade872b931 Mon Sep 17 00:00:00 2001 From: rcooke Date: Thu, 10 Oct 2024 21:38:47 +0100 Subject: [PATCH 44/54] flexure only for when slits is not None --- pypeit/images/rawimage.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 10cb29579e..1800aca225 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -672,8 +672,7 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl # function checks that the slits exist if running the spatial flexure # correction, so no need to do it again here. self.spat_flexure_shift = None - if slits is not None and \ - (self.par['spat_flexure_method'] != "skip" or not np.ma.is_masked(manual_spat_flexure)): + if self.par['spat_flexure_method'] != "skip" or not np.ma.is_masked(manual_spat_flexure): self.spat_flexure_shift = self.spatial_flexure_shift(slits, method=self.par['spat_flexure_method'], manual_spat_flexure=manual_spat_flexure, maxlag=self.par['spat_flexure_maxlag']) @@ -777,7 +776,7 @@ def spatial_flexure_shift(self, slits, force=False, manual_spat_flexure=np.ma.ma :func:`~pypeit.core.flexure.spat_flexure_shift`. Args: - slits (:class:`~pypeit.slittrace.SlitTraceSet`, optional): + slits (:class:`~pypeit.slittrace.SlitTraceSet`): Slit edge traces force (:obj:`bool`, optional): Force the image to be field flattened, even if the step log @@ -812,6 +811,14 @@ def spatial_flexure_shift(self, slits, force=False, manual_spat_flexure=np.ma.ma msgs.error('CODING ERROR: Must use a single image (single detector or detector ' 'mosaic) to determine spatial flexure.') + # Check if the slits are provided + if slits is None: + if not np.ma.is_masked(manual_spat_flexure): + msgs.warn('Manual spatial flexure provided without slits - assuming no spatial flexure.') + else: + msgs.warn('Cannot calculate spatial flexure without slits - assuming no spatial flexure.') + return + # First check for manual flexure if not np.ma.is_masked(manual_spat_flexure): msgs.info(f'Adopting a manual spatial flexure of {manual_spat_flexure} pixels') From f48c7672cb85e71963958dbb906a545ac3c9bf51 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 11 Oct 2024 12:20:28 +0100 Subject: [PATCH 45/54] deal with masked values --- pypeit/inputfiles.py | 11 ++++++----- pypeit/metadata.py | 3 ++- pypeit/pypeit.py | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index e7e1e96749..7f33d6d520 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -481,20 +481,21 @@ def path_and_files(self, key:str, skip_blank=False, include_commented_out=False, ## Build full paths to file and set frame types data_files = [] for row in self.data: + rowkey = '' if np.ma.is_masked(row[key]) else row[key] # Skip Empty entries? - if skip_blank and row[key].strip() in ['', 'none', 'None']: + if skip_blank and rowkey.strip() in ['', 'none', 'None']: continue # Skip commented out entries - if row[key].strip().startswith("#"): + if rowkey.strip().startswith("#"): if not include_commented_out: continue # Strip the comment character and any whitespace following it # from the filename - name = row[key].strip("# ") + name = rowkey.strip("# ") else: - name = row[key] + name = rowkey # Searching.. if len(self.file_paths) > 0: @@ -503,7 +504,7 @@ def path_and_files(self, key:str, skip_blank=False, include_commented_out=False, if os.path.isfile(filename): break else: - filename = row[key] + filename = rowkey # Check we got a good hit if check_exists and not os.path.isfile(filename): diff --git a/pypeit/metadata.py b/pypeit/metadata.py index 366466b6b9..8fd849d9c4 100644 --- a/pypeit/metadata.py +++ b/pypeit/metadata.py @@ -497,8 +497,9 @@ def construct_basename(self, row, obstime=None): _obstime = self.construct_obstime(row) if obstime is None else obstime tiso = time.Time(_obstime, format='isot') dtime = datetime.datetime.strptime(tiso.value, '%Y-%m-%dT%H:%M:%S.%f') + this_target = "TargetName" if np.ma.is_masked(self['target'][row]) else self['target'][row].replace(" ", "") return '{0}-{1}_{2}_{3}{4}'.format(self['filename'][row].split('.fits')[0], - self['target'][row].replace(" ", ""), + this_target, self.spectrograph.camera, datetime.datetime.strftime(dtime, '%Y%m%dT'), tiso.value.split("T")[1].replace(':','')) diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index 08b5319443..414a761447 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -823,7 +823,7 @@ def objfind_one(self, frames, det, bg_frames=None, std_outfile=None): # Deal with manual extraction row = self.fitstbl[frames[0]] manual_obj = ManualExtractionObj.by_fitstbl_input( - row['filename'], row['manual'], self.spectrograph) if len(row['manual'].strip()) > 0 else None + row['filename'], row['manual'], self.spectrograph) if not np.ma.is_masked(row['manual']) else None # Instantiate Reduce object # Required for pypeline specific object From 11e2100788234a5f1dd847fd7c7bf5c669632b95 Mon Sep 17 00:00:00 2001 From: rcooke Date: Fri, 11 Oct 2024 14:34:57 +0100 Subject: [PATCH 46/54] masked manual extraction --- pypeit/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/metadata.py b/pypeit/metadata.py index 8fd849d9c4..2a2639a743 100644 --- a/pypeit/metadata.py +++ b/pypeit/metadata.py @@ -1596,7 +1596,7 @@ def set_user_added_columns(self): """ if 'manual' not in self.keys(): - self['manual'] = '' + self['manual'] = np.ma.array(np.zeros(len(self)), mask=np.ones(len(self), dtype=bool)) if 'shift' not in self.keys(): # Instantiate a masked array self['shift'] = np.ma.array(np.zeros(len(self)), mask=np.ones(len(self), dtype=bool)) From 30022e3a851ce8344b9793e27b6a84949199a2ad Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 12 Oct 2024 16:54:34 +0100 Subject: [PATCH 47/54] testing --- pypeit/specobjs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index 85151e1663..4e0a5cf8af 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -811,7 +811,10 @@ def write_to_fits(self, subheader, outfile, overwrite=True, update_det=None, for line in str(subheader[key.upper()]).split('\n'): header[key.upper()] = line else: - header[key.upper()] = subheader[key] + try: + header[key.upper()] = subheader[key] + except: + embed() # Also store the datetime in ISOT format if key.upper() == 'MJD': if isinstance(subheader[key], (list, tuple)): From 4e10c8b35a301b405898eb586279e83156eb0b22 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 12 Oct 2024 18:48:42 +0100 Subject: [PATCH 48/54] masked header cards --- pypeit/specobjs.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index 4e0a5cf8af..303b144d10 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -811,10 +811,8 @@ def write_to_fits(self, subheader, outfile, overwrite=True, update_det=None, for line in str(subheader[key.upper()]).split('\n'): header[key.upper()] = line else: - try: - header[key.upper()] = subheader[key] - except: - embed() + _value = ('', subheader[key][1]) if np.ma.is_masked(subheader[key][0]) else subheader[key] + header[key.upper()] = _value # Also store the datetime in ISOT format if key.upper() == 'MJD': if isinstance(subheader[key], (list, tuple)): From e80f42e8650840c3043b3630b5832885eb8eefb1 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 12 Oct 2024 20:00:45 +0100 Subject: [PATCH 49/54] masked header cards --- pypeit/specobjs.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index 303b144d10..2ebcfbb25a 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -811,7 +811,13 @@ def write_to_fits(self, subheader, outfile, overwrite=True, update_det=None, for line in str(subheader[key.upper()]).split('\n'): header[key.upper()] = line else: - _value = ('', subheader[key][1]) if np.ma.is_masked(subheader[key][0]) else subheader[key] + if isinstance(subheader[key], (tuple, list)): + # value + comment + _value = ('', subheader[key][1]) if np.ma.is_masked(subheader[key][0]) else subheader[key] + else: + # value only + _value = '' if np.ma.is_masked(subheader[key]) else subheader[key] + # Update the header card with the corresponding value header[key.upper()] = _value # Also store the datetime in ISOT format if key.upper() == 'MJD': From 0056842d929f782c6c9ad598ffcf73fcd3e793a2 Mon Sep 17 00:00:00 2001 From: rcooke Date: Sat, 12 Oct 2024 20:41:46 +0100 Subject: [PATCH 50/54] masked header cards --- pypeit/spec2dobj.py | 10 +++++++++- pypeit/specobjs.py | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pypeit/spec2dobj.py b/pypeit/spec2dobj.py index da076bc983..c065ade575 100644 --- a/pypeit/spec2dobj.py +++ b/pypeit/spec2dobj.py @@ -580,7 +580,15 @@ def build_primary_hdr(self, raw_header, spectrograph, calib_dir=None, # Add the spectrograph-specific sub-header if subheader is not None: for key in subheader.keys(): - hdr[key.upper()] = subheader[key] + # Find the value and check if it is masked + if isinstance(subheader[key], (tuple, list)): + # value + comment + _value = ('', subheader[key][1]) if np.ma.is_masked(subheader[key][0]) else subheader[key] + else: + # value only + _value = '' if np.ma.is_masked(subheader[key]) else subheader[key] + # Update the header card with the corresponding value + hdr[key.upper()] = _value # PYPEIT # TODO Should the spectrograph be written to the header? diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index 2ebcfbb25a..1d523ed2a2 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -811,6 +811,7 @@ def write_to_fits(self, subheader, outfile, overwrite=True, update_det=None, for line in str(subheader[key.upper()]).split('\n'): header[key.upper()] = line else: + # Find the value and check if it is masked if isinstance(subheader[key], (tuple, list)): # value + comment _value = ('', subheader[key][1]) if np.ma.is_masked(subheader[key][0]) else subheader[key] From 06b95d57a9dfcd2ad49179626b8f1bd95bd81d11 Mon Sep 17 00:00:00 2001 From: Ryan Cooke Date: Tue, 15 Oct 2024 09:26:26 +0100 Subject: [PATCH 51/54] Update pypeit/calibrations.py Co-authored-by: Debora Pelliccia --- pypeit/calibrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index ff495993aa..fd0000da8e 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -392,7 +392,7 @@ def get_arc(self): self.msarc = buildimage.buildimage_fromlist(self.spectrograph, self.det, self.par['arcframe'], raw_files, bias=self.msbias, bpm=self.msbpm, - dark=self.msdark, calib_dir=self.calib_dir, + dark=self.msdark, slits=self.slits, calib_dir=self.calib_dir, setup=setup, calib_id=calib_id, manual_spat_flexure=manual_spat_flexure) # Save the result From 8dbf228d38edf2343c35325874b162b4b427177b Mon Sep 17 00:00:00 2001 From: rcooke Date: Tue, 15 Oct 2024 09:35:22 +0100 Subject: [PATCH 52/54] update docstrings --- pypeit/images/buildimage.py | 3 +++ pypeit/images/rawimage.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/pypeit/images/buildimage.py b/pypeit/images/buildimage.py index acbb4a9c0b..1642bb4c6f 100644 --- a/pypeit/images/buildimage.py +++ b/pypeit/images/buildimage.py @@ -222,6 +222,9 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm= Flag processed image will be a mosaic of multiple detectors. By default, this is determined by the format of ``det`` and whether or not this is a bias or dark frame. *Only used for testing purposes.* + manual_spat_flexure (:obj:`list`, `numpy.ndarray`_, optional): + A list of the spatial flexures for each image in file_list. This is only + used to manually correct the slit traces for spatial flexure of each image. calib_dir (:obj:`str`, `Path`_, optional): The directory for processed calibration files. Required for elements of :attr:`frame_image_classes`, ignored otherwise. diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 1800aca225..4e5bcd225d 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -519,6 +519,9 @@ def process(self, par, bpm=None, scattlight=None, flatimages=None, bias=None, sl mosaic. If flats or slits are provided (and used), this *must* be true because these objects are always defined in the mosaic frame. + manual_spat_flexure (:obj:`float`, optional): + The spatial flexure of the image. This is only set if the user wishes to + manually correct the slit traces of this image for spatial flexure. debug (:obj:`bool`, optional): Run in debug mode. From 0dce8e9cf735ee8e912f1ccb3b549d866c7be0bc Mon Sep 17 00:00:00 2001 From: rcooke Date: Tue, 15 Oct 2024 11:12:43 +0100 Subject: [PATCH 53/54] consistent setting of pixel indices --- pypeit/calibrations.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index fd0000da8e..1241f2b33f 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -771,8 +771,8 @@ def get_flats(self): # get pixel flat frames pixel_frame = {'type': 'pixelflat', 'class': flatfield.FlatImages} - raw_pixel_files, pixel_cal_file, pixel_calib_key, pixel_setup, pixel_calib_id, detname \ - = [], None, None, illum_setup, None, detname + raw_pixel_files, raw_pixel_index, pixel_cal_file, pixel_calib_key, pixel_setup, pixel_calib_id, detname \ + = [], [], None, None, illum_setup, None, detname # read in the raw pixelflat frames only if the user has not provided a pixelflat_file if self.par['flatfield']['pixelflat_file'] is None: raw_pixel_files, raw_pixel_index, pixel_cal_file, pixel_calib_key, pixel_setup, pixel_calib_id, detname \ From 668e6d51c249a380949aeefb5a779ed757b12c11 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Wed, 6 Nov 2024 10:45:42 -0800 Subject: [PATCH 54/54] doc update --- doc/help/run_pypeit.rst | 2 +- doc/include/dev_suite_readme.rst | 10 ++- doc/include/gemini_gnirs_echelle_A.pypeit.rst | 4 +- doc/include/keck_deimos.sorted.rst | 6 +- doc/include/keck_deimos_A.pypeit.rst | 4 +- doc/include/keck_nires_A.pypeit.rst | 4 +- doc/include/shane_kast_blue_A.pypeit.rst | 4 +- doc/pypeit_par.rst | 62 +++++++++---------- doc/whatsnew.rst | 4 ++ 9 files changed, 54 insertions(+), 46 deletions(-) diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index b0262023ad..bca4578b95 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -4,7 +4,7 @@ usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c] pypeit_file - ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.16.1.dev635+g9540496b9 + ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.16.1.dev645+gfd0074f38 ## ## Available spectrographs include: ## aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, diff --git a/doc/include/dev_suite_readme.rst b/doc/include/dev_suite_readme.rst index 676dc66976..86ddf07b00 100644 --- a/doc/include/dev_suite_readme.rst +++ b/doc/include/dev_suite_readme.rst @@ -360,7 +360,7 @@ Finally, set the ``QT_QPA_PLATFORM`` environment variable to Running in Nautilus ------------------- -The dev-suite can be run in the `Nautilus cluster `__. +The dev-suite can be run in the `Nautilus cluster `__. To generate the YAML for a dev-suite job, use ``gen_kube_devsuite``. If needed, a specific branch of both the PypeIt repository and the Pypeit-development-suite repository can be chosen using ``-p`` and ``-d`` respectively. These default to @@ -386,6 +386,10 @@ follows: ``rclone`` can also be used access the Nautilus S3 storage. When configuring use ``https://s3-west.nrp-nautilus.io`` as the endpoint. +The default job created by ``gen_kube_devsuite`` runs directly from git. However it can be changed to run +by building a new PypeIt package and installing from that, simulating what a user would get when +installing from PyPi. This is enabled with the ``--from_wheel`` or ``-w`` option. + ``gen_kube_devsuite`` has additional code for generating coverage information and the test priority list. If ``--coverage`` and ``--priority_list`` are used, these files are also copied to S3: @@ -419,8 +423,8 @@ To monitor a test in Nautilus as it is running, the logs can be tailed: Additionally they can be monitored with the `Nautilus Grafana page `__. -By default ``gen_kube_devsuite`` creates a job using a default container with PypeIt -pre-installed. It also supports running with different python versions by +By default ``gen_kube_devsuite`` creates a job using a the latest Python container +available on Docker Hub. It also supports running with different python versions by selecting a different container. For example: .. code-block:: console diff --git a/doc/include/gemini_gnirs_echelle_A.pypeit.rst b/doc/include/gemini_gnirs_echelle_A.pypeit.rst index 9f67b44af5..23fb41a9dd 100644 --- a/doc/include/gemini_gnirs_echelle_A.pypeit.rst +++ b/doc/include/gemini_gnirs_echelle_A.pypeit.rst @@ -1,7 +1,7 @@ .. code-block:: console - # Auto-generated PypeIt input file using PypeIt version: 1.15.1.dev38+g075fdecaa.d20240212 - # UTC 2024-10-02T12:11:46.067+00:00 + # Auto-generated PypeIt input file using PypeIt version: 1.17.0 + # UTC 2024-11-04T15:38:09 # User-defined execution parameters [rdx] diff --git a/doc/include/keck_deimos.sorted.rst b/doc/include/keck_deimos.sorted.rst index 655eedb3a6..828519122f 100644 --- a/doc/include/keck_deimos.sorted.rst +++ b/doc/include/keck_deimos.sorted.rst @@ -14,13 +14,13 @@ d0527_0030.fits.gz | arc,tilt | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.077631 | 1.41291034 | 1.0 | 8099.98291016 | SINGLE:B | OG550 | Kr Xe Ar Ne | 2017-05-27 | 01:51:53.87 | 30 | 0 d0527_0031.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.07851 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:53:10.93 | 31 | 0 DE.20170527.06790.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.07851 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:53:10.93 | 31 | 0 - d0527_0032.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.079356 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:54:24.03 | 32 | 0 DE.20170527.06864.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.079356 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:54:24.03 | 32 | 0 - DE.20170527.06936.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.080211 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:55:36.93 | 33 | 0 + d0527_0032.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.079356 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:54:24.03 | 32 | 0 d0527_0033.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.080211 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:55:36.93 | 33 | 0 + DE.20170527.06936.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.080211 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:55:36.93 | 33 | 0 DE.20170527.37601.fits | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.435131 | 1.03078874 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:26:41.61 | 80 | 0 - DE.20170527.38872.fits | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.449842 | 1.01267696 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:47:52.92 | 81 | 0 d0527_0081.fits.gz | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.449842 | 1.01267696 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:47:52.92 | 81 | 0 + DE.20170527.38872.fits | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.449842 | 1.01267696 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:47:52.92 | 81 | 0 DE.20170527.41775.fits | science | 261.0362916666666 | 19.028888888888886 | P261_OFF | 830G | LongMirr | 1,1 | 57900.483427 | 1.00093023 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 11:36:15.35 | 83 | 0 d0527_0083.fits.gz | science | 261.0362916666666 | 19.028888888888886 | P261_OFF | 830G | LongMirr | 1,1 | 57900.483427 | 1.00093023 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 11:36:15.35 | 83 | 0 DE.20170527.43045.fits | science | 261.0362916666666 | 19.028888888888886 | P261_OFF | 830G | LongMirr | 1,1 | 57900.498135 | 1.00838805 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 11:57:25.35 | 84 | 0 diff --git a/doc/include/keck_deimos_A.pypeit.rst b/doc/include/keck_deimos_A.pypeit.rst index a2a9ecca04..fd224bb74b 100644 --- a/doc/include/keck_deimos_A.pypeit.rst +++ b/doc/include/keck_deimos_A.pypeit.rst @@ -1,7 +1,7 @@ .. code-block:: console - # Auto-generated PypeIt input file using PypeIt version: 1.15.1.dev38+g075fdecaa.d20240212 - # UTC 2024-10-02T12:11:45.568+00:00 + # Auto-generated PypeIt input file using PypeIt version: 1.17.0 + # UTC 2024-11-04T15:38:09 # User-defined execution parameters [rdx] diff --git a/doc/include/keck_nires_A.pypeit.rst b/doc/include/keck_nires_A.pypeit.rst index b5129b39c9..d33cec00c4 100644 --- a/doc/include/keck_nires_A.pypeit.rst +++ b/doc/include/keck_nires_A.pypeit.rst @@ -1,7 +1,7 @@ .. code-block:: console - # Auto-generated PypeIt input file using PypeIt version: 1.15.1.dev38+g075fdecaa.d20240212 - # UTC 2024-10-02T12:11:46.443+00:00 + # Auto-generated PypeIt input file using PypeIt version: 1.17.0 + # UTC 2024-11-04T15:38:09 # User-defined execution parameters [rdx] diff --git a/doc/include/shane_kast_blue_A.pypeit.rst b/doc/include/shane_kast_blue_A.pypeit.rst index 6c6900edd1..f8f4693439 100644 --- a/doc/include/shane_kast_blue_A.pypeit.rst +++ b/doc/include/shane_kast_blue_A.pypeit.rst @@ -1,7 +1,7 @@ .. code-block:: console - # Auto-generated PypeIt input file using PypeIt version: 1.15.1.dev38+g075fdecaa.d20240212 - # UTC 2024-10-02T12:11:39.981+00:00 + # Auto-generated PypeIt input file using PypeIt version: 1.17.0 + # UTC 2024-11-04T15:38:09 # User-defined execution parameters [rdx] diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index 60024e7eb2..929b97b6ba 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -592,21 +592,21 @@ Collate1DPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.Collate1DPar` -========================= =============== ======= ===================================== ================================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -========================= =============== ======= ===================================== ================================================================================================================================================================================================================================================================================================================================================================================================================== -``dry_run`` bool .. False If set, the script will display the matching File and Object Ids but will not flux, coadd or archive. -``exclude_serendip`` bool .. False Whether to exclude SERENDIP objects from collating. -``exclude_slit_trace_bm`` list, str .. A list of slit trace bitmask bits that should be excluded. -``flux`` bool .. False If set, the script will flux calibrate using archived sensfuncs before coadding. -``ignore_flux`` bool .. False If set, the script will only coadd non-fluxed spectra even if flux data is present. Otherwise fluxed spectra are coadded if all spec1ds have been fluxed calibrated. -``match_using`` str .. ``ra/dec`` Determines how 1D spectra are matched as being the same object. Must be either 'pixel' or 'ra/dec'. -``outdir`` str .. ``/Users/rcooke/Software/PypeIt/doc`` The path where all coadded output files and report files will be placed. -``refframe`` str .. .. Perform reference frame correction prior to coadding. Options are: observed, heliocentric, barycentric -``spec1d_outdir`` str .. .. The path where all modified spec1d files are placed. These are only created if flux calibration or refframe correction are asked for. -``tolerance`` str, float, int .. 1.0 The tolerance used when comparing the coordinates of objects. If two objects are within this distance from each other, they are considered the same object. If match_using is 'ra/dec' (the default) this is an angular distance. The defaults units are arcseconds but other units supported by astropy.coordinates.Angle can be used (`e.g.`, '0.003d' or '0h1m30s'). If match_using is 'pixel' this is a float. -``wv_rms_thresh`` float .. .. If set, any objects with a wavelength RMS > this value are skipped, else all wavelength RMS values are accepted. -========================= =============== ======= ===================================== ================================================================================================================================================================================================================================================================================================================================================================================================================== +========================= =============== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +========================= =============== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== +``dry_run`` bool .. False If set, the script will display the matching File and Object Ids but will not flux, coadd or archive. +``exclude_serendip`` bool .. False Whether to exclude SERENDIP objects from collating. +``exclude_slit_trace_bm`` list, str .. A list of slit trace bitmask bits that should be excluded. +``flux`` bool .. False If set, the script will flux calibrate using archived sensfuncs before coadding. +``ignore_flux`` bool .. False If set, the script will only coadd non-fluxed spectra even if flux data is present. Otherwise fluxed spectra are coadded if all spec1ds have been fluxed calibrated. +``match_using`` str .. ``ra/dec`` Determines how 1D spectra are matched as being the same object. Must be either 'pixel' or 'ra/dec'. +``outdir`` str .. ``/Users/westfall/Work/packages/pypeit/doc`` The path where all coadded output files and report files will be placed. +``refframe`` str .. .. Perform reference frame correction prior to coadding. Options are: observed, heliocentric, barycentric +``spec1d_outdir`` str .. .. The path where all modified spec1d files are placed. These are only created if flux calibration or refframe correction are asked for. +``tolerance`` str, float, int .. 1.0 The tolerance used when comparing the coordinates of objects. If two objects are within this distance from each other, they are considered the same object. If match_using is 'ra/dec' (the default) this is an angular distance. The defaults units are arcseconds but other units supported by astropy.coordinates.Angle can be used (`e.g.`, '0.003d' or '0h1m30s'). If match_using is 'pixel' this is a float. +``wv_rms_thresh`` float .. .. If set, any objects with a wavelength RMS > this value are skipped, else all wavelength RMS values are accepted. +========================= =============== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== ---- @@ -659,22 +659,22 @@ ReduxPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.ReduxPar` -====================== ============== ======= ===================================== ========================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -====================== ============== ======= ===================================== ========================================================================================================================================================================================================================================================================================================================================================================================================== -``calwin`` int, float .. 0 The window of time in hours to search for calibration frames for a science frame -``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that all files were created with the current version of PypeIt. If set to False, the code will attempt to read out-of-date files and keep going. Beware (!!) that this can lead to unforeseen bugs that either cause the code to crash or lead to erroneous results. I.e., you really need to know what you are doing if you set this to False! -``detnum`` int, list .. .. Restrict reduction to a list of detector indices. In case of mosaic reduction (currently only available for Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of tuples of the detector indices that are mosaiced together. E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]`` -``ignore_bad_headers`` bool .. False Ignore bad headers (NOT recommended unless you know it is safe). -``maskIDs`` str, int, list .. .. Restrict reduction to a set of slitmask IDs Example syntax -- ``maskIDs = 818006,818015`` This must be used with detnum (for now). -``qadir`` str .. ``QA`` Directory relative to calling directory to write quality assessment files. -``quicklook`` bool .. False Run a quick look reduction? This is usually good if you want to quickly reduce the data (usually at the telescope in real time) to get an initial estimate of the data quality. -``redux_path`` str .. ``/Users/rcooke/Software/PypeIt/doc`` Path to folder for performing reductions. Default is the current working directory. -``scidir`` str .. ``Science`` Directory relative to calling directory to write science files. -``slitspatnum`` str, list .. .. Restrict reduction to a set of slit DET:SPAT values (closest slit is used). Example syntax -- slitspatnum = DET01:175,DET01:205 or MSC02:2234 If you are re-running the code, (i.e. modifying one slit) you *must* have the precise SPAT_ID index. -``sortroot`` str .. .. A filename given to output the details of the sorted files. If None, the default is the root name of the pypeit file. If off, no output is produced. -``spectrograph`` str .. .. Spectrograph that provided the data to be reduced. See :ref:`instruments` for valid options. -====================== ============== ======= ===================================== ========================================================================================================================================================================================================================================================================================================================================================================================================== +====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== +``calwin`` int, float .. 0 The window of time in hours to search for calibration frames for a science frame +``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that all files were created with the current version of PypeIt. If set to False, the code will attempt to read out-of-date files and keep going. Beware (!!) that this can lead to unforeseen bugs that either cause the code to crash or lead to erroneous results. I.e., you really need to know what you are doing if you set this to False! +``detnum`` int, list .. .. Restrict reduction to a list of detector indices. In case of mosaic reduction (currently only available for Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of tuples of the detector indices that are mosaiced together. E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]`` +``ignore_bad_headers`` bool .. False Ignore bad headers (NOT recommended unless you know it is safe). +``maskIDs`` str, int, list .. .. Restrict reduction to a set of slitmask IDs Example syntax -- ``maskIDs = 818006,818015`` This must be used with detnum (for now). +``qadir`` str .. ``QA`` Directory relative to calling directory to write quality assessment files. +``quicklook`` bool .. False Run a quick look reduction? This is usually good if you want to quickly reduce the data (usually at the telescope in real time) to get an initial estimate of the data quality. +``redux_path`` str .. ``/Users/westfall/Work/packages/pypeit/doc`` Path to folder for performing reductions. Default is the current working directory. +``scidir`` str .. ``Science`` Directory relative to calling directory to write science files. +``slitspatnum`` str, list .. .. Restrict reduction to a set of slit DET:SPAT values (closest slit is used). Example syntax -- slitspatnum = DET01:175,DET01:205 or MSC02:2234 If you are re-running the code, (i.e. modifying one slit) you *must* have the precise SPAT_ID index. +``sortroot`` str .. .. A filename given to output the details of the sorted files. If None, the default is the root name of the pypeit file. If off, no output is produced. +``spectrograph`` str .. .. Spectrograph that provided the data to be reduced. See :ref:`instruments` for valid options. +====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== ---- diff --git a/doc/whatsnew.rst b/doc/whatsnew.rst index 85de8cae70..9313a9bfe5 100644 --- a/doc/whatsnew.rst +++ b/doc/whatsnew.rst @@ -11,6 +11,10 @@ What's New in PypeIt ---- +.. include:: releases/1.17.1dev.rst + +---- + .. include:: releases/1.17.0.rst ----