diff --git a/stixcore/io/fits/processors.py b/stixcore/io/fits/processors.py index e4524b31..5e0ebc58 100644 --- a/stixcore/io/fits/processors.py +++ b/stixcore/io/fits/processors.py @@ -28,6 +28,10 @@ NAN = 2 ** 32 - 1 +def empty_if_nan(val): + return "" if np.isnan(val) else val + + def version_format(version): # some very strange work around for direct use of format '{0:02d}'.format(version) # as this is not supported by magicMoc @@ -741,8 +745,8 @@ def generate_primary_header(self, filename, product, *, version=0): headers = FitsProcessor.generate_common_header(filename, product, version=version) data_headers = ( - ('DATAMIN', product.dmin, 'Minimum valid physical value'), - ('DATAMAX', product.dmax, 'Maximum valid physical value'), + ('DATAMIN', empty_if_nan(product.dmin), 'Minimum valid physical value'), + ('DATAMAX', empty_if_nan(product.dmax), 'Maximum valid physical value'), ('BUNIT', product.bunit, 'Units of physical value, after application of BSCALE, BZERO'), ('XPOSURE', product.exposure, '[s] shortest exposure time'), ('XPOMAX', product.max_exposure, '[s] maximum exposure time') @@ -934,8 +938,8 @@ def write_fits(self, product, *, version=0): if version == 0: version = product.get_processing_version() - # TODO remove writeout supression of all products but aux files - if product.type == 'aux': + # TODO remove writeout supression of all products but ANC files + if product.level == 'ANC': return super().write_fits(product, version=version) else: logger.info(f"no writeout of L2 {product.type}-{product.name} FITS files.") @@ -954,13 +958,13 @@ def generate_primary_header(self, filename, product, *, version=0): # new or override keywords L2headers = ( # Name, Value, Comment - ('LEVEL', 'L2', 'Processing level of the data'), + ('LEVEL', product.level, 'Processing level of the data'), ('VERS_SW', str(stixcore.__version__), 'Version of SW that provided FITS file'), ('VERS_CFG', str(stixcore.__version_conf__), 'Version of the common instrument configuration package'), ('HISTORY', 'Processed by STIXCore L2'), - ('DATAMIN', product.dmin, 'Minimum valid physical value'), - ('DATAMAX', product.dmax, 'Maximum valid physical value'), + ('DATAMIN', empty_if_nan(product.dmin), 'Minimum valid physical value'), + ('DATAMAX', empty_if_nan(product.dmax), 'Maximum valid physical value'), ('BUNIT', product.bunit, 'Units of physical value, after application of BSCALE, BZERO'), ('XPOSURE', product.exposure, '[s] shortest exposure time'), ('XPOMAX', product.max_exposure, '[s] maximum exposure time') diff --git a/stixcore/processing/publish.py b/stixcore/processing/publish.py index 68fff6dd..21bfa816 100644 --- a/stixcore/processing/publish.py +++ b/stixcore/processing/publish.py @@ -358,7 +358,7 @@ def send_mail_report(files): def update_ephemeris_headers(fits_file, spice): - """Updates all SPICE related data in FITS header. + """Updates all SPICE related data in FITS header and data table. Parameters ---------- @@ -368,7 +368,7 @@ def update_ephemeris_headers(fits_file, spice): Spice kernel manager with loaded spice kernels. """ product = Product(fits_file) - if product.level in ['L1', 'L2']: + if product.level in ['L1', 'L2', 'ANC']: ephemeris_headers = \ spice.get_fits_headers(start_time=product.utc_timerange.start, average_time=product.utc_timerange.center) @@ -379,7 +379,7 @@ def update_ephemeris_headers(fits_file, spice): # rename the header filename to be complete hdu.header['FILENAME'] = get_complete_file_name(hdu.header['FILENAME']) hdu.header.update(ephemeris_headers) - # just update the first prima HDU + # just update the first primary HDU break logger.info(f"updated ephemeris headers of {fits_file}") @@ -430,7 +430,7 @@ def add_history(p, name): def copy_file(scp, p, target_dir, add_history_entry=True): - """Copies FITS file top the designated SOAR out directory. Will use remote copy if set up. + """Copies FITS file to the designated SOAR out directory. Will use remote copy if set up. Parameters ---------- @@ -539,12 +539,12 @@ def publish_fits_to_esa(args): parser.add_argument("-l", "--include_levels", help="what levels should be published", type=str, - default=CONFIG.get('Publish', 'include_levels', fallback="L0, L1, L2")) + default=CONFIG.get('Publish', 'include_levels', fallback="L0, L1, L2, ANC")) parser.add_argument("-p", "--include_products", help="what products should be published", type=str, default=CONFIG.get('Publish', 'include_products', - fallback="ql,hk,sci,aux,cal")) + fallback="ql,hk,sci,asp,cal")) parser.add_argument("-f", "--fits_dir", help="input FITS directory for files to publish ", diff --git a/stixcore/products/level0/quicklookL0.py b/stixcore/products/level0/quicklookL0.py index 9166951d..61471e0d 100644 --- a/stixcore/products/level0/quicklookL0.py +++ b/stixcore/products/level0/quicklookL0.py @@ -583,11 +583,11 @@ def from_levelb(cls, levelb, parent=''): @property def dmin(self): - return min([self.data['loc_y'].min(), self.data['loc_z'].min()]) + return np.nanmin([self.data['loc_y'].min(), self.data['loc_z'].min()]) @property def dmax(self): - return max([self.data['loc_y'].max(), self.data['loc_z'].max()]) + return np.nanmax([self.data['loc_y'].max(), self.data['loc_z'].max()]) @property def bunit(self): diff --git a/stixcore/products/level0/scienceL0.py b/stixcore/products/level0/scienceL0.py index b7bd598b..62e64be6 100644 --- a/stixcore/products/level0/scienceL0.py +++ b/stixcore/products/level0/scienceL0.py @@ -956,17 +956,17 @@ def from_levelb(cls, levelb, parent=''): @property def dmin(self): - return min([self.data['cha_diode0'].min(), - self.data['cha_diode1'].min(), - self.data['chb_diode0'].min(), - self.data['chb_diode1'].min()]) + return np.nanmin([self.data['cha_diode0'].min(), + self.data['cha_diode1'].min(), + self.data['chb_diode0'].min(), + self.data['chb_diode1'].min()]) @property def dmax(self): - return max([self.data['cha_diode0'].max(), - self.data['cha_diode1'].max(), - self.data['chb_diode0'].max(), - self.data['chb_diode1'].max()]) + return np.nanmax([self.data['cha_diode0'].max(), + self.data['cha_diode1'].max(), + self.data['chb_diode0'].max(), + self.data['chb_diode1'].max()]) @property def bunit(self): diff --git a/stixcore/products/level1/quicklookL1.py b/stixcore/products/level1/quicklookL1.py index df4564c7..c36635e1 100644 --- a/stixcore/products/level1/quicklookL1.py +++ b/stixcore/products/level1/quicklookL1.py @@ -4,6 +4,8 @@ from collections import defaultdict +import numpy as np + from stixcore.products.level0.quicklookL0 import QLProduct from stixcore.products.product import L1Mixin from stixcore.time import SCETimeRange @@ -132,11 +134,11 @@ def __init__(self, *, service_type, service_subtype, ssid, control, data, @property def dmin(self): - return min([self.data['loc_y'].min(), self.data['loc_z'].min()]) + return np.nanmin([self.data['loc_y'].min(), self.data['loc_z'].min()]) @property def dmax(self): - return max([self.data['loc_y'].max(), self.data['loc_z'].max()]) + return np.nanmax([self.data['loc_y'].max(), self.data['loc_z'].max()]) @property def bunit(self): diff --git a/stixcore/products/level1/scienceL1.py b/stixcore/products/level1/scienceL1.py index dc7c529b..aedcc66b 100644 --- a/stixcore/products/level1/scienceL1.py +++ b/stixcore/products/level1/scienceL1.py @@ -1,5 +1,7 @@ from collections import defaultdict +import numpy as np + from stixcore.products.level0.scienceL0 import ScienceProduct from stixcore.products.product import L1Mixin from stixcore.time import SCETimeRange @@ -130,17 +132,17 @@ def __init__(self, *, service_type, service_subtype, ssid, control, @property def dmin(self): - return min([self.data['cha_diode0'].min(), - self.data['cha_diode1'].min(), - self.data['chb_diode0'].min(), - self.data['chb_diode1'].min()]) + return np.nanmin([self.data['cha_diode0'].min(), + self.data['cha_diode1'].min(), + self.data['chb_diode0'].min(), + self.data['chb_diode1'].min()]) @property def dmax(self): - return max([self.data['cha_diode0'].max(), - self.data['cha_diode1'].max(), - self.data['chb_diode0'].max(), - self.data['chb_diode1'].max()]) + return np.nanmax([self.data['cha_diode0'].max(), + self.data['cha_diode1'].max(), + self.data['chb_diode0'].max(), + self.data['chb_diode1'].max()]) @property def bunit(self): diff --git a/stixcore/products/level2/housekeepingL2.py b/stixcore/products/level2/housekeepingL2.py index 360aae53..aa57d67c 100644 --- a/stixcore/products/level2/housekeepingL2.py +++ b/stixcore/products/level2/housekeepingL2.py @@ -336,20 +336,20 @@ def __init__(self, *, service_type=0, service_subtype=0, ssid=1, control, data, ssid=ssid, control=control, data=data, idb_versions=idb_versions, **kwargs) self.name = 'ephemeris' - self.level = 'L2' - self.type = 'aux' + self.level = 'ANC' + self.type = 'asp' self.ssid = 1 self.service_subtype = 0 self.service_type = 0 @property def dmin(self): - return min([self.data['y_srf'].min().to_value('arcsec'), + return np.nanmin([self.data['y_srf'].min().to_value('arcsec'), self.data['z_srf'].min().to_value('arcsec')]) @property def dmax(self): - return max([self.data['y_srf'].max().to_value('arcsec'), + return np.nanmax([self.data['y_srf'].max().to_value('arcsec'), self.data['z_srf'].max().to_value('arcsec')]) @property @@ -358,5 +358,5 @@ def bunit(self): @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): - return (kwargs['level'] == 'L2' and service_type == 0 + return (kwargs['level'] == 'ANC' and service_type == 0 and service_subtype == 0 and ssid == 1)