From 9fd6fc7fb12493c8b093659a2cb78635cc08fd93 Mon Sep 17 00:00:00 2001 From: Shane Maloney Date: Thu, 11 Apr 2024 15:17:36 +0100 Subject: [PATCH] Add warning if trigger scaling changes in request (#388) --- stixcore/data/test.py | 3 ++- ...00000-9999999999_V02_2402021788-59493.fits | 3 +++ stixcore/data/test/publish/rid_lut.csv | 4 +-- stixcore/data/test/publish/update_rid_lut.csv | 4 +-- stixcore/processing/tests/test_processing.py | 13 +++++++++- stixcore/products/level0/scienceL0.py | 26 ++++++++++++++----- 6 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 stixcore/data/test/products/solo_LB_stix-21-6-24_0000000000-9999999999_V02_2402021788-59493.fits diff --git a/stixcore/data/test.py b/stixcore/data/test.py index b3c64919..28bd901e 100644 --- a/stixcore/data/test.py +++ b/stixcore/data/test.py @@ -34,7 +34,8 @@ def __init__(self, data_dir): self.DIR / "solo_L1_stix-ql-lightcurve_20210116_V01.fits"] self.L1_fits = list(self.DIR.glob('solo_L1_stix-*.fits')) self.LB_21_6_30_fits = self.DIR / "solo_LB_stix-21-6-30_0664156800_V01.fits" - self.LB_21_6_21_fits = self.DIR / "solo_LB_stix-21-6-21_0000000000-9999999999_V02_2312148821-53879.fits" # noqa + self.LB_21_6_21_fits_scaled = self.DIR / "solo_LB_stix-21-6-21_0000000000-9999999999_V02_2312148821-53879.fits" # noqa + self.LB_21_6_24_scale_change = self.DIR / "solo_LB_stix-21-6-24_0000000000-9999999999_V02_2402021788-59493.fits" # noqa self.__doc__ = "\n".join([f'{str(k)}: {repr(v)}\n\n' for k, v in self.__dict__.items()]) diff --git a/stixcore/data/test/products/solo_LB_stix-21-6-24_0000000000-9999999999_V02_2402021788-59493.fits b/stixcore/data/test/products/solo_LB_stix-21-6-24_0000000000-9999999999_V02_2402021788-59493.fits new file mode 100644 index 00000000..5682514e --- /dev/null +++ b/stixcore/data/test/products/solo_LB_stix-21-6-24_0000000000-9999999999_V02_2402021788-59493.fits @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4fd6424e8f964da5c99f605fa1dbdd2f428a7a11737537be7c57c24f5fdd7f1d +size 184320 diff --git a/stixcore/data/test/publish/rid_lut.csv b/stixcore/data/test/publish/rid_lut.csv index 933f9af1..7e23869c 100644 --- a/stixcore/data/test/publish/rid_lut.csv +++ b/stixcore/data/test/publish/rid_lut.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9aed9734a1e809e241520ba31eba3d20aab5e7a6c75d99e8d64d4c93863018aa -size 888 +oid sha256:a04105a1f367e11ec1c68a0d1271832b83060bdd7dfb02af46da148b7dd197bd +size 1009 diff --git a/stixcore/data/test/publish/update_rid_lut.csv b/stixcore/data/test/publish/update_rid_lut.csv index 933f9af1..7e23869c 100644 --- a/stixcore/data/test/publish/update_rid_lut.csv +++ b/stixcore/data/test/publish/update_rid_lut.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9aed9734a1e809e241520ba31eba3d20aab5e7a6c75d99e8d64d4c93863018aa -size 888 +oid sha256:a04105a1f367e11ec1c68a0d1271832b83060bdd7dfb02af46da148b7dd197bd +size 1009 diff --git a/stixcore/processing/tests/test_processing.py b/stixcore/processing/tests/test_processing.py index 0d07cde7..aabb1192 100644 --- a/stixcore/processing/tests/test_processing.py +++ b/stixcore/processing/tests/test_processing.py @@ -60,7 +60,7 @@ def packet(): def test_level_0_descaling_trigger(out_dir): - lb = test_data.products.LB_21_6_21_fits + lb = test_data.products.LB_21_6_21_fits_scaled l0 = Level0(out_dir / 'LB', out_dir) res = l0.process_fits_files(files=[lb]) assert len(res) == 1 @@ -70,6 +70,17 @@ def test_level_0_descaling_trigger(out_dir): assert factor == 30 +def test_level_0_descaling_warning(out_dir): + lb = test_data.products.LB_21_6_24_scale_change + l0 = Level0(out_dir / 'LB', out_dir) + res = l0.process_fits_files(files=[lb]) + assert len(res) == 1 + datawarn = fits.getval(res[0], 'DATAWARN') + comment = fits.getval(res[0], 'COMMENT') + assert datawarn == 1 + assert "Multiple compression schemes detected, trigger values maybe incorrect." in comment + + @pytest.mark.skip(reason="needs proper spice pointing kernels") def test_level_2(out_dir, spicekernelmanager): SOOPManager.instance = SOOPManager(Path(__file__).parent.parent.parent diff --git a/stixcore/products/level0/scienceL0.py b/stixcore/products/level0/scienceL0.py index 169b7a57..82370326 100644 --- a/stixcore/products/level0/scienceL0.py +++ b/stixcore/products/level0/scienceL0.py @@ -302,6 +302,10 @@ def __init__(self, *, service_type, service_subtype, ssid, control, data, @classmethod def from_levelb(cls, levelb, parent=''): + header_comments = [] + header_history = [] + additional_header_keywords = [] + packets, idb_versions, control = ScienceProduct.from_levelb(levelb, parent=parent) c_skm, c_skm_meta = _get_compression_scheme(packets, 'NIX00260') @@ -310,8 +314,10 @@ def from_levelb(cls, levelb, parent=''): t_skm, t_skm_meta = _get_compression_scheme(packets, 'NIX00242') control.add_data('compression_scheme_triggers_skm', (t_skm[0].reshape(1, 3), t_skm_meta)) - header_history = [] - additional_header_keywords = [] + if np.unique(t_skm, axis=0).shape[0] != 1: + additional_header_keywords.append(('DATAWARN', 1, 'See comments')) + header_comments.append('Multiple compression schemes detected, ' + 'trigger values maybe incorrect') data = Data() try: @@ -530,7 +536,8 @@ def from_levelb(cls, levelb, parent=''): data=data, idb_versions=idb_versions, packets=packets, - history=header_history) + history=header_history, + comment=header_comments) prod.add_additional_header_keywords(additional_header_keywords) return prod @@ -696,6 +703,10 @@ def __init__(self, *, service_type, service_subtype, ssid, control, data, @classmethod def from_levelb(cls, levelb, parent=''): + header_comments = [] + header_history = [] + additional_header_keywords = [] + packets, idb_versions, control = ScienceProduct.from_levelb(levelb, parent=parent) c_skm, c_skm_meta = _get_compression_scheme(packets, 'NIX00268') @@ -704,8 +715,10 @@ def from_levelb(cls, levelb, parent=''): t_skm, t_skm_meta = _get_compression_scheme(packets, 'NIX00267') control.add_data('compression_scheme_triggers_skm', (t_skm[0].reshape(1, 3), t_skm_meta)) - header_history = [] - additional_header_keywords = [] + if np.unique(t_skm, axis=0).shape[0] != 1: + additional_header_keywords.append(('DATAWARN', 1, 'See comments')) + header_comments.append('Multiple compression schemes detected, ' + 'trigger values maybe incorrect.') control['detector_masks'] = np.unique(_get_detector_mask(packets)[0], axis=0) control['detector_masks'] = fix_detector_mask(control, control['detector_masks']) @@ -836,7 +849,8 @@ def from_levelb(cls, levelb, parent=''): data=data, idb_versions=idb_versions, packets=packets, - history=header_history) + history=header_history, + comment=header_comments) prod.add_additional_header_keywords(additional_header_keywords) return prod