From 962dc11934cd0425d8d9178e08468dc60b79ea2a Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 12 Jul 2023 11:02:14 +0200 Subject: [PATCH 01/62] clean commit history : -update SPE fit parameters initialization following updates of the ctapipe extractors -improve the limits of generated charge histogram plots --- .../calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py index a34e0205..05aae793 100644 --- a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py +++ b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py @@ -4,7 +4,7 @@ from iminuit import Minuit import random import astropy.units as u -from astropy.table import QTable,Column,MaskedColumn +from astropy.table import Table,QTable,Column,MaskedColumn import yaml import os from datetime import date From 517bfc70e11b6331d3db772169393117b727cda2 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 14 Jun 2023 09:01:35 +0200 Subject: [PATCH 02/62] test script --- .../user_scripts/ggrolleron/test.py | 67 ++++++++++++++----- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/src/nectarchain/user_scripts/ggrolleron/test.py b/src/nectarchain/user_scripts/ggrolleron/test.py index 5575a3d4..1556f5e9 100644 --- a/src/nectarchain/user_scripts/ggrolleron/test.py +++ b/src/nectarchain/user_scripts/ggrolleron/test.py @@ -15,6 +15,8 @@ handler.setFormatter(formatter) log.addHandler(handler) +#sys.path.append("/home/ggroller/projects/nectarchain/src/nectarchain") + from nectarchain.calibration.container import ChargeContainer,WaveformsContainer,WaveformsContainers,ChargeContainers @@ -22,36 +24,64 @@ def test_multicontainers() : run_number = [3731] - #waveforms = WaveformsContainers(run_number[0],max_events=20000) - #log.info("waveforms created") - #waveforms.load_wfs() - #log.info("waveforms loaded") - # + waveforms = WaveformsContainers(run_number[0],max_events=1000) + log.info("waveforms created") + waveforms.load_wfs() + log.info("waveforms loaded") + #waveforms.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",overwrite = True) #log.info('waveforms written') #waveforms = WaveformsContainers.load(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}") #log.info("waveforms loaded") - #charge = ChargeContainers.from_waveforms(waveforms,method = "LocalPeakWindowSum", window_width = 16, window_shift = 4) - #log.info("charge computed") - # + charge = ChargeContainers.from_waveforms(waveforms,method = "LocalPeakWindowSum", window_width = 16, window_shift = 4) + log.info("charge computed") + path = "LocalPeakWindowSum_4-12" #charge.write(f"{os.environ['NECTARCAMDATA']}/charges/{path}/",overwrite = True) #log.info("charge written") - charge = ChargeContainers.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{path}/",run_number = run_number[0]) - log.info("charge loaded") + #charge = ChargeContainers.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{path}/",run_number = run_number[0]) + #log.info("charge loaded") charge_merged = charge.merge() log.info('charge merged') +def test_white_target() : + run_number = [4129] + waveforms = WaveformsContainers(run_number[0],max_events=10000) + log.info("waveforms created") + waveforms.load_wfs() + log.info("waveforms loaded") + + #waveforms.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",overwrite = True) + #log.info('waveforms written') + + #waveforms = WaveformsContainers.load(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}") + #log.info("waveforms loaded") + + charge = ChargeContainers.from_waveforms(waveforms,method = "LocalPeakWindowSum", window_width = 16, window_shift = 4) + log.info("charge computed") + + path = "LocalPeakWindowSum_4-12" + #charge.write(f"{os.environ['NECTARCAMDATA']}/charges/{path}/",overwrite = True) + #log.info("charge written") + + #charge = ChargeContainers.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{path}/",run_number = run_number[0]) + #log.info("charge loaded") + + #charge_merged = charge.merge() + #log.info('charge merged') + + + + -""" def test_simplecontainer() : run_number = [2633] ped_run_number = [2630] @@ -61,22 +91,23 @@ def test_simplecontainer() : spe_run_1000V.load_wfs() - spe_run_1000V.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",overwrite = True) + #spe_run_1000V.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",overwrite = True) + + #spe_run_1000V.load(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}.fits") - spe_run_1000V.load(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}.fits") + charge = ChargeContainer.compute_charge(spe_run_1000V,1,method = "LocalPeakWindowSum",extractor_kwargs = {'window_width' : 16, 'window_shift' : 4}) - charge = ChargeContainer.compute_charge(spe_run_1000V,1,method = "gradient_extractor") + charge = ChargeContainer.from_waveforms(spe_run_1000V) - charge = ChargeContainer.from_waveform(spe_run_1000V) + #charge.write(f"{os.environ['NECTARCAMDATA']}/charges/std/",overwrite = True) - charge.write(f"{os.environ['NECTARCAMDATA']}/charges/std/",overwrite = True) -""" if __name__ == "__main__" : - test_multicontainers() + #test_multicontainers() + test_simplecontainer() print("work completed") \ No newline at end of file From 1631a591fb564b83754923ce0d3e56a26ee38c41 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 15 Jun 2023 11:39:33 +0200 Subject: [PATCH 03/62] add method to sort WaveformsContainer and ChargesContainer following events_id --- .../calibration/container/charge.py | 20 +++++++++++++++++++ .../calibration/container/waveforms.py | 17 ++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/nectarchain/calibration/container/charge.py b/src/nectarchain/calibration/container/charge.py index add0e784..9fe724e9 100644 --- a/src/nectarchain/calibration/container/charge.py +++ b/src/nectarchain/calibration/container/charge.py @@ -412,6 +412,23 @@ def select_charge_lg(self,pixel_id : np.ndarray) : for pixel in pixel_id[~mask_contain_pixels_id] : log.warning(f"You asked for pixel_id {pixel} but it is not present in this ChargeContainer, skip this one") return np.array([self.charge_lg.T[np.where(self.pixels_id == pixel)[0][0]] for pixel in pixel_id[mask_contain_pixels_id]]).T + def sort(self, method = 'event_id') : + if method == 'event_id' : + log.info('sorting ChargeContaineur with event_id') + index = np.argsort(self.event_id) + self.ucts_timestamp = self.ucts_timestamp[index] + self.ucts_busy_counter = self.ucts_busy_counter[index] + self.ucts_event_counter = self.ucts_event_counter[index] + self.event_type = self.event_type[index] + self.event_id = self.event_id[index] + self.trig_pattern_all = self.trig_pattern_all[index] + self.charge_hg = self.charge_hg[index] + self.charge_lg = self.charge_lg[index] + self.peak_hg = self.peak_hg[index] + self.peak_lg = self.peak_lg[index] + else : + raise ArgumentError(f"{method} is not a valid method for sorting") + @property def run_number(self) : return self._run_number @@ -549,4 +566,7 @@ def merge(self) -> ChargeContainer : cls.event_type = np.concatenate([chargecontainer.event_type for chargecontainer in self.chargeContainers ]) cls.event_id = np.concatenate([chargecontainer.event_id for chargecontainer in self.chargeContainers ]) cls.trig_pattern_all = np.concatenate([chargecontainer.trig_pattern_all for chargecontainer in self.chargeContainers ],axis = 0) + + cls.sort() + return cls \ No newline at end of file diff --git a/src/nectarchain/calibration/container/waveforms.py b/src/nectarchain/calibration/container/waveforms.py index bceb9b94..79267f85 100644 --- a/src/nectarchain/calibration/container/waveforms.py +++ b/src/nectarchain/calibration/container/waveforms.py @@ -109,7 +109,7 @@ def __compute_broken_pixels(self,**kwargs) : def load_run(run_number : int,max_events : int = None, run_file = None) : """Static method to load from $NECTARCAMDATA directory data for specified run with max_events - Args: + Args:self.__run_number = run_number run_number (int): run_id maxevents (int, optional): max of events to be loaded. Defaults to 0, to load everythings. merge_file (optional) : if True will load all fits.fz files of the run, else merge_file can be integer to select the run fits.fz file according to this number @@ -137,6 +137,7 @@ def check_events(self): previous_trigger = None n_events = 0 for i,event in enumerate(self.__reader): + log.debug(f"events i has ID : {np.uint16(event.index.event_id)}") if previous_trigger is not None : has_changed = (previous_trigger.value != event.trigger.event_type.value) previous_trigger = event.trigger.event_type @@ -290,7 +291,19 @@ def load(path : str) : - + def sort(self, method = 'event_id') : + if method == 'event_id' : + index = np.argsort(self.event_id) + self.ucts_timestamp = self.ucts_timestamp[index] + self.ucts_busy_counter = self.ucts_busy_counter[index] + self.ucts_event_counter = self.ucts_event_counter[index] + self.event_type = self.event_type[index] + self.event_id = self.event_id[index] + self.trig_pattern_all = self.trig_pattern_all[index] + self.wfs_hg = self.wfs_hg[index] + self.wfs_lg = self.wfs_lg[index] + else : + raise ArgumentError(f"{method} is not a valid method for sorting") def compute_trigger_patern(self) : """(preliminary) function to compute the trigger patern From cfcb0dccbb656389743f0950628948980a1575e6 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 16 Jun 2023 10:24:00 +0200 Subject: [PATCH 04/62] -waveformContainer contructor from event list --- .../calibration/container/waveforms.py | 64 ++++++++++++++----- .../user_scripts/ggrolleron/test.py | 2 +- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/src/nectarchain/calibration/container/waveforms.py b/src/nectarchain/calibration/container/waveforms.py index 79267f85..c077fa5a 100644 --- a/src/nectarchain/calibration/container/waveforms.py +++ b/src/nectarchain/calibration/container/waveforms.py @@ -105,6 +105,34 @@ def __compute_broken_pixels(self,**kwargs) : self.__broken_pixels_hg = np.zeros((self.npixels),dtype = bool) self.__broken_pixels_lg = np.zeros((self.npixels),dtype = bool) + @staticmethod + def create_from_events_list(events_list : list, + run_number : int, + npixels : int, + nsamples : int, + subarray, + pixels_id : int, + ) : + cls = WaveformsContainer.__new__(WaveformsContainer) + + cls.__run_number = run_number + cls.__nevents = len(events_list) + cls.__npixels = npixels + cls.__nsamples = nsamples + cls.__subarray = subarray + cls.__pixels_id = pixels_id + + + wfs_hg_tmp=np.zeros((npixels,nsamples),dtype = np.uint16) + wfs_lg_tmp=np.zeros((npixels,nsamples),dtype = np.uint16) + + for i, event in enumerate(events_list): + cls.fill_wfs_from_event(event,i,wfs_hg_tmp,wfs_lg_tmp) + + cls.__compute_broken_pixels() + + + @staticmethod def load_run(run_number : int,max_events : int = None, run_file = None) : """Static method to load from $NECTARCAMDATA directory data for specified run with max_events @@ -164,28 +192,32 @@ def load_wfs(self,compute_trigger_patern = False): for i,event in enumerate(self.__reader): if i%100 == 0: log.info(f"reading event number {i}") - - self.event_id[i] = np.uint16(event.index.event_id) - self.ucts_timestamp[i]=event.nectarcam.tel[WaveformsContainer.TEL_ID].evt.ucts_timestamp - self.event_type[i]=event.trigger.event_type.value - self.ucts_busy_counter[i] = event.nectarcam.tel[WaveformsContainer.TEL_ID].evt.ucts_busy_counter - self.ucts_event_counter[i] = event.nectarcam.tel[WaveformsContainer.TEL_ID].evt.ucts_event_counter - - - self.trig_pattern_all[i] = event.nectarcam.tel[WaveformsContainer.TEL_ID].evt.trigger_pattern.T - - for pix in range(self.npixels): - wfs_lg_tmp[pix]=event.r0.tel[0].waveform[1,self.pixels_id[pix]] - wfs_hg_tmp[pix]=event.r0.tel[0].waveform[0,self.pixels_id[pix]] - - self.wfs_hg[i] = wfs_hg_tmp - self.wfs_lg[i] = wfs_lg_tmp + self.fill_wfs_from_event(event,i,wfs_hg_tmp,wfs_lg_tmp) self.__compute_broken_pixels() #if compute_trigger_patern and np.max(self.trig_pattern) == 0: # self.compute_trigger_patern() + def fill_wfs_from_event(self, + event, + i : int, + wfs_hg_tmp : np.ndarray, + wfs_lg_tmp : np.ndarray + ) : + self.event_id[i] = np.uint16(event.index.event_id) + self.ucts_timestamp[i]=event.nectarcam.tel[WaveformsContainer.TEL_ID].evt.ucts_timestamp + self.event_type[i]=event.trigger.event_type.value + self.ucts_busy_counter[i] = event.nectarcam.tel[WaveformsContainer.TEL_ID].evt.ucts_busy_counter + self.ucts_event_counter[i] = event.nectarcam.tel[WaveformsContainer.TEL_ID].evt.ucts_event_counter + self.trig_pattern_all[i] = event.nectarcam.tel[WaveformsContainer.TEL_ID].evt.trigger_pattern.T + + for pix in range(self.npixels): + wfs_lg_tmp[pix]=event.r0.tel[0].waveform[1,self.pixels_id[pix]] + wfs_hg_tmp[pix]=event.r0.tel[0].waveform[0,self.pixels_id[pix]] + + self.wfs_hg[i] = wfs_hg_tmp + self.wfs_lg[i] = wfs_lg_tmp def write(self,path : str, **kwargs) : """method to write in an output FITS file the WaveformsContainer. Two files are created, one FITS representing the data diff --git a/src/nectarchain/user_scripts/ggrolleron/test.py b/src/nectarchain/user_scripts/ggrolleron/test.py index 1556f5e9..a1a56f30 100644 --- a/src/nectarchain/user_scripts/ggrolleron/test.py +++ b/src/nectarchain/user_scripts/ggrolleron/test.py @@ -108,6 +108,6 @@ def test_simplecontainer() : if __name__ == "__main__" : #test_multicontainers() - test_simplecontainer() + test_white_target() print("work completed") \ No newline at end of file From e8a5e1f431ed483edbf6e8c393e90eb49e354ccc Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 30 Mar 2023 17:24:43 +0200 Subject: [PATCH 05/62] user script for test ggroller --- .../user_scripts/ggrolleron/gain_SPEfit_computation.sh | 3 +++ src/nectarchain/user_scripts/ggrolleron/test.py | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh index c290c10c..bf6e8446 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh @@ -8,3 +8,6 @@ python gain_SPEfit_computation.py -r 2634 --voltage_tag "1400V" --free_pp_n --ch #to perform SPE fit of a run at nominal voltage python gain_SPEfit_computation.py -r 3936 --voltage_tag "nominal" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 + +#to perform SPE fit of a HHV run +python gain_SPEfit_computation.py -r 3942 --output_fig_tag "testMyRes" --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 \ No newline at end of file diff --git a/src/nectarchain/user_scripts/ggrolleron/test.py b/src/nectarchain/user_scripts/ggrolleron/test.py index a1a56f30..eeb9e2b5 100644 --- a/src/nectarchain/user_scripts/ggrolleron/test.py +++ b/src/nectarchain/user_scripts/ggrolleron/test.py @@ -5,6 +5,7 @@ import matplotlib.cm as cm from pathlib import Path import sys,os +import time import logging logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level=logging.INFO) log = logging.getLogger(__name__) @@ -97,10 +98,16 @@ def test_simplecontainer() : charge = ChargeContainer.compute_charge(spe_run_1000V,1,method = "LocalPeakWindowSum",extractor_kwargs = {'window_width' : 16, 'window_shift' : 4}) +charge = ChargeContainer.from_waveforms(wfs, method = "SlidingWindowMaxSum", window_width = 16, window_shift = 4) +log.info(f"SlidingWindowMaxSum duration : {time.time() - t} seconds") +#charge = ChargeContainer.from_waveforms(wfs, method = "NeighborPeakWindowSum", window_width = 16, window_shift = 4) +#log.info(f"NeighborPeakWindowSum duration : {time.time() - t} seconds") charge = ChargeContainer.from_waveforms(spe_run_1000V) +charge = ChargeContainer.from_waveforms(wfs, method = "TwoPassWindowSum", window_width = 16, window_shift = 4) +log.info(f"TwoPassWindowSum duration : {time.time() - t} seconds") #charge.write(f"{os.environ['NECTARCAMDATA']}/charges/std/",overwrite = True) From ec426d2a37914b6ca2e8ff48faaa4a18452a8541 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Tue, 11 Apr 2023 16:56:24 +0200 Subject: [PATCH 06/62] change values of pp and n in fit parameters file by median --- .../calibration/NectarGain/SPEfit/parameters_signal.yaml | 4 ++-- .../calibration/NectarGain/SPEfit/parameters_signalStd.yaml | 4 ++-- .../NectarGain/SPEfit/parameters_signal_combined.yaml | 4 ++-- .../NectarGain/SPEfit/parameters_signal_fromHHVFit.yaml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal.yaml b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal.yaml index df2ec0f0..89ece153 100644 --- a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal.yaml +++ b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.3735, + value : 0.45, min : 0.2, max : 0.8 }, @@ -20,7 +20,7 @@ max : 0.7 }, n : { - value : 0.708, + value : 0.697, min : 0.5, max : 0.9 }, diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signalStd.yaml b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signalStd.yaml index b887be64..12746d7f 100644 --- a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signalStd.yaml +++ b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signalStd.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.463, + value : 0.45, min : .NAN, max : .NAN }, @@ -20,7 +20,7 @@ max : 0.7 }, n : { - value : 0.698, + value : 0.697, min : .NAN, max : .NAN }, diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_combined.yaml b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_combined.yaml index 93dd3124..a04fc05a 100644 --- a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_combined.yaml +++ b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_combined.yaml @@ -15,7 +15,7 @@ max : 5.0 }, pp : { - value : 0.463, + value : 0.45, min : .NAN, max : .NAN }, @@ -25,7 +25,7 @@ max : 0.7 }, n : { - value : 0.698, + value : 0.697, min : .NAN, max : .NAN }, diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_fromHHVFit.yaml b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_fromHHVFit.yaml index ea6a7a0e..62515304 100644 --- a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_fromHHVFit.yaml +++ b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_fromHHVFit.yaml @@ -10,7 +10,7 @@ max : 5.0 }, pp : { - value : 0.463, + value : 0.45, min : .NAN, max : .NAN }, @@ -20,7 +20,7 @@ max : .NAN }, n : { - value : 0.698, + value : 0.697, min : .NAN, max : .NAN }, From 75215103d8095a816cc37acc80abb314567875de Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 17 May 2023 15:17:31 +0200 Subject: [PATCH 07/62] improve test --- .../ggrolleron/gain_SPEfit_combined_computation.sh | 2 +- src/nectarchain/user_scripts/ggrolleron/test.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.sh b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.sh index 7a4dd733..9e796f30 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.sh +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.sh @@ -6,4 +6,4 @@ python gain_SPEfit_combined_computation.py -r 2633 --SPE_fit_results_tag 2634 -- python gain_SPEfit_combined_computation.py -r 2633 --SPE_fit_results_tag 2634 --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 --same_luminosity --combined #to perform SPE fit of run 3936 (supposed to be taken at nominal voltage) from SPE fit performed on 2634 (taken at 1400V) (luminosity can not be supposed the same) -python gain_SPEfit_combined_computation.py -r 3936 --SPE_fit_results_tag 3942 --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 --VVH_fitted_results "/data/users/ggroller/NECTARCAM/SPEfit/data/MULTI-1400V-SPEStd-2634-LocalPeakWindowSum_4-12/output_table.ecsv" +python gain_SPEfit_combined_computation.py -r 3936 --SPE_fit_results_tag 3942 --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 --VVH_fitted_results "/data/users/ggroller/NECTARCAM/SPEfit/data/MULTI-1400V-SPEStd-3942-LocalPeakWindowSum_4-12/output_table.ecsv" diff --git a/src/nectarchain/user_scripts/ggrolleron/test.py b/src/nectarchain/user_scripts/ggrolleron/test.py index eeb9e2b5..4a1cdf7b 100644 --- a/src/nectarchain/user_scripts/ggrolleron/test.py +++ b/src/nectarchain/user_scripts/ggrolleron/test.py @@ -16,7 +16,7 @@ handler.setFormatter(formatter) log.addHandler(handler) -#sys.path.append("/home/ggroller/projects/nectarchain/src/nectarchain") +import glob @@ -98,16 +98,16 @@ def test_simplecontainer() : charge = ChargeContainer.compute_charge(spe_run_1000V,1,method = "LocalPeakWindowSum",extractor_kwargs = {'window_width' : 16, 'window_shift' : 4}) -charge = ChargeContainer.from_waveforms(wfs, method = "SlidingWindowMaxSum", window_width = 16, window_shift = 4) -log.info(f"SlidingWindowMaxSum duration : {time.time() - t} seconds") + charge = ChargeContainer.from_waveforms(wfs, method = "SlidingWindowMaxSum", window_width = 16, window_shift = 4) + log.info(f"SlidingWindowMaxSum duration : {time.time() - t} seconds") -#charge = ChargeContainer.from_waveforms(wfs, method = "NeighborPeakWindowSum", window_width = 16, window_shift = 4) -#log.info(f"NeighborPeakWindowSum duration : {time.time() - t} seconds") + #charge = ChargeContainer.from_waveforms(wfs, method = "NeighborPeakWindowSum", window_width = 16, window_shift = 4) + #log.info(f"NeighborPeakWindowSum duration : {time.time() - t} seconds") charge = ChargeContainer.from_waveforms(spe_run_1000V) -charge = ChargeContainer.from_waveforms(wfs, method = "TwoPassWindowSum", window_width = 16, window_shift = 4) -log.info(f"TwoPassWindowSum duration : {time.time() - t} seconds") + charge = ChargeContainer.from_waveforms(wfs, method = "TwoPassWindowSum", window_width = 16, window_shift = 4) + log.info(f"TwoPassWindowSum duration : {time.time() - t} seconds") #charge.write(f"{os.environ['NECTARCAMDATA']}/charges/std/",overwrite = True) From 8ca58d856b804fcb65de92cf077ad046eff2ff07 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 17 May 2023 15:21:40 +0200 Subject: [PATCH 08/62] repare test --- .../user_scripts/ggrolleron/test.py | 110 +++++++++--------- 1 file changed, 56 insertions(+), 54 deletions(-) diff --git a/src/nectarchain/user_scripts/ggrolleron/test.py b/src/nectarchain/user_scripts/ggrolleron/test.py index 4a1cdf7b..8e2d2b27 100644 --- a/src/nectarchain/user_scripts/ggrolleron/test.py +++ b/src/nectarchain/user_scripts/ggrolleron/test.py @@ -20,73 +20,81 @@ -from nectarchain.calibration.container import ChargeContainer,WaveformsContainer,WaveformsContainers,ChargeContainers +from nectarchain.calibration.container import ChargeContainer,WaveformsContainer,ChargeContainers,WaveformsContainers from nectarchain.calibration.container.utils import DataManagement -def test_multicontainers() : - run_number = [3731] - waveforms = WaveformsContainers(run_number[0],max_events=1000) - log.info("waveforms created") - waveforms.load_wfs() - log.info("waveforms loaded") - - #waveforms.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",overwrite = True) - #log.info('waveforms written') - #waveforms = WaveformsContainers.load(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}") - #log.info("waveforms loaded") +def test_check_wfs() : + run_number = [3731] + files = glob.glob(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}_*.fits") + if len(files) == 0 : + raise FileNotFoundError(f"no splitted waveforms found") + else : + charge_container = ChargeContainers() + for j,file in enumerate(files): + log.debug(f"loading wfs from file {file}") + wfs = WaveformsContainer.load(file) + print(f"min : {wfs.wfs_hg.min()}") + #fig,ax = wfs.plot_waveform_hg(0) + #for i in range(wfs.nevents) : + # wfs.plot_waveform_hg(i,figure = fig,ax = ax) + + log.debug(f"computation of charge for file {file}") + charge = ChargeContainer.from_waveforms(wfs, + method = "LocalPeakWindowSum", + window_width = 16, + window_shift = 4) + hist = charge.histo_hg() + charge_container.append(charge) + + + +def test_extractor() : + run_number = [2633] - charge = ChargeContainers.from_waveforms(waveforms,method = "LocalPeakWindowSum", window_width = 16, window_shift = 4) - log.info("charge computed") - - path = "LocalPeakWindowSum_4-12" - #charge.write(f"{os.environ['NECTARCAMDATA']}/charges/{path}/",overwrite = True) - #log.info("charge written") - #charge = ChargeContainers.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{path}/",run_number = run_number[0]) - #log.info("charge loaded") + wfs = WaveformsContainer(run_number[0],max_events = 200) + # + wfs.load_wfs() + # + #spe_run_1000V.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",overwrite = True) - charge_merged = charge.merge() - log.info('charge merged') + #wfs = WaveformsContainer.load(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}.fits") -def test_white_target() : - run_number = [4129] - waveforms = WaveformsContainers(run_number[0],max_events=10000) - log.info("waveforms created") - waveforms.load_wfs() - log.info("waveforms loaded") - - #waveforms.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",overwrite = True) - #log.info('waveforms written') + #charge = ChargeContainer.compute_charge(spe_run_1000V,1,method = "gradient_extractor",) - #waveforms = WaveformsContainers.load(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}") - #log.info("waveforms loaded") + t = time.time() + charge = ChargeContainer.from_waveforms(wfs, method = "LocalPeakWindowSum", window_width = 16, window_shift = 4) + log.info(f"LocalPeakWindowSum duration : {time.time() - t} seconds") - charge = ChargeContainers.from_waveforms(waveforms,method = "LocalPeakWindowSum", window_width = 16, window_shift = 4) - log.info("charge computed") - - path = "LocalPeakWindowSum_4-12" - #charge.write(f"{os.environ['NECTARCAMDATA']}/charges/{path}/",overwrite = True) - #log.info("charge written") + charge = ChargeContainer.from_waveforms(wfs, method = "GlobalPeakWindowSum", window_width = 16, window_shift = 4) + log.info(f"GlobalPeakWindowSum duration : {time.time() - t} seconds") - #charge = ChargeContainers.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{path}/",run_number = run_number[0]) - #log.info("charge loaded") + charge = ChargeContainer.from_waveforms(wfs, method = "FullWaveformSum", window_width = 16, window_shift = 4) + log.info(f"FullWaveformSum duration : {time.time() - t} seconds") - #charge_merged = charge.merge() - #log.info('charge merged') + charge = ChargeContainer.from_waveforms(wfs, method = "FixedWindowSum", window_width = 16, window_shift = 4, peak_index = 30) + log.info(f"FullWindowSum duration : {time.time() - t} seconds") + charge = ChargeContainer.from_waveforms(wfs, method = "SlidingWindowMaxSum", window_width = 16, window_shift = 4) + log.info(f"SlidingWindowMaxSum duration : {time.time() - t} seconds") + #charge = ChargeContainer.from_waveforms(wfs, method = "NeighborPeakWindowSum", window_width = 16, window_shift = 4) + #log.info(f"NeighborPeakWindowSum duration : {time.time() - t} seconds") + #charge = ChargeContainer.from_waveforms(wfs, method = "BaselineSubtractedNeighborPeakWindowSum", baseline_start = 2, baseline_end = 12, window_width = 16, window_shift = 4) + #log.info(f"BaselineSubtractedNeighborPeakWindowSum duration : {time.time() - t} seconds") + charge = ChargeContainer.from_waveforms(wfs, method = "TwoPassWindowSum", window_width = 16, window_shift = 4) + log.info(f"TwoPassWindowSum duration : {time.time() - t} seconds") + #charge.write(f"{os.environ['NECTARCAMDATA']}/charges/std/",overwrite = False) def test_simplecontainer() : run_number = [2633] - ped_run_number = [2630] - FF_run_number = [2609] spe_run_1000V = WaveformsContainer(run_number[0],max_events = 1000) @@ -98,23 +106,17 @@ def test_simplecontainer() : charge = ChargeContainer.compute_charge(spe_run_1000V,1,method = "LocalPeakWindowSum",extractor_kwargs = {'window_width' : 16, 'window_shift' : 4}) - charge = ChargeContainer.from_waveforms(wfs, method = "SlidingWindowMaxSum", window_width = 16, window_shift = 4) - log.info(f"SlidingWindowMaxSum duration : {time.time() - t} seconds") - #charge = ChargeContainer.from_waveforms(wfs, method = "NeighborPeakWindowSum", window_width = 16, window_shift = 4) - #log.info(f"NeighborPeakWindowSum duration : {time.time() - t} seconds") charge = ChargeContainer.from_waveforms(spe_run_1000V) - charge = ChargeContainer.from_waveforms(wfs, method = "TwoPassWindowSum", window_width = 16, window_shift = 4) - log.info(f"TwoPassWindowSum duration : {time.time() - t} seconds") #charge.write(f"{os.environ['NECTARCAMDATA']}/charges/std/",overwrite = True) + if __name__ == "__main__" : - #test_multicontainers() - test_white_target() + test_check_wfs() - print("work completed") \ No newline at end of file + print("work completed") From 1f0b255e19b7216ad56aa5f26c22c1902d7f7e9e Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 18 May 2023 14:09:50 +0200 Subject: [PATCH 09/62] reduce pvalue digit in label --- .../calibration/NectarGain/SPEfit/NectarGainSPE.py | 2 +- .../NectarGain/SPEfit/NectarGainSPE_combined.py | 4 ++-- .../NectarGain/SPEfit/NectarGainSPE_singlerun.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE.py b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE.py index 245ec50d..1cb9c282 100644 --- a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE.py +++ b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE.py @@ -81,7 +81,7 @@ def _update_parameters_postfit(m : Minuit,parameters : Parameters) : tmp.error = m.errors[i] @staticmethod - def _make_output_dict_obs(m : Minuit,valid,pixels_id,parameters : Parameters,ndof : int) : + def _make_output_dict_obs(m : Minuit, valid, pixels_id, parameters : Parameters, ndof : int) : __class__._update_parameters_postfit(m,parameters) output = {"is_valid" : valid, "pixel" : pixels_id} for parameter in parameters.parameters : diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_combined.py b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_combined.py index 1d1a31da..89c59f24 100644 --- a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_combined.py +++ b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_combined.py @@ -231,7 +231,7 @@ def _run_obs(self,pixel,**kwargs) : np.trapz(self.nectarGain.histo[pixel],self.nectarGain.charge[pixel])*MPE2(self.nectarGain.charge[pixel],self.__pp.value,self.__resolution.value,self.__mean.value,self.__n.value,self.__pedestal.value,self.__pedestalWidth.value,self.__luminosity.value), zorder=1, linewidth=2, - label = f"SPE model fit \n gain : {self.__gain[pixel,0] - self.__gain[pixel,1]:.2f} < {self.__gain[pixel,0]:.2f} < {self.__gain[pixel,0] + self.__gain[pixel,2]:.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval)},\n likelihood = {fit.fval:.2f}") + label = f"SPE model fit \n gain : {self.__gain[pixel,0] - self.__gain[pixel,1]:.2f} < {self.__gain[pixel,0]:.2f} < {self.__gain[pixel,0] + self.__gain[pixel,2]:.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval):.3f},\n likelihood = {fit.fval:.2f}") ax[0].set_xlabel("Charge (ADC)", size=15) ax[0].set_ylabel("Events", size=15) ax[0].set_title(f"SPE fit pixel : {pixel} (pixel id : {self.pixels_id[pixel]})") @@ -242,7 +242,7 @@ def _run_obs(self,pixel,**kwargs) : np.trapz(self.nectarGainHHV.histo[pixel],self.nectarGainHHV.charge[pixel])*MPE2(self.nectarGainHHV.charge[pixel],self.__pp.value,self.__resolution.value,self.__meanHHV.value,self.__n.value,self.__pedestalHHV.value,self.__pedestalWidth.value,self.__luminosity.value), zorder=1, linewidth=2, - label = f"SPE model fit \n gainHHV : {self.__gainHHV[pixel,0] - self.__gainHHV[pixel,1]:.2f} < {self.__gainHHV[pixel,0]:.2f} < {self.__gainHHV[pixel,0] + self.__gainHHV[pixel,2]:.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval)},\n likelihood = {fit.fval:.2f}") + label = f"SPE model fit \n gainHHV : {self.__gainHHV[pixel,0] - self.__gainHHV[pixel,1]:.2f} < {self.__gainHHV[pixel,0]:.2f} < {self.__gainHHV[pixel,0] + self.__gainHHV[pixel,2]:.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval):.3f},\n likelihood = {fit.fval:.2f}") ax[1].set_xlabel("Charge (ADC)", size=15) ax[1].set_ylabel("Events", size=15) ax[1].set_title(f"SPE fit pixel : {pixel} (pixel id : {self.pixels_id[pixel]})") diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py index 05aae793..7be1da35 100644 --- a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py +++ b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py @@ -463,7 +463,7 @@ def _run_obs_static(cls,it : int, funct,parameters : Parameters, pixels_id : int np.trapz(histo,charge)*MPE2(charge,parameters['pp'].value, parameters['resolution'].value, parameters['mean'].value, parameters['n'].value, parameters['pedestal'].value, parameters['pedestalWidth'].value, parameters['luminosity'].value), zorder=1, linewidth=2, - label = f"SPE model fit \n gain : {gain[0] - gain[1]:.2f} < {gain[0]:.2f} < {gain[0] + gain[2]:.2f} ADC/pe,\n pvalue = {Statistics.chi2_pvalue(ndof,fit.fval)},\n likelihood = {fit.fval:.2f}") + label = f"SPE model fit \n gain : {gain[0] - gain[1]:.2f} < {gain[0]:.2f} < {gain[0] + gain[2]:.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval):.3f},\n likelihood = {fit.fval:.2f}") ax.set_xlabel("Charge (ADC)", size=15) ax.set_ylabel("Events", size=15) ax.set_title(f"SPE fit pixel {it} with pixel_id : {pixels_id}") @@ -510,7 +510,7 @@ def _run_obs(self,pixel,prescan = False,**kwargs) : np.trapz(self.histo[pixel],self.charge[pixel])*MPE2(self.charge[pixel],self.__pp.value,self.__resolution.value,self.__mean.value,self.__n.value,self.pedestal.value,self.__pedestalWidth.value,self.__luminosity.value), zorder=1, linewidth=2, - label = f"SPE model fit \n gain : {self.__gain[pixel,0] - self.__gain[pixel,1]:.2f} < {self.__gain[pixel,0]:.2f} < {self.__gain[pixel,0] + self.__gain[pixel,2]:.2f} ADC/pe,\n pvalue = {Statistics.chi2_pvalue(ndof,fit.fval)},\n likelihood = {fit.fval:.2f}") + label = f"SPE model fit \n gain : {self.__gain[pixel,0] - self.__gain[pixel,1]:.2f} < {self.__gain[pixel,0]:.2f} < {self.__gain[pixel,0] + self.__gain[pixel,2]:.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval):.3f},\n likelihood = {fit.fval:.2f}") ax.set_xlabel("Charge (ADC)", size=15) ax.set_ylabel("Events", size=15) ax.set_title(f"SPE fit pixel : {pixel} (pixel id : {self.pixels_id[pixel]})") @@ -867,7 +867,7 @@ def _run_obs(self,pixel,prescan = False,**kwargs) : np.trapz(self.histo[pixel],self.charge[pixel])*gaussian(self.charge[pixel],self.pedestal.value,self.__pedestalWidth.value), zorder=1, linewidth=2, - label = f"MPE model fit \n Pedestal = {round(self.__pedestalFitted[pixel,0]):.2f} +/- {round(self.__pedestalFitted[pixel,1],2):.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval)},\n likelihood = {fit.fval:.2f}") + label = f"MPE model fit \n Pedestal = {round(self.__pedestalFitted[pixel,0]):.2f} +/- {round(self.__pedestalFitted[pixel,1],2):.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval):.3f},\n likelihood = {fit.fval:.2f}") ax.set_xlabel("Charge (ADC)", size=15) ax.set_ylabel("Events", size=15) ax.set_title(f"Pedestal fit pixel : {pixel} (pixel id : {self.pixels_id[pixel]})") @@ -912,7 +912,7 @@ def _run_obs_static(cls,it : int, funct,parameters : Parameters, pixels_id : int np.trapz(histo, charge)*gaussian(charge, parameters["pedestal"].value,parameters["pedestalWidth"].value), zorder=1, linewidth=2, - label = f"MPE model fit \n Pedestal = {round(parameters['pedestal'].value):.2f} +/- {round(parameters['pedestal'].error,2):.2e} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval)},\n likelihood = {fit.fval:.2f}") + label = f"MPE model fit \n Pedestal = {round(parameters['pedestal'].value):.2f} +/- {round(parameters['pedestal'].error,2):.2e} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval):.3f},\n likelihood = {fit.fval:.2f}") ax.set_xlabel("Charge (ADC)", size=15) ax.set_ylabel("Events", size=15) ax.set_title(f"Pedestal fit pixel_id = {pixels_id})") From 627363c03b43c43ea754c9921419880c9013f8c4 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Sun, 9 Jul 2023 11:34:27 +0200 Subject: [PATCH 10/62] -add time counter to SPE fit --- .../SPEfit/NectarGainSPE_singlerun.py | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py index 7be1da35..1667631d 100644 --- a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py +++ b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py @@ -13,6 +13,8 @@ import matplotlib matplotlib.use('AGG',force = True) +import time + import pandas as pd import logging @@ -36,8 +38,10 @@ class NectarGainSPESingle(NectarGainSPE): - _Ncall = 4000000 + _Ncall = 100 _Nproc_Multiprocess = mlp.cpu_count() // 2 + _FitTol = 1e20 + _FitStrategy = 0 def __init__(self,signal : ChargeContainer,**kwargs) : log.info("initialisation of the SPE fit instance") @@ -91,6 +95,10 @@ def make_table_from_output_multi(self,list_dict : list) : def _run_fit(funct,parameters,pixels_id,prescan = False,**kwargs) : minuitParameters = UtilsMinuit.make_minuit_par_kwargs(parameters.unfrozen) fit = Minuit(funct,**minuitParameters['values']) + fit.errordef = Minuit.LIKELIHOOD + fit.tol = __class__._FitTol + #fit.precision = 1e-2 + UtilsMinuit.set_minuit_parameters_limits_and_errors(fit,minuitParameters) log.info(f"Initial value of Likelihood = {funct(**minuitParameters['values'])}") log.info(f"Initial parameters value : {fit.values}") @@ -105,15 +113,19 @@ def _run_fit(funct,parameters,pixels_id,prescan = False,**kwargs) : if log.getEffectiveLevel() == logging.DEBUG : fit.print_level = 3 - fit.strategy = 2 + fit.strategy = __class__._FitStrategy fit.throw_nan = True if prescan : log.info("let's do a fisrt brut force scan") fit.scan() try : log.info('migrad execution') - fit.migrad(ncall=super(__class__,__class__)._Ncall) - fit.hesse() + t = time.time() + fit.migrad(ncall=__class__._Ncall) + log.debug(f'time for migrad execution is {time.time() - t:.0f} sec') + t = time.time() + fit.hesse(ncall=__class__._Ncall) + log.debug(f'time for hesse execution is {time.time() - t:.0f} sec') valid = fit.valid except RuntimeError as e : log.warning(e,exc_info = True) @@ -125,7 +137,7 @@ def _run_fit(funct,parameters,pixels_id,prescan = False,**kwargs) : fit.scan() try : log.info('simplex execution') - fit.simplex(ncall=super(__class__,__class__)._Ncall) + fit.simplex(ncall=__class__._Ncall) fit.hesse() valid = fit.valid except Exception as e : From 482435a43c1665f19f085b3855f82b96ccbfa625 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Sun, 9 Jul 2023 11:34:58 +0200 Subject: [PATCH 11/62] add possibility to freez parameters into minuit --- src/nectarchain/calibration/NectarGain/SPEfit/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/utils.py b/src/nectarchain/calibration/NectarGain/SPEfit/utils.py index 0dbb91f3..44686680 100644 --- a/src/nectarchain/calibration/NectarGain/SPEfit/utils.py +++ b/src/nectarchain/calibration/NectarGain/SPEfit/utils.py @@ -42,6 +42,8 @@ def make_minuit_par_kwargs(parameters : Parameters): error = 0.1 if np.isnan(parameter.error) else parameter.error kwargs[f"limit_{parameter.name}"] = (min_, max_) kwargs[f"error_{parameter.name}"] = error + if parameter.frozen : + kwargs[f"fix_{parameter.name}"] = True return kwargs @staticmethod @@ -55,6 +57,8 @@ def set_minuit_parameters_limits_and_errors(m : Minuit,parameters : dict) : for name in parameters["names"] : m.limits[name] = parameters[f"limit_{name}"] m.errors[name] = parameters[f"error_{name}"] + if parameters.get(f"fix_{name}",False) : + m.fixed[name] = True # Usefull fucntions for the fit From 4fa0f82be3c196b1349e28004da7d582e28e5e7b Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 12 Jul 2023 10:52:19 +0200 Subject: [PATCH 12/62] begin : redesign of the SPE fit class to speed up --- .../NectarGain/SPEfit/NectarGainSPE.py | 19 ++ .../SPEfit/NectarGainSPE_singlerun.py | 205 ++++++++++++++++++ 2 files changed, 224 insertions(+) diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE.py b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE.py index 1cb9c282..4f7c747c 100644 --- a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE.py +++ b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE.py @@ -28,6 +28,25 @@ __all__ = ['NectarGainSPE'] +class NectarGainSPE_new(ABC) : + _Ncall = 4000000 + _Windows_lenght = 40 + _Order = 2 + + def __init__(self) : + #set parameters value for fit + self.__parameters = Parameters() + #output + self._output_table = QTable() + #self.create_output_table() #need to be done in the child class __init__ + + + + + + + + class NectarGainSPE(ABC) : _Ncall = 4000000 _Windows_lenght = 40 diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py index 1667631d..fae81bc2 100644 --- a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py +++ b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py @@ -37,6 +37,211 @@ __all__ = ["NectarGainSPESingleSignalStd","NectarGainSPESingleSignal","NectarGainSPESinglePed","NectarGainSPESingleSignalfromHHVFit"] + +class NectarGainSPESingle(NectarGainSPE): + + def __init__(self,signal : ChargeContainer,**kwargs) : + log.info("initialisation of the SPE fit instance") + super().__init__(**kwargs) + + histo = signal.histo_hg(autoscale = True) + #access data + self.__charge = histo[1] + self.__histo = histo[0] + self.__mask_fitted_pixel = np.zeros((self.__charge.shape[0]),dtype = bool) + self.__pixels_id = signal.pixels_id + + self.__pedestal = Parameter(name = "pedestal", + value = (np.min(self.__charge) + np.sum(self.__charge * self.__histo)/(np.sum(self.__histo)))/2, + min = np.min(self.__charge), + max = np.sum(self.__charge*self.__histo)/np.sum(self.__histo), + unit = u.dimensionless_unscaled) + + + self._parameters.append(self.__pedestal) + + def _update_parameters() + + + def _make_fit_array_from_parameters(self) : + fit_array = np.empty((self.npixels),dtype = np.object_) + for i in range(len(fit_array)) : + parameters = self._update_parameters(parameters,charge[i].data[~charge[i].mask],counts[i].data[~charge[i].mask]) + minuitParameters = UtilsMinuit.make_minuit_par_kwargs(parameters) + #log.info('creation of fit') + fit_array[i] = Minuit(cost(charge[i].data[~charge[i].mask],counts[i].data[~charge[i].mask]), + pedestal = minuitParameters['values']['pedestal'], + pp = minuitParameters['values']['pp'], + luminosity = minuitParameters['values']['luminosity'], + resolution = minuitParameters['values']['resolution'], + mean = minuitParameters['values']['mean'], + n = minuitParameters['values']['n'], + pedestalWidth = minuitParameters['values']['pedestalWidth'] + ) + #log.info('fit created') + fit_array[i].errordef = Minuit.LIKELIHOOD + fit_array[i].strategy = 0 + fit_array[i].tol = 1e40 + fit_array[i].print_level = 1 + fit_array[i].throw_nan = True + UtilsMinuit.set_minuit_parameters_limits_and_errors(fit_array[i],minuitParameters) + #log.info(fit_array[i].values) + #log.info(fit_array[i].limits) + #log.info(fit_array[i].fixed) + return fit_array + + + +class NectarGainSPESingleSignal_new(NectarGainSPESingle_new) : + + """class to perform fit of the SPE signal with all free parameters""" + __parameters_file = 'parameters_signal.yaml' + #def __new__(cls) : + # print("NectarGainSPESingleSignal is not instanciable") + # return 0 + def __init__(self,signal : ChargeContainer,parameters_file = None,**kwargs) : + super().__init__(signal,**kwargs) + self.__gain = np.empty((self.charge.shape[0],3)) + #if parameters file is provided + if parameters_file is None : + parameters_file = NectarGainSPESingleSignal.__parameters_file + with open(f"{os.path.dirname(os.path.abspath(__file__))}/{parameters_file}") as parameters : + param = yaml.safe_load(parameters) + self.__pp = Parameter(name = "pp", + value = param["pp"]["value"], + min = param["pp"].get("min",np.nan), + max = param["pp"].get("max",np.nan), + unit = param["pp"].get("unit",u.dimensionless_unscaled)) + self._parameters.append(self.__pp) + + self.__luminosity = Parameter(name = "luminosity", + value = param["luminosity"]["value"], + min = param["luminosity"].get("min",np.nan), + max = param["luminosity"].get("max",np.nan), + unit = param["luminosity"].get("unit",u.dimensionless_unscaled)) + self._parameters.append(self.__luminosity) + + self.__resolution = Parameter(name = "resolution", + value = param["resolution"]["value"], + min = param["resolution"].get("min",np.nan), + max = param["resolution"].get("max",np.nan), + unit = param["resolution"].get("unit",u.dimensionless_unscaled)) + self._parameters.append(self.__resolution) + + self.__mean = Parameter(name = "mean", + value = param["mean"]["value"], + min = param["mean"].get("min",0), + max = param["mean"].get("max",np.nan), + unit = param["mean"].get("unit",u.dimensionless_unscaled)) + self._parameters.append(self.__mean) + + self.__n = Parameter(name = "n", + value = param["n"]["value"], + min = param["n"].get("min",np.nan), + max = param["n"].get("max",np.nan), + unit = param["n"].get("unit",u.dimensionless_unscaled)) + self._parameters.append(self.__n) + + self.__pedestalWidth = Parameter(name = "pedestalWidth", + value = param["pedestalWidth"]["value"], + min = param["pedestalWidth"].get("min",np.nan), + max = param["pedestalWidth"].get("max",np.nan), + unit = param["pedestalWidth"].get("unit",u.dimensionless_unscaled)) + self._parameters.append(self.__pedestalWidth) + + self._make_minuit_parameters() + + self.create_output_table() + + + @classmethod + def NG_Likelihood_Chi2(cls,pp,res,mu2,n,muped,sigped,lum,charge,histo,**kwargs): + pdf = MPE2(charge,pp,res,mu2,n,muped,sigped,lum,**kwargs) + #log.debug(f"pdf : {np.sum(pdf)}") + Ntot = np.sum(histo) + #log.debug(f'Ntot : {Ntot}') + mask = histo > 0 + Lik = np.sum(((pdf*Ntot-histo)[mask])**2/histo[mask]) #2 times faster + return Lik + + def Chi2(self,pixel : int): + def _Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity) : + #assert not(np.isnan(pp) or np.isnan(resolution) or np.isnan(mean) or np.isnan(n) or np.isnan(pedestal) or np.isnan(pedestalWidth) or np.isnan(luminosity)) + if self.__old_lum != luminosity : + for i in range(1000): + if (gammainc(i+1,luminosity) < 1e-5): + self.__old_ntotalPE = i + break + self.__old_lum = luminosity + kwargs = {"ntotalPE" : self.__old_ntotalPE} + + return self.NG_Likelihood_Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity,self.charge[pixel],self.histo[pixel],**kwargs) + return _Chi2 + + def Chi2_static(self,pixel : int) : + charge = copy.deepcopy(self.charge[pixel].data[~self.charge[pixel].mask]) + histo = copy.deepcopy(self.histo[pixel].data[~self.histo[pixel].mask]) + def _Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity) : + #assert not(np.isnan(pp) or np.isnan(resolution) or np.isnan(mean) or np.isnan(n) or np.isnan(pedestal) or np.isnan(pedestalWidth) or np.isnan(luminosity)) + ntotalPE = 0 + for i in range(1000): + if (gammainc(i+1,luminosity) < 1e-5): + ntotalPE = i + break + kwargs = {"ntotalPE" : ntotalPE} + return __class__.NG_Likelihood_Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity,charge,histo,**kwargs) + return _Chi2 + + @classmethod + def _update_parameters_prefit_static(cls,it : int, parameters : Parameters, charge : np.ndarray, histo : np.ndarray,**kwargs) : + super(__class__, cls)._update_parameters_prefit_static(it,parameters,charge,histo,**kwargs) + pedestal = parameters['pedestal'] + pedestalWidth = parameters["pedestalWidth"] + pedestalWidth.value = pedestal.max - pedestal.value + pedestalWidth.max = 3 * pedestalWidth.value + log.debug(f"pedestalWidth updated : {pedestalWidth}") + try : + coeff,var_matrix = NectarGainSPE._get_mean_gaussian_fit(charge,histo,f'{it}_nominal') + if (coeff[1] - pedestal.value < 0) or ((coeff[1] - coeff[2]) - pedestal.max < 0) : raise Exception("mean gaussian fit not good") + mean = parameters['mean'] + mean.value = coeff[1] - pedestal.value + mean.min = (coeff[1] - coeff[2]) - pedestal.max + mean.max = (coeff[1] + coeff[2]) - pedestal.min + log.debug(f"mean updated : {mean}") + except Exception as e : + log.warning(e,exc_info=True) + log.warning("mean parameters limits and starting value not changed") + + #fit parameters + @property + def pp(self) : return self.__pp + @property + def luminosity(self) : return self.__luminosity + @property + def mean(self) : return self.__mean + @property + def n(self) : return self.__n + @property + def resolution(self) : return self.__resolution + @property + def pedestalWidth(self) : return self.__pedestalWidth + @property + def gain(self) : return self.__gain + + + #intern parameters + @property + def _old_lum(self) : return self.__old_lum + @_old_lum.setter + def _old_lum(self,value) : self.__old_lum = value + @property + def _old_ntotalPE(self) : return self.__old_ntotalPE + @_old_ntotalPE.setter + def _old_ntotalPE(self,value) : self.__old_ntotalPE = value + + + + class NectarGainSPESingle(NectarGainSPE): _Ncall = 100 _Nproc_Multiprocess = mlp.cpu_count() // 2 From 622b94f740309980a6768d4845d5fdc4790ada3e Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Tue, 5 Sep 2023 00:08:55 +0200 Subject: [PATCH 13/62] new skeleton for the calibration pipeline : -container module for the data structure (to be extend) -makers module to make the computation of calibration quantities on the data structure --- .../calibration/makers/__init__.py | 2 + src/nectarchain/calibration/makers/core.py | 27 ++ .../calibration/makers/flatfieldMakers.py | 16 + .../makers/gain/FlatFieldSPEMakers.py | 7 + .../makers/gain/PhotoStatisticMakers.py | 284 ++++++++++++++++++ .../makers/gain/WhiteTargetSPEMakers.py | 0 .../calibration/makers/gain/__init__.py | 3 + .../calibration/makers/gain/gainMakers.py | 17 ++ .../calibration/makers/pedestalMakers.py | 18 ++ 9 files changed, 374 insertions(+) create mode 100644 src/nectarchain/calibration/makers/__init__.py create mode 100644 src/nectarchain/calibration/makers/core.py create mode 100644 src/nectarchain/calibration/makers/flatfieldMakers.py create mode 100644 src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py create mode 100644 src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py create mode 100644 src/nectarchain/calibration/makers/gain/WhiteTargetSPEMakers.py create mode 100644 src/nectarchain/calibration/makers/gain/__init__.py create mode 100644 src/nectarchain/calibration/makers/gain/gainMakers.py create mode 100644 src/nectarchain/calibration/makers/pedestalMakers.py diff --git a/src/nectarchain/calibration/makers/__init__.py b/src/nectarchain/calibration/makers/__init__.py new file mode 100644 index 00000000..6c4b6168 --- /dev/null +++ b/src/nectarchain/calibration/makers/__init__.py @@ -0,0 +1,2 @@ +from .pedestalMakers import * +from .gain import * \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/core.py b/src/nectarchain/calibration/makers/core.py new file mode 100644 index 00000000..059b5fc9 --- /dev/null +++ b/src/nectarchain/calibration/makers/core.py @@ -0,0 +1,27 @@ +import sys +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +from abc import ABC, abstractmethod +__all__ = "" + +class CalibrationMaker(ABC) : + """mother class for all the calibration makers that can be defined to compute calibration coeficients from data + """ + + def __new__(cls) : + return super().__new__() + + + def __init__(self) -> None: + super().__init__() + + + @abstractmethod + def make(self,*args,**kwargs) : + pass + + + diff --git a/src/nectarchain/calibration/makers/flatfieldMakers.py b/src/nectarchain/calibration/makers/flatfieldMakers.py new file mode 100644 index 00000000..33fead89 --- /dev/null +++ b/src/nectarchain/calibration/makers/flatfieldMakers.py @@ -0,0 +1,16 @@ +import sys +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +from .core import CalibrationMaker + + +__all__ = "FlatfieldMaker" + +class FlatfieldMaker(CalibrationMaker) : + def __init__(self) : + pass + def make(self) : + raise NotImplementedError("The computation of the flatfield calibration is not yet implemented, feel free to contribute !:)") \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py new file mode 100644 index 00000000..5bc0f9b5 --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py @@ -0,0 +1,7 @@ +import sys +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +from . import GainMaker \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py b/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py new file mode 100644 index 00000000..826c6fa6 --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py @@ -0,0 +1,284 @@ +import sys +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + + +import numpy as np +from scipy.stats import linregress +from matplotlib import pyplot as plt +import astropy.units as u +from astropy.visualization import quantity_support +from astropy.table import QTable,Column +import os +from datetime import date +from pathlib import Path + +from . import GainMaker + +from ctapipe_io_nectarcam import constants + +from ...container import ChargeContainer + + +__all__ = ["PhotoStatGainFFandPed"] + +class PhotoStatGain(ABC): + + def _readFF(self,FFRun,maxevents: int = None,**kwargs) : + log.info('reading FF data') + method = kwargs.get('method','FullWaveformSum') + FFchargeExtractorWindowLength = kwargs.get('FFchargeExtractorWindowLength',None) + if method != 'FullWaveformSum' : + if FFchargeExtractorWindowLength is None : + e = Exception(f"we have to specify FFchargeExtractorWindowLength argument if charge extractor method is not FullwaveformSum") + log.error(e,exc_info=True) + raise e + else : + self.__coefCharge_FF_Ped = FFchargeExtractorWindowLength / constants.N_SAMPLES + else : + self.__coefCharge_FF_Ped = 1 + if isinstance(FFRun,int) : + try : + self.FFcharge = ChargeContainer.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{method}",FFRun) + log.info(f'charges have ever been computed for FF run {FFRun}') + except Exception as e : + log.error("charge have not been yet computed") + raise e + + #log.info(f'loading waveforms for FF run {FFRun}') + #FFwaveforms = WaveformsContainer(FFRun,maxevents) + #FFwaveforms.load_wfs() + #if method != 'std' : + # log.info(f'computing charge for FF run {FFRun} with following method : {method}') + # self.FFcharge = ChargeContainer.from_waveforms(FFwaveforms,method = method) + #else : + # log.info(f'computing charge for FF run {FFRun} with std method') + # self.FFcharge = ChargeContainer.from_waveforms(FFwaveforms) + #log.debug('writting on disk charge for further works') + #os.makedirs(f"{os.environ['NECTARCAMDATA']}/charges/{method}",exist_ok = True) + #self.FFcharge.write(f"{os.environ['NECTARCAMDATA']}/charges/{method}",overwrite = True) + + elif isinstance(FFRun,ChargeContainer): + self.FFcharge = FFRun + else : + e = TypeError("FFRun must be int or ChargeContainer") + log.error(e,exc_info = True) + raise e + + def _readPed(self,PedRun,maxevents: int = None,**kwargs) : + log.info('reading Ped data') + method = 'FullWaveformSum'#kwargs.get('method','std') + if isinstance(PedRun,int) : + try : + self.Pedcharge = ChargeContainer.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{method}",PedRun) + log.info(f'charges have ever been computed for Ped run {PedRun}') + except Exception as e : + log.error("charge have not been yet computed") + raise e + + #log.info(f'loading waveforms for Ped run {PedRun}') + #Pedwaveforms = WaveformsContainer(PedRun,maxevents) + #Pedwaveforms.load_wfs() + #if method != 'std' : + # log.info(f'computing charge for Ped run {PedRun} with following method : {method}') + # self.Pedcharge = ChargeContainer.from_waveforms(Pedwaveforms,method = method) + #else : + # log.info(f'computing charge for Ped run {PedRun} with std method') + # self.Pedcharge = ChargeContainer.from_waveforms(Pedwaveforms) + #log.debug('writting on disk charge for further works') + #os.makedirs(f"{os.environ['NECTARCAMDATA']}/charges/{method}",exist_ok = True) + #self.Pedcharge.write(f"{os.environ['NECTARCAMDATA']}/charges/{method}",overwrite = True) + + elif isinstance(PedRun,ChargeContainer): + self.Pedcharge = PedRun + else : + e = TypeError("PedRun must be int or ChargeContainer") + log.error(e,exc_info = True) + raise e + + def _readSPE(self,SPEresults) : + log.info(f'reading SPE resolution from {SPEresults}') + table = QTable.read(SPEresults) + table.sort('pixel') + self.SPEResolution = table['resolution'] + self.SPEGain = table['gain'] + self.SPEGain_error = table['gain_error'] + self._SPEvalid = table['is_valid'] + self._SPE_pixels_id = np.array(table['pixel'].value,dtype = np.uint16) + + def _reshape_all(self) : + log.info("reshape of SPE, Ped and FF data with intersection of pixel ids") + FFped_intersection = np.intersect1d(self.Pedcharge.pixels_id,self.FFcharge.pixels_id) + SPEFFPed_intersection = np.intersect1d(FFped_intersection,self._SPE_pixels_id[self._SPEvalid]) + self._pixels_id = SPEFFPed_intersection + log.info(f"data have {len(self._pixels_id)} pixels in common") + + self._FFcharge_hg = self.FFcharge.select_charge_hg(SPEFFPed_intersection) + self._FFcharge_lg = self.FFcharge.select_charge_lg(SPEFFPed_intersection) + + self._Pedcharge_hg = self.Pedcharge.select_charge_hg(SPEFFPed_intersection) + self._Pedcharge_lg = self.Pedcharge.select_charge_lg(SPEFFPed_intersection) + + #self._mask_FF = np.array([self.FFcharge.pixels_id[i] in SPEFFPed_intersection for i in range(self.FFcharge.npixels)],dtype = bool) + #self._mask_Ped = np.array([self.Pedcharge.pixels_id[i] in SPEFFPed_intersection for i in range(self.Pedcharge.npixels)],dtype = bool) + self._mask_SPE = np.array([self._SPE_pixels_id[i] in SPEFFPed_intersection for i in range(len(self._SPE_pixels_id))],dtype = bool) + + + + def create_output_table(self) : + self._output_table = QTable() + self._output_table.meta['npixel'] = self.npixels + self._output_table.meta['comments'] = f'Produced with NectarGain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' + + self._output_table.add_column(Column(np.ones((self.npixels),dtype = bool),"is_valid",unit = u.dimensionless_unscaled)) + self._output_table.add_column(Column(self.pixels_id,"pixel",unit = u.dimensionless_unscaled)) + self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),"high gain",unit = u.dimensionless_unscaled)) + self._output_table.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"high gain error",unit = u.dimensionless_unscaled)) + self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),"low gain",unit = u.dimensionless_unscaled)) + self._output_table.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"low gain error",unit = u.dimensionless_unscaled)) + + def run(self,**kwargs): + log.info('running photo statistic method') + + self._output_table["high gain"] = self.gainHG + self._output_table["low gain"] = self.gainLG + #self._output_table["is_valid"] = self._SPEvalid + + def save(self,path,**kwargs) : + path = Path(path) + os.makedirs(path,exist_ok = True) + log.info(f'data saved in {path}') + self._output_table.write(f"{path}/output_table.ecsv", format='ascii.ecsv',overwrite = kwargs.get("overwrite",False)) + + def plot_correlation(self) : + mask = (self._output_table["high gain"]>20) * (self.SPEGain[self._mask_SPE]>0) * (self._output_table["high gain"]<80) * self._output_table['is_valid'] + a, b, r, p_value, std_err = linregress(self._output_table["high gain"][mask], self.SPEGain[self._mask_SPE][mask],'greater') + x = np.linspace(self._output_table["high gain"][mask].min(),self._output_table["high gain"][mask].max(),1000) + y = lambda x: a * x + b + with quantity_support() : + fig,ax = plt.subplots(1,1,figsize=(8, 6)) + ax.scatter(self._output_table["high gain"][mask],self.SPEGain[self._mask_SPE][mask],marker =".") + ax.plot(x,y(x),color = 'red', label = f"linear fit,\n a = {a:.2e},\n b = {b:.2e},\n r = {r:.2e},\n p_value = {p_value:.2e},\n std_err = {std_err:.2e}") + ax.plot(x,x,color = 'black',label = "y = x") + ax.set_xlabel("Gain Photo stat (ADC)", size=15) + ax.set_ylabel("Gain SPE fit (ADC)", size=15) + #ax.set_xlim(xmin = 0) + #ax.set_ylim(ymin = 0) + + ax.legend(fontsize=15) + return fig + + @property + def npixels(self) : return self._pixels_id.shape[0] + + @property + def pixels_id(self) : return self._pixels_id + + @property + def sigmaPedHG(self) : return np.std(self._Pedcharge_hg ,axis = 0) * np.sqrt(self.__coefCharge_FF_Ped) + + @property + def sigmaChargeHG(self) : return np.std(self._FFcharge_hg - self.meanPedHG, axis = 0) + + @property + def meanPedHG(self) : return np.mean(self._Pedcharge_hg ,axis = 0) * self.__coefCharge_FF_Ped + + @property + def meanChargeHG(self) : return np.mean(self._FFcharge_hg - self.meanPedHG, axis = 0) + + @property + def BHG(self) : + min_events = np.min((self._FFcharge_hg.shape[0],self._Pedcharge_hg.shape[0])) + upper = (np.power(self._FFcharge_hg.mean(axis = 1)[:min_events] - self._Pedcharge_hg.mean(axis = 1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeHG.mean(),2)).mean(axis = 0) + lower = np.power(self.meanChargeHG.mean(),2)#np.power(self.meanChargeHG,2)#np.power(self.meanChargeHG.mean(),2) + return np.sqrt(upper/lower) + + @property + def gainHG(self) : + return ((np.power(self.sigmaChargeHG,2) - np.power(self.sigmaPedHG,2) - np.power(self.BHG * self.meanChargeHG,2)) + /(self.meanChargeHG * (1 + np.power(self.SPEResolution[self._mask_SPE],2)))) + + + @property + def sigmaPedLG(self) : return np.std(self._Pedcharge_lg ,axis = 0) * np.sqrt(self.__coefCharge_FF_Ped) + + @property + def sigmaChargeLG(self) : return np.std(self._FFcharge_lg - self.meanPedLG,axis = 0) + + @property + def meanPedLG(self) : return np.mean(self._Pedcharge_lg,axis = 0) * self.__coefCharge_FF_Ped + + @property + def meanChargeLG(self) : return np.mean(self._FFcharge_lg - self.meanPedLG,axis = 0) + + @property + def BLG(self) : + min_events = np.min((self._FFcharge_lg.shape[0],self._Pedcharge_lg.shape[0])) + upper = (np.power(self._FFcharge_lg.mean(axis = 1)[:min_events] - self._Pedcharge_lg.mean(axis = 1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeLG.mean(),2)).mean(axis = 0) + lower = np.power(self.meanChargeLG.mean(),2) #np.power(self.meanChargeLG,2) #np.power(self.meanChargeLG.mean(),2) + return np.sqrt(upper/lower) + + @property + def gainLG(self) : return ((np.power(self.sigmaChargeLG,2) - np.power(self.sigmaPedLG,2) - np.power(self.BLG * self.meanChargeLG,2)) + /(self.meanChargeLG * (1 + np.power(self.SPEResolution[self._mask_SPE],2)))) + + + +class PhotoStatGainFFandPed(PhotoStatGain): + def __init__(self, FFRun, PedRun, SPEresults : str, maxevents : int = None, **kwargs) : + self._readFF(FFRun,maxevents,**kwargs) + self._readPed(PedRun,maxevents,**kwargs) + + """ + if self.FFcharge.charge_hg.shape[1] != self.Pedcharge.charge_hg.shape[1] : + e = Exception("Ped run and FF run must have the same number of pixels") + log.error(e,exc_info = True) + raise e + """ + + self._readSPE(SPEresults) + ##need to implement reshape of SPE results with FF and Ped pixels ids + self._reshape_all() + + """ + if (self.FFcharge.pixels_id.shape[0] != self._SPE_pixels_id.shape[0]) : + e = Exception("Ped run and FF run must have the same number of pixels as SPE fit results") + log.error(e,exc_info = True) + raise e + + if (self.FFcharge.pixels_id != self.Pedcharge.pixels_id).any() or (self.FFcharge.pixels_id != self._SPE_pixels_id).any() or (self.Pedcharge.pixels_id != self._SPE_pixels_id).any() : + e = DifferentPixelsID("Ped run, FF run and SPE run need to have same pixels id") + log.error(e,exc_info = True) + raise e + else : + self._pixels_id = self.FFcharge.pixels_id + """ + self.create_output_table() + + + +class PhotoStatGainFF(PhotoStatGain): + def __init__(self, FFRun, PedRun, SPEresults : str, maxevents : int = None, **kwargs) : + e = NotImplementedError("PhotoStatGainFF is not yet implemented") + log.error(e, exc_info = True) + raise e + self._readFF(FFRun,maxevents,**kwargs) + + self._readSPE(SPEresults) + + """ + if self.FFcharge.pixels_id != self._SPE_pixels_id : + e = DifferentPixelsID("Ped run, FF run and SPE run need to have same pixels id") + log.error(e,exc_info = True) + raise e + else : + self._pixels_id = self.FFcharge.pixels_id + """ + + self._reshape_all() + + self.create_output_table() + +class PhotoStatisticMaker(GainMaker) : diff --git a/src/nectarchain/calibration/makers/gain/WhiteTargetSPEMakers.py b/src/nectarchain/calibration/makers/gain/WhiteTargetSPEMakers.py new file mode 100644 index 00000000..e69de29b diff --git a/src/nectarchain/calibration/makers/gain/__init__.py b/src/nectarchain/calibration/makers/gain/__init__.py new file mode 100644 index 00000000..5de277b6 --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/__init__.py @@ -0,0 +1,3 @@ +from .FlatFieldSPEMakers import * +from .WhiteTargetSPEMakers import * +from .PhotoStatisticMakers import * \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/gainMakers.py b/src/nectarchain/calibration/makers/gain/gainMakers.py new file mode 100644 index 00000000..ee6c0b5b --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/gainMakers.py @@ -0,0 +1,17 @@ +import sys +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +from ..core import CalibrationMaker + +__all__ = "GainMaker" + +class GainMaker(CalibrationMaker) : + """mother class for of the gain calibration + """ + + def __init__(self) -> None: + super().__init__() + \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/pedestalMakers.py b/src/nectarchain/calibration/makers/pedestalMakers.py new file mode 100644 index 00000000..79f6ee72 --- /dev/null +++ b/src/nectarchain/calibration/makers/pedestalMakers.py @@ -0,0 +1,18 @@ +import sys +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +from .core import CalibrationMaker + + +__all__ = "PedestalMaker" + +class PedestalMaker(CalibrationMaker) : + def __init__(self) : + pass + def make(self) : + raise NotImplementedError("The computation of the pedestal calibration is not yet implemented, feel free to contribute !:)") + + \ No newline at end of file From 55725c0b42b18ea231f4098d9050c4d97acb65e6 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 7 Sep 2023 18:47:21 +0200 Subject: [PATCH 14/62] mothers class for the maker module --- .../calibration/makers/__init__.py | 3 +- src/nectarchain/calibration/makers/core.py | 43 ++++++++++++++++--- .../calibration/makers/flatfieldMakers.py | 7 +-- .../calibration/makers/pedestalMakers.py | 7 ++- 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/nectarchain/calibration/makers/__init__.py b/src/nectarchain/calibration/makers/__init__.py index 6c4b6168..2baa1183 100644 --- a/src/nectarchain/calibration/makers/__init__.py +++ b/src/nectarchain/calibration/makers/__init__.py @@ -1,2 +1,3 @@ from .pedestalMakers import * -from .gain import * \ No newline at end of file +from .gain import * +from .flatfieldMakers import * \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/core.py b/src/nectarchain/calibration/makers/core.py index 059b5fc9..e0ed8a0e 100644 --- a/src/nectarchain/calibration/makers/core.py +++ b/src/nectarchain/calibration/makers/core.py @@ -4,24 +4,53 @@ log = logging.getLogger(__name__) log.handlers = logging.getLogger('__main__').handlers -from abc import ABC, abstractmethod -__all__ = "" +from abc import ABC, ABCMeta, abstractmethod + +from astropy.table import QTable,Column +import astropy.units as u +from copy import copy +from datetime import date + + +__all__ = [""] class CalibrationMaker(ABC) : """mother class for all the calibration makers that can be defined to compute calibration coeficients from data """ - def __new__(cls) : - return super().__new__() +#constructors + def __new__(cls,*args,**kwargs) : + return super(CalibrationMaker,cls).__new__(cls) - - def __init__(self) -> None: + def __init__(self,*args,**kwargs) -> None: super().__init__() + self.__pixels_id = kwargs.get("pixels_id", None) + self.__results = QTable() + self.__results.add_column(Column(self.__pixels_id,"pixels_id",unit = u.dimensionless_unscaled)) + self.__results.meta['npixels'] = self.npixels + self.__results.meta['comments'] = f'Produced with NectarChain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' - +#methods @abstractmethod def make(self,*args,**kwargs) : pass +#getters and setters + @property + def _pixels_id(self) : return self.__pixels_id + @_pixels_id.setter + def _pixels_id(self,value) : self.__pixels_id = value + @property + def pixels_id(self) : return copy(self.__pixels_id) + + @property + def npixels(self) : return len(self.__pixels_id) + + @property + def _results(self) : return self.__results + @property + def results(self) : return copy(self.__results) + + diff --git a/src/nectarchain/calibration/makers/flatfieldMakers.py b/src/nectarchain/calibration/makers/flatfieldMakers.py index 33fead89..f2cf419d 100644 --- a/src/nectarchain/calibration/makers/flatfieldMakers.py +++ b/src/nectarchain/calibration/makers/flatfieldMakers.py @@ -7,10 +7,11 @@ from .core import CalibrationMaker -__all__ = "FlatfieldMaker" +__all__ = ["FlatfieldMaker"] class FlatfieldMaker(CalibrationMaker) : - def __init__(self) : - pass + def __init__(self,**kwargs) : + super().__init__(**kwargs) + def make(self) : raise NotImplementedError("The computation of the flatfield calibration is not yet implemented, feel free to contribute !:)") \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/pedestalMakers.py b/src/nectarchain/calibration/makers/pedestalMakers.py index 79f6ee72..b99d90fd 100644 --- a/src/nectarchain/calibration/makers/pedestalMakers.py +++ b/src/nectarchain/calibration/makers/pedestalMakers.py @@ -6,12 +6,11 @@ from .core import CalibrationMaker - -__all__ = "PedestalMaker" +__all__ = ["PedestalMaker"] class PedestalMaker(CalibrationMaker) : - def __init__(self) : - pass + def __init__(self,**kwargs) : + super().__init__(**kwargs) def make(self) : raise NotImplementedError("The computation of the pedestal calibration is not yet implemented, feel free to contribute !:)") From 8bb7ca1566c9d826734882cee3cbc726b46c6a1e Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 7 Sep 2023 18:48:06 +0200 Subject: [PATCH 15/62] mother class for the gain makers --- .../calibration/makers/gain/gainMakers.py | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/nectarchain/calibration/makers/gain/gainMakers.py b/src/nectarchain/calibration/makers/gain/gainMakers.py index ee6c0b5b..abe8d3bf 100644 --- a/src/nectarchain/calibration/makers/gain/gainMakers.py +++ b/src/nectarchain/calibration/makers/gain/gainMakers.py @@ -4,14 +4,35 @@ log = logging.getLogger(__name__) log.handlers = logging.getLogger('__main__').handlers +import os +import numpy as np +from copy import copy +from astropy.table import Column +import astropy.units as u + from ..core import CalibrationMaker -__all__ = "GainMaker" +__all__ = ["GainMaker"] class GainMaker(CalibrationMaker) : """mother class for of the gain calibration """ - def __init__(self) -> None: - super().__init__() +#constructors + def __init__(self,*args,**kwargs) -> None: + super().__init__(*args,**kwargs) + self.__gain = np.empty((self.npixels),dtype = np.float64) + self._results.add_column(Column(data = self.__gain,name = "gain",unit = u.dimensionless_unscaled)) + self._results.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"gain_error",unit = u.dimensionless_unscaled)) + + self._results.add_column(Column(np.zeros((self.npixels),dtype = bool),"is_valid",unit = u.dimensionless_unscaled)) + + +#getters and setters + @property + def _gain(self) : return self.__gain + @_gain.setter + def _gain(self,value) : self.__gain = value + @property + def gain(self) : return copy(self.__gain) \ No newline at end of file From 86161c4ccf64e9747265f141dc7e66d47e099220 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 7 Sep 2023 18:48:55 +0200 Subject: [PATCH 16/62] unchanged modeuls for the new structure --- .../calibration/makers/gain/parameters.py | 104 ++++++ .../makers/gain/parameters_ped.yaml | 7 + .../makers/gain/parameters_signal.yaml | 32 ++ .../makers/gain/parameters_signalStd.yaml | 32 ++ .../gain/parameters_signal_combined.yaml | 42 +++ .../gain/parameters_signal_fromHHVFit.yaml | 32 ++ .../calibration/makers/gain/utils.py | 298 ++++++++++++++++++ 7 files changed, 547 insertions(+) create mode 100644 src/nectarchain/calibration/makers/gain/parameters.py create mode 100644 src/nectarchain/calibration/makers/gain/parameters_ped.yaml create mode 100644 src/nectarchain/calibration/makers/gain/parameters_signal.yaml create mode 100644 src/nectarchain/calibration/makers/gain/parameters_signalStd.yaml create mode 100644 src/nectarchain/calibration/makers/gain/parameters_signal_combined.yaml create mode 100644 src/nectarchain/calibration/makers/gain/parameters_signal_fromHHVFit.yaml create mode 100644 src/nectarchain/calibration/makers/gain/utils.py diff --git a/src/nectarchain/calibration/makers/gain/parameters.py b/src/nectarchain/calibration/makers/gain/parameters.py new file mode 100644 index 00000000..b01e48f8 --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/parameters.py @@ -0,0 +1,104 @@ +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) + +import copy +import numpy as np + +import astropy.units as u + +__all__ = ["Parameter","Parameters"] + +class Parameter() : + def __init__(self, name, value, min = np.nan, max = np.nan, error = np.nan, unit = u.dimensionless_unscaled, frozen : bool = False): + self.__name = name + self.__value = value + self.__error = error + self.__min = min + self.__max = max + self.__unit = unit + self.__frozen = frozen + + @classmethod + def from_instance(cls,parameter): + return cls(parameter.name,parameter.value,parameter.min,parameter.max,parameter.unit,parameter.frozen) + + def __str__(self): + return f"name : {self.__name}, value : {self.__value}, error : {self.__error}, unit : {self.__unit}, min : {self.__min}, max : {self.__max},frozen : {self.__frozen}" + + + @property + def name(self) : return self.__name + @name.setter + def name(self,value) : self.__name = value + + @property + def value(self) : return self.__value + @value.setter + def value(self,value) : self.__value = value + + @property + def min(self): return self.__min + @min.setter + def min(self,value) : self.__min = value + + @property + def max(self): return self.__max + @max.setter + def max(self,value) : self.__max = value + + @property + def unit(self) : return self.__unit + @unit.setter + def unit(self,value) : self.__unit = value + + @property + def error(self) : return self.__error + @error.setter + def error(self,value) : self.__error = value + + @property + def frozen(self) : return self.__frozen + @frozen.setter + def frozen(self,value : bool) : self.__frozen = value + +class Parameters() : + def __init__(self,parameters_liste : list = []) -> None: + self.__parameters = copy.deepcopy(parameters_liste) + + + def append(self,parameter : Parameter) : + self.__parameters.append(parameter) + + def __getitem__(self,key) : + for parameter in self.__parameters : + if parameter.name == key : + return parameter + return [] + + def __str__(self): + string="" + for parameter in self.__parameters : + string += str(parameter)+"\n" + return string + + + @property + def parameters(self) : return self.__parameters + + @property + def size(self) : return len(self.__parameters) + + @property + def parnames(self) : return [parameter.name for parameter in self.__parameters] + + @property + def parvalues(self) : return [parameter.value for parameter in self.__parameters] + + @property + def unfrozen(self) : + parameters = Parameters() + for parameter in self.__parameters : + if not(parameter.frozen) : + parameters.append(parameter) + return parameters diff --git a/src/nectarchain/calibration/makers/gain/parameters_ped.yaml b/src/nectarchain/calibration/makers/gain/parameters_ped.yaml new file mode 100644 index 00000000..e1fab9db --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/parameters_ped.yaml @@ -0,0 +1,7 @@ +{ + pedestalWidth : { + value : 50, + min : 1, + max : 150 + } +} \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/parameters_signal.yaml b/src/nectarchain/calibration/makers/gain/parameters_signal.yaml new file mode 100644 index 00000000..89ece153 --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/parameters_signal.yaml @@ -0,0 +1,32 @@ +{ + pedestalWidth : { + value : 50, + min : 1, + max : 150 + }, + luminosity : { + value : 1, + min : 0.01, + max : 5.0 + }, + pp : { + value : 0.45, + min : 0.2, + max : 0.8 + }, + resolution : { + value : 0.5, + min : 0.3, + max : 0.7 + }, + n : { + value : 0.697, + min : 0.5, + max : 0.9 + }, + mean : { + value : 500, + min : 200, + max : 600 + } +} \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/parameters_signalStd.yaml b/src/nectarchain/calibration/makers/gain/parameters_signalStd.yaml new file mode 100644 index 00000000..12746d7f --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/parameters_signalStd.yaml @@ -0,0 +1,32 @@ +{ + pedestalWidth : { + value : 50, + min : 1, + max : 150 + }, + luminosity : { + value : 1, + min : 0.01, + max : 5.0 + }, + pp : { + value : 0.45, + min : .NAN, + max : .NAN + }, + resolution : { + value : 0.5, + min : 0.3, + max : 0.7 + }, + n : { + value : 0.697, + min : .NAN, + max : .NAN + }, + mean : { + value : 500, + min : 200, + max : 600 + } +} \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/parameters_signal_combined.yaml b/src/nectarchain/calibration/makers/gain/parameters_signal_combined.yaml new file mode 100644 index 00000000..a04fc05a --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/parameters_signal_combined.yaml @@ -0,0 +1,42 @@ +{ + pedestalWidth : { + value : 50, + min : 1, + max : 150 + }, + luminosity : { + value : 1, + min : 0.01, + max : 5.0 + }, + luminosityHHV : { + value : 1, + min : 0.01, + max : 5.0 + }, + pp : { + value : 0.45, + min : .NAN, + max : .NAN + }, + resolution : { + value : 0.5, + min : 0.3, + max : 0.7 + }, + n : { + value : 0.697, + min : .NAN, + max : .NAN + }, + mean : { + value : 50, + min : 10, + max : 200 + }, + meanHHV : { + value : 500, + min : 200, + max : 600 + } +} \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/parameters_signal_fromHHVFit.yaml b/src/nectarchain/calibration/makers/gain/parameters_signal_fromHHVFit.yaml new file mode 100644 index 00000000..62515304 --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/parameters_signal_fromHHVFit.yaml @@ -0,0 +1,32 @@ +{ + pedestalWidth : { + value : 50, + min : 1, + max : 150 + }, + luminosity : { + value : 1, + min : 0.01, + max : 5.0 + }, + pp : { + value : 0.45, + min : .NAN, + max : .NAN + }, + resolution : { + value : 0.5, + min : .NAN, + max : .NAN + }, + n : { + value : 0.697, + min : .NAN, + max : .NAN + }, + mean : { + value : 50, + min : 10, + max : 200 + } +} \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/utils.py b/src/nectarchain/calibration/makers/gain/utils.py new file mode 100644 index 00000000..44686680 --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/utils.py @@ -0,0 +1,298 @@ +import logging +import math + +import numpy as np +from iminuit import Minuit +from scipy import interpolate, signal +from scipy.special import gammainc +from scipy.stats import norm,chi2 + +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +__all__ = ['UtilsMinuit','Multiprocessing'] +from .parameters import Parameters + +class Multiprocessing() : + @staticmethod + def custom_error_callback(error): + log.error(f'Got an error: {error}') + log.error(error,exc_info=True) + +class Statistics() : + @staticmethod + def chi2_pvalue(ndof : int, likelihood : float) : + return 1 - chi2(df = ndof).cdf(likelihood) + +class UtilsMinuit() : + @staticmethod + def make_minuit_par_kwargs(parameters : Parameters): + """Create *Parameter Keyword Arguments* for the `Minuit` constructor. + + updated for Minuit >2.0 + """ + names = parameters.parnames + kwargs = {"names": names,"values" : {}} + + for parameter in parameters.parameters: + kwargs["values"][parameter.name] = parameter.value + min_ = None if np.isnan(parameter.min) else parameter.min + max_ = None if np.isnan(parameter.max) else parameter.max + error = 0.1 if np.isnan(parameter.error) else parameter.error + kwargs[f"limit_{parameter.name}"] = (min_, max_) + kwargs[f"error_{parameter.name}"] = error + if parameter.frozen : + kwargs[f"fix_{parameter.name}"] = True + return kwargs + + @staticmethod + def set_minuit_parameters_limits_and_errors(m : Minuit,parameters : dict) : + """function to set minuit parameter limits and errors with Minuit >2.0 + + Args: + m (Minuit): a Minuit instance + parameters (dict): dict containing parameters names, limits errors and values + """ + for name in parameters["names"] : + m.limits[name] = parameters[f"limit_{name}"] + m.errors[name] = parameters[f"error_{name}"] + if parameters.get(f"fix_{name}",False) : + m.fixed[name] = True + + +# Usefull fucntions for the fit +def gaussian(x, mu, sig): + #return (1./(sig*np.sqrt(2*math.pi)))*np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.))) + return norm.pdf(x,loc = mu,scale = sig) + +def weight_gaussian(x,N, mu, sig) : + return N * gaussian(x, mu, sig) + +def doubleGauss(x,sig1,mu2,sig2,p): + return p *2 *gaussian(x, 0, sig1) + (1-p) * gaussian(x, mu2, sig2) + +def PMax(r): + """p_{max} in equation 6 in Caroff et al. (2019) + + Args: + r (float): SPE resolution + + Returns: + float : p_{max} + """ + if r > np.sqrt((np.pi -2 )/ 2) : + pmax = np.pi/(2 * (r**2 + 1)) + else : + pmax = np.pi*r**2/(np.pi*r**2 + np.pi - 2*r**2 - 2) + return pmax + +def ax(p,res): + """a in equation 4 in Caroff et al. (2019) + + Args: + p (float): proportion of the low charge component (2 gaussians model) + res (float): SPE resolution + + Returns: + float : a + """ + return ((2/np.pi)*p**2-p/(res**2+1)) + +def bx(p,mu2): + """b in equation 4 in Caroff et al. (2019) + + Args: + p (float): proportion of the low charge component (2 gaussians model) + mu2 (float): position of the high charge Gaussian + + Returns: + float : b + """ + return (np.sqrt(2/np.pi)*2*p*(1-p)*mu2) + +def cx(sig2,mu2,res,p): + """c in equation 4 in Caroff et al. (2019) + Note : There is a typo in the article 1-p**2 -> (1-p)**2 + + Args: + sig2 (float): width of the high charge Gaussian + mu2 (float): position of the high charge Gaussian + res (float): SPE resolution + p (float): proportion of the low charge component (2 gaussians model) + + Returns: + float : c + """ + return (1-p)**2*mu2**2 - (1-p)*(sig2**2+mu2**2)/(res**2+1) + +def delta(p,res,sig2,mu2): + """well known delta in 2nd order polynom + + Args: + p (_type_): _description_ + res (_type_): _description_ + sig2 (_type_): _description_ + mu2 (_type_): _description_ + + Returns: + float : b**2 - 4*a*c + """ + return bx(p,mu2)*bx(p,mu2) - 4*ax(p,res)*cx(sig2,mu2,res,p) + +def ParamU(p,r): + """d in equation 6 in Caroff et al. (2019) + + Args: + p (float): proportion of the low charge component (2 gaussians model) + r (float): SPE resolution + + Returns: + float : d + """ + return ((8*(1-p)**2*p**2)/np.pi - 4*(2*p**2/np.pi - p/(r**2+1))*((1-p)**2-(1-p)/(r**2+1))) + +def ParamS(p,r): + """e in equation 6 in Caroff et al. (2019) + + Args: + p (float): proportion of the low charge component (2 gaussians model) + r (float): SPE resolution + + Returns: + float : e + """ + e = (4*(2*p**2/np.pi - p/(r**2+1))*(1-p))/(r**2+1) + return e + +def SigMin(p,res,mu2): + """sigma_{high,min} in equation 6 in Caroff et al. (2019) + + Args: + p (float): proportion of the low charge component (2 gaussians model) + res (float): SPE resolution + mu2 (float): position of the high charge Gaussian + + Returns: + float : sigma_{high,min} + """ + return mu2*np.sqrt((-ParamU(p,res)+(bx(p,mu2)**2/mu2**2))/(ParamS(p,res))) + +def SigMax(p,res,mu2): + """sigma_{high,min} in equation 6 in Caroff et al. (2019) + + Args: + p (float): proportion of the low charge component (2 gaussians model) + res (float): SPE resolution + mu2 (float): position of the high charge Gaussian + + Returns: + float : sigma_{high,min} + """ + temp = (-ParamU(p,res))/(ParamS(p,res)) + if temp < 0 : + err = ValueError("-d/e must be < 0") + log.error(err,exc_info=True) + raise err + else : + return mu2*np.sqrt(temp) + +def sigma1(p,res,sig2,mu2): + """sigma_{low} in equation 5 in Caroff et al. (2019) + + Args: + sig2 (float): width of the high charge Gaussian + mu2 (float): position of the high charge Gaussian + res (float): SPE resolution + p (float): proportion of the low charge component (2 gaussians model) + + Returns: + float : sigma_{low} + """ + return (-bx(p,mu2)+np.sqrt(delta(p,res,sig2,mu2)))/(2*ax(p,res)) + +def sigma2(n,p,res,mu2): + """sigma_{high} in equation 7 in Caroff et al. (2019) + + Args: + n (float): parameter n in equation + mu2 (float): position of the high charge Gaussian + res (float): SPE resolution + p (float): proportion of the low charge component (2 gaussians model) + + Returns: + float : sigma_{high} + """ + if ((-ParamU(p,res)+(bx(p,mu2)**2/mu2**2))/(ParamS(p,res)) > 0): + return SigMin(p,res,mu2)+n*(SigMax(p,res,mu2)-SigMin(p,res,mu2)) + else: + return n*SigMax(p,res,mu2) + + # The real final model callign all the above for luminosity (lum) + PED, wil return probability of number of Spe +def MPE2(x,pp,res,mu2,n,muped,sigped,lum,**kwargs): + log.debug(f"pp = {pp}, res = {res}, mu2 = {mu2}, n = {n}, muped = {muped}, sigped = {sigped}, lum = {lum}") + f = 0 + ntotalPE = kwargs.get("ntotalPE",0) + if ntotalPE == 0 : + #about 1sec + for i in range(1000): + if (gammainc(i+1,lum) < 1e-5): + ntotalPE = i + break + #print(ntotalPE) + #about 8 sec, 1 sec by nPEPDF call + #for i in range(ntotalPE): + # f = f + ((lum**i)/math.factorial(i)) * np.exp(-lum) * nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) + + f = np.sum([(lum**i)/math.factorial(i) * np.exp(-lum) * nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) for i in range(ntotalPE)],axis = 0) # 10 % faster + return f + +# Fnal model shape/function (for one SPE) +def doubleGaussConstrained(x,pp,res,mu2,n): + p = pp*PMax(res) + sig2 = sigma2(n,p,res,mu2) + sig1 = sigma1(p,res,sig2,mu2) + return doubleGauss(x,sig1,mu2,sig2,p) + +# Get the gain from the parameters model +def Gain(pp,res,mu2,n): + """analytic gain computatuon + + Args: + mu2 (float): position of the high charge Gaussian + res (float): SPE resolution + pp (float): p' in equation 7 in Caroff et al. (2019) + n (float): n in equation 7 in Caroff et al. (2019) + + Returns: + float : gain + """ + p = pp*PMax(res) + sig2 = sigma2(n,p,res,mu2) + return (1-p)*mu2 + 2*p*sigma1(p,res,sig2,mu2)/np.sqrt(2*np.pi) + +def nPEPDF(x,pp,res,mu2,n,muped,sigped,nph,size_charge): + allrange = np.linspace(-1 * size_charge,size_charge,size_charge*2) + spe = [] + #about 2 sec this is the main pb + #for i in range(len(allrange)): + # if (allrange[i]>=0): + # spe.append(doubleGaussConstrained(allrange[i],pp,res,mu2,n)) + # else: + # spe.append(0) + + spe = doubleGaussConstrained(allrange,pp,res,mu2,n) * (allrange>=0 * np.ones(allrange.shape)) #100 times faster + + # ~ plt.plot(allrange,spe) + #npe = semi_gaussian(allrange, muped, sigped) + npe = gaussian(allrange, 0, sigped) + # ~ plt.plot(allrange,npe) + # ~ plt.show() + for i in range(nph): + #npe = np.convolve(npe,spe,"same") + npe = signal.fftconvolve(npe,spe,"same") + # ~ plt.plot(allrange,npe) + # ~ plt.show() + fff = interpolate.UnivariateSpline(allrange,npe,ext=1,k=3,s=0) + norm = np.trapz(fff(allrange),allrange) + return fff(x-muped)/norm \ No newline at end of file From ebd86fc4be109289f2275d85f35cc328f8b1f851 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 7 Sep 2023 18:49:21 +0200 Subject: [PATCH 17/62] FlatFieldSPEMaker implementation --- .../makers/gain/FlatFieldSPEMakers.py | 428 +++++++++++++++++- .../calibration/makers/gain/__init__.py | 4 +- 2 files changed, 429 insertions(+), 3 deletions(-) diff --git a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py index 5bc0f9b5..3600edc5 100644 --- a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py @@ -4,4 +4,430 @@ log = logging.getLogger(__name__) log.handlers = logging.getLogger('__main__').handlers -from . import GainMaker \ No newline at end of file +import numpy as np +import astropy.units as u +from astropy.table import Column + +import copy +import os +import yaml +import time + +from pathlib import Path + +import matplotlib.pyplot as plt +from matplotlib.patches import Rectangle +from matplotlib.colors import to_rgba + +import numpy as np + +from iminuit import Minuit + +from scipy.signal import find_peaks +from scipy.signal import savgol_filter +from scipy.optimize import curve_fit +from scipy.special import gammainc + +from multiprocessing import Pool + +from inspect import signature + + +from .gainMakers import GainMaker + +from ...container import ChargeContainer + +from .parameters import Parameter, Parameters + +from .utils import UtilsMinuit,weight_gaussian,Statistics,MPE2 + + + + +__all__ = ["FlatFieldSingleSPEMaker"] + + + +class FlatFieldSPEMaker(GainMaker) : + _Windows_lenght = 40 + _Order = 2 + +#constructors + def __init__(self,*args,**kwargs) : + super().__init__(*args,**kwargs) + self.__parameters = Parameters() + +#getters and setters + @property + def npixels(self) : return len(self._pixels_id) + @property + def parameters(self) : return copy.deepcopy(self.__parameters) + @property + def _parameters(self) : return self.__parameters + +#methods + def read_param_from_yaml(self,parameters_file,only_update = False) : + with open(f"{os.path.dirname(os.path.abspath(__file__))}/{parameters_file}") as parameters : + param = yaml.safe_load(parameters) + if only_update : + for i,name in enumerate(self.__parameters.parnames) : + dico = param.get(name,False) + if dico : + self._parameters.parameters[i].value = dico.get('value') + self._parameters.parameters[i].min = dico.get("min",np.nan) + self._parameters.parameters[i].max = dico.get("max",np.nan) + else : + for name,dico in param.items() : + setattr(self, + f"__{name}", + Parameter( + name = name, + value = dico["value"], + min = dico.get("min",np.nan), + max = dico.get("max",np.nan), + unit = dico.get("unit",u.dimensionless_unscaled) + )) + self._parameters.append(eval(f"self.__{name}")) + + @staticmethod + def _update_parameters(parameters,charge,counts) : + coeff_ped,coeff_mean = __class__._get_mean_gaussian_fit(charge,counts) + pedestal = parameters['pedestal'] + pedestal.value = coeff_ped[1] + pedestal.min = coeff_ped[1] - coeff_ped[2] + pedestal.max = coeff_ped[1] + coeff_ped[2] + log.debug(f"pedestal updated : {pedestal}") + pedestalWidth = parameters["pedestalWidth"] + pedestalWidth.value = pedestal.max - pedestal.value + pedestalWidth.max = 3 * pedestalWidth.value + log.debug(f"pedestalWidth updated : {pedestalWidth.value}") + try : + if (coeff_mean[1] - pedestal.value < 0) or ((coeff_mean[1] - coeff_mean[2]) - pedestal.max < 0) : raise Exception("mean gaussian fit not good") + mean = parameters['mean'] + mean.value = coeff_mean[1] - pedestal.value + mean.min = (coeff_mean[1] - coeff_mean[2]) - pedestal.max + mean.max = (coeff_mean[1] + coeff_mean[2]) - pedestal.min + log.debug(f"mean updated : {mean}") + except Exception as e : + log.warning(e,exc_info=True) + log.warning("mean parameters limits and starting value not changed") + return parameters + + @staticmethod + def _get_mean_gaussian_fit(charge, counts ,extension = ""): + #charge = charge_in.data[~histo_in.mask] + #histo = histo_in.data[~histo_in.mask] + windows_lenght = __class__._Windows_lenght + order = __class__._Order + histo_smoothed = savgol_filter(counts, windows_lenght, order) + peaks = find_peaks(histo_smoothed,10) + peak_max = np.argmax(histo_smoothed[peaks[0]]) + peak_pos,peak_value = charge[peaks[0][peak_max]], counts[peaks[0][peak_max]] + coeff, _ = curve_fit(weight_gaussian, charge[:peaks[0][peak_max]], histo_smoothed[:peaks[0][peak_max]],p0 = [peak_value,peak_pos,1]) + if log.getEffectiveLevel() == logging.DEBUG : + log.debug('plotting figures with prefit parameters computation') + fig,ax = plt.subplots(1,1,figsize = (5,5)) + ax.errorbar(charge,counts,np.sqrt(counts),zorder=0,fmt=".",label = "data") + ax.plot(charge,histo_smoothed,label = f'smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})') + ax.plot(charge,weight_gaussian(charge,coeff[0],coeff[1],coeff[2]),label = 'gaussian fit of the pedestal, left tail only') + ax.set_xlim([peak_pos - 500,None]) + ax.vlines(coeff[1],0,peak_value,label = f'pedestal initial value = {coeff[1]:.0f}',color = 'red') + ax.add_patch(Rectangle((coeff[1]-coeff[2], 0), 2 * coeff[2], peak_value,fc=to_rgba('red', 0.5))) + ax.set_xlabel("Charge (ADC)", size=15) + ax.set_ylabel("Events", size=15) + ax.legend(fontsize=7) + os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/",exist_ok=True) + fig.savefig(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/initialization_pedestal_pixel{extension}_{os.getpid()}.pdf") + fig.clf() + plt.close(fig) + del fig,ax + #nosw find SPE peak excluding pedestal data + mask = charge > coeff[1]+3*coeff[2] + peaks_mean = find_peaks(histo_smoothed[mask]) + + peak_max_mean = np.argmax(histo_smoothed[mask][peaks_mean[0]]) + peak_pos_mean,peak_value_mean = charge[mask][peaks_mean[0][peak_max_mean]], histo_smoothed[mask][peaks_mean[0][peak_max_mean]] + mask = (charge > ((coeff[1]+peak_pos_mean)/2)) * (charge < (peak_pos_mean + (peak_pos_mean-coeff[1])/2)) + coeff_mean, _ = curve_fit(weight_gaussian, charge[mask], histo_smoothed[mask],p0 = [peak_value_mean,peak_pos_mean,1]) + if log.getEffectiveLevel() == logging.DEBUG : + log.debug('plotting figures with prefit parameters computation') + fig,ax = plt.subplots(1,1,figsize = (5,5)) + ax.errorbar(charge,counts,np.sqrt(counts),zorder=0,fmt=".",label = "data") + ax.plot(charge,histo_smoothed,label = f'smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})') + ax.plot(charge,weight_gaussian(charge,coeff_mean[0],coeff_mean[1],coeff_mean[2]),label = 'gaussian fit of the SPE') + ax.vlines(coeff_mean[1],0,peak_value,label = f'mean initial value = {coeff_mean[1] - coeff[1]:.0f}',color = "red") + ax.add_patch(Rectangle((coeff_mean[1]-coeff_mean[2], 0), 2 * coeff_mean[2], peak_value_mean,fc=to_rgba('red', 0.5))) + ax.set_xlim([peak_pos - 500,None]) + ax.set_xlabel("Charge (ADC)", size=15) + ax.set_ylabel("Events", size=15) + ax.legend(fontsize=7) + os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/",exist_ok=True) + fig.savefig(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/initialization_mean_pixel{extension}_{os.getpid()}.pdf") + fig.clf() + plt.close(fig) + del fig,ax + + return coeff, coeff_mean + + def _update_table_from_parameters(self) : + for param in self._parameters.parameters : + if not(param.name in self._results.colnames) : + self._results.add_column(Column(data = np.empty((self.npixels),dtype = np.float64),name = param.name,unit = param.unit)) + self._results.add_column(Column(data = np.empty((self.npixels),dtype = np.float64),name = f"{param.name}_error",unit = param.unit)) + + + + +""" + def make_table_from_output_multi(self,list_dict : list) : + self._output_table = QTable.from_pandas(pd.DataFrame.from_dict(list_dict)) + for param in self._parameters.parameters : + self._output_table[param.name] = Column(self._output_table[param.name].value, param.name, unit=param.unit) + if 'gain' in self._output_table.colnames : + if isinstance(self._output_table['gain'],MaskedColumn) : + self._output_table['gain'] = self._output_table['gain']._data + self._output_table['gain_error'] = self._output_table['gain_error']._data + self._output_table.meta['npixel'] = self.npixels + self._output_table.meta['comments'] = f'Produced with NectarGain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' +""" + + +class FlatFieldSingleSPEMaker(FlatFieldSPEMaker) : + __parameters_file = 'parameters_signal.yaml' + __fit_array = None + __reduced_name = "FlatFieldSingleSPE" + +#constructors + def __init__(self,charge,counts,*args,**kwargs) : + super().__init__(*args,**kwargs) + self.__charge = charge + self.__counts = counts + self.__mask_fitted_pixel = np.zeros((self.__charge.shape[0]),dtype = bool) + + + + self.__pedestal = Parameter(name = "pedestal", + value = (np.min(self.__charge) + np.sum(self.__charge * self.__counts)/(np.sum(self.__counts)))/2, + min = np.min(self.__charge), + max = np.sum(self.__charge*self.__counts)/np.sum(self.__counts), + unit = u.dimensionless_unscaled) + + + self._parameters.append(self.__pedestal) + + self.read_param_from_yaml(kwargs.get('parameters_file',self.__parameters_file)) + + self._update_table_from_parameters() + + self._results.add_column(Column(np.zeros((self.npixels),dtype = np.float64),"likelihood",unit = u.dimensionless_unscaled)) + self._results.add_column(Column(np.zeros((self.npixels),dtype = np.float64),"pvalue",unit = u.dimensionless_unscaled)) + + @classmethod + def create_from_chargeContainer(cls, signal : ChargeContainer,**kwargs) : + histo = signal.histo_hg(autoscale = True) + return cls(charge = histo[1],counts = histo[0],pixels_id = signal.pixels_id) + +#getters and setters + @property + def charge(self) : return copy.deepcopy(self.__charge) + @property + def _charge(self) : return self.__charge + + @property + def counts(self) : return copy.deepcopy(self.__counts) + @property + def _counts(self) : return self.__counts + +#I/O method + def save(self,path,**kwargs) : + path = Path(path) + os.makedirs(path,exist_ok = True) + log.info(f'data saved in {path}') + self._results.write(f"{path}/results_{self.__reduced_name}.ecsv", format='ascii.ecsv',overwrite = kwargs.get("overwrite",False)) + +#methods + def _fill_results_table_from_dict(self,dico,pixels_id) : + chi2_sig = signature(__class__.cost(self._charge,self._counts)) + for i in range(len(pixels_id)) : + values = dico[i].get(f"values_{i}",None) + errors = dico[i].get(f"errors_{i}",None) + if not((values is None) or (errors is None)) : + index = np.argmax(self._results["pixels_id"] == pixels_id[i]) + if len(values) != len(chi2_sig.parameters) : + e = Exception("the size out the minuit output parameters values array does not fit the signature of the minimized cost function") + log.error(e,exc_info=True) + raise e + for j,key in enumerate(chi2_sig.parameters) : + self._results[key][index] = values[j] + self._results[f"{key}_error"][index] = errors[j] + if key == 'mean' : + self._gain[index] = values[j] + self._results[f"gain_error"][index] = [errors[j],errors[j]] + self._results[f"gain"][index] = values[j] + self._results['is_valid'][index] = True + self._results["likelihood"][index] = __class__.__fit_array[i].fcn(__class__.__fit_array[i].values) + ndof = self._counts.data[index][~self._counts.mask[index]].shape[0] - __class__.__fit_array[i].nfit + self._results["pvalue"][index] = Statistics.chi2_pvalue(ndof,__class__.__fit_array[i].fcn(__class__.__fit_array[i].values)) + + @staticmethod + def _NG_Likelihood_Chi2(pp,res,mu2,n,muped,sigped,lum,charge,counts,**kwargs) : + pdf = MPE2(charge,pp,res,mu2,n,muped,sigped,lum,**kwargs) + #log.debug(f"pdf : {np.sum(pdf)}") + Ntot = np.sum(counts) + #log.debug(f'Ntot : {Ntot}') + mask = counts > 0 + Lik = np.sum(((pdf*Ntot-counts)[mask])**2/counts[mask]) #2 times faster + return Lik + + @staticmethod + def cost(charge,counts) : + def Chi2(pedestal,pp,luminosity,resolution,mean,n,pedestalWidth) : + #assert not(np.isnan(pp) or np.isnan(resolution) or np.isnan(mean) or np.isnan(n) or np.isnan(pedestal) or np.isnan(pedestalWidth) or np.isnan(luminosity)) + for i in range(1000): + if (gammainc(i+1,luminosity) < 1e-5): + ntotalPE = i + break + kwargs = {"ntotalPE" : ntotalPE} + return __class__._NG_Likelihood_Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity,charge,counts,**kwargs) + return Chi2 + + def _make_fit_array_from_parameters(self, pixels_id = None, **kwargs) : + if pixels_id is None : + npix = self.npixels + pixels_id = self.pixels_id + else : + npix = len(pixels_id) + + fit_array = np.empty((npix),dtype = np.object_) + + for _id in pixels_id : + i = np.where(self.pixels_id == _id)[0][0] + parameters = __class__._update_parameters(self.parameters,self._charge[i].data[~self._charge[i].mask],self._counts[i].data[~self._charge[i].mask]) + minuitParameters = UtilsMinuit.make_minuit_par_kwargs(parameters) + minuit_kwargs = {parname : minuitParameters['values'][parname] for parname in minuitParameters['values']} + log.info(f'creation of fit instance for pixel : {_id}') + fit_array[i] = Minuit(__class__.cost(self._charge[i].data[~self._charge[i].mask],self._counts[i].data[~self._charge[i].mask]),**minuit_kwargs) + log.debug('fit created') + fit_array[i].errordef = Minuit.LIKELIHOOD + fit_array[i].strategy = 0 + fit_array[i].tol = 1e40 + fit_array[i].print_level = 1 + fit_array[i].throw_nan = True + UtilsMinuit.set_minuit_parameters_limits_and_errors(fit_array[i],minuitParameters) + log.debug(fit_array[i].values) + log.debug(fit_array[i].limits) + log.debug(fit_array[i].fixed) + return fit_array + + @staticmethod + def run_fit(i) : + log.info("Starting") + __class__.__fit_array[i].migrad() + __class__.__fit_array[i].hesse() + _values = np.array([params.value for params in __class__.__fit_array[i].params]) + _errors = np.array([params.error for params in __class__.__fit_array[i].params]) + log.info("Finished") + return {f"values_{i}" : _values, f"errors_{i}" : _errors} + + def make(self,pixels_id = None, display = True,**kwargs) : + log.info("running maker") + log.info('checking asked pixels id') + if pixels_id is None : + npix = self.npixels + else : + log.debug('checking that asked pixels id are in data') + mask = np.array([_id in self.pixels_id for _id in pixels_id]) + if False in mask : + log.debug(f"The following pixels are not in data : {pixels_id[~mask]}") + pixels_id = pixels_id[mask] + npix = len(pixels_id) + + if npix == 0 : + log.warning('The asked pixels id are all out of the data') + return None + else : + log.info("creation of the fits instance array") + __class__.__fit_array = self._make_fit_array_from_parameters( + pixels_id = pixels_id + ) + + log.info("running fits") + t = time.time() + with Pool(8) as pool: + result = pool.starmap_async(__class__.run_fit, + [(i,) for i in range(npix)]) + result.wait() + try : + res = result.get() + except Exception as e : + log.error(e,exc_info=True) + raise e + log.debug(res) + log.info(f'time for multiproc with starmap_async execution is {time.time() - t:.2e} sec') + + log.info("filling result table from fits results") + self._fill_results_table_from_dict(res,pixels_id) + + output = copy.copy(__class__.__fit_array) + __class__.__fit_array = None + + if display : + log.info("plotting") + self.display(pixels_id,**kwargs) + + return output + + + def plot_single(pixel_id,charge,counts,pp,resolution,gain,gain_error,n,pedestal,pedestalWidth,luminosity,likelihood) : + fig,ax = plt.subplots(1,1,figsize=(8, 8)) + ax.errorbar(charge,counts,np.sqrt(counts),zorder=0,fmt=".",label = "data") + ax.plot(charge, + np.trapz(counts,charge)*MPE2( + charge, + pp, + resolution, + gain, + n, + pedestal, + pedestalWidth, + luminosity, + ), + zorder=1, + linewidth=2, + label = f"SPE model fit \n gain : {gain - gain_error:.2f} < {gain:.2f} < {gain + gain_error:.2f} ADC/pe,\n likelihood : {likelihood:.2f}") + ax.set_xlabel("Charge (ADC)", size=15) + ax.set_ylabel("Events", size=15) + ax.set_title(f"SPE fit pixel id : {pixel_id}") + ax.set_xlim([pedestal - 6 * pedestalWidth, None]) + ax.legend(fontsize=18) + return fig,ax + + + def display(self,pixels_id,**kwargs) : + figpath = kwargs.get('figpath',f"/tmp/NectarGain_pid{os.getpid()}") + for _id in pixels_id : + index = np.argmax(self._results['pixels_id'] == _id) + fig,ax = __class__.plot_single( + _id, + self._charge[index], + self._counts[index], + self._results['pp'][index].value, + self._results['resolution'][index].value, + self._results['gain'][index].value, + self._results['gain_error'][index].value.mean(), + self._results['n'][index].value, + self._results['pedestal'][index].value, + self._results['pedestalWidth'][index].value, + self._results['luminosity'][index].value, + self._results['likelihood'][index], + ) + + os.makedirs(figpath,exist_ok = True) + fig.savefig(f"{figpath}/fit_SPE_pixel{_id}.pdf") + fig.clf() + plt.close(fig) + del fig,ax + + + + \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/__init__.py b/src/nectarchain/calibration/makers/gain/__init__.py index 5de277b6..0897bc0b 100644 --- a/src/nectarchain/calibration/makers/gain/__init__.py +++ b/src/nectarchain/calibration/makers/gain/__init__.py @@ -1,3 +1,3 @@ from .FlatFieldSPEMakers import * -from .WhiteTargetSPEMakers import * -from .PhotoStatisticMakers import * \ No newline at end of file +#from .WhiteTargetSPEMakers import * +#from .PhotoStatisticMakers import * \ No newline at end of file From 408281ac414bd605436f6c06dcb975ab54868482 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 7 Sep 2023 23:14:29 +0200 Subject: [PATCH 18/62] photostatistic maker (not re-implemented yet) --- .../makers/gain/PhotoStatisticMakers.py | 25 +------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py b/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py index 826c6fa6..d0d4edeb 100644 --- a/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py +++ b/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py @@ -24,7 +24,7 @@ __all__ = ["PhotoStatGainFFandPed"] -class PhotoStatGain(ABC): +class PhotoStatisticMaker(ABC): def _readFF(self,FFRun,maxevents: int = None,**kwargs) : log.info('reading FF data') @@ -258,27 +258,4 @@ def __init__(self, FFRun, PedRun, SPEresults : str, maxevents : int = None, **kw self.create_output_table() - -class PhotoStatGainFF(PhotoStatGain): - def __init__(self, FFRun, PedRun, SPEresults : str, maxevents : int = None, **kwargs) : - e = NotImplementedError("PhotoStatGainFF is not yet implemented") - log.error(e, exc_info = True) - raise e - self._readFF(FFRun,maxevents,**kwargs) - - self._readSPE(SPEresults) - - """ - if self.FFcharge.pixels_id != self._SPE_pixels_id : - e = DifferentPixelsID("Ped run, FF run and SPE run need to have same pixels id") - log.error(e,exc_info = True) - raise e - else : - self._pixels_id = self.FFcharge.pixels_id - """ - - self._reshape_all() - - self.create_output_table() - class PhotoStatisticMaker(GainMaker) : From 2049a7c91e982902e2081d5b398f21286ea3a7fb Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Sat, 9 Sep 2023 01:50:02 +0200 Subject: [PATCH 19/62] implementation of the FFSPE fit for single run with n and p' fixed --- .../makers/gain/FlatFieldSPEMakers.py | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py index 3600edc5..1aba276d 100644 --- a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py @@ -44,7 +44,7 @@ -__all__ = ["FlatFieldSingleSPEMaker"] +__all__ = ["FlatFieldSingleSPEMaker","FlatFieldSingleStdSPEMaker"] @@ -176,23 +176,9 @@ def _update_table_from_parameters(self) : self._results.add_column(Column(data = np.empty((self.npixels),dtype = np.float64),name = f"{param.name}_error",unit = param.unit)) - - -""" - def make_table_from_output_multi(self,list_dict : list) : - self._output_table = QTable.from_pandas(pd.DataFrame.from_dict(list_dict)) - for param in self._parameters.parameters : - self._output_table[param.name] = Column(self._output_table[param.name].value, param.name, unit=param.unit) - if 'gain' in self._output_table.colnames : - if isinstance(self._output_table['gain'],MaskedColumn) : - self._output_table['gain'] = self._output_table['gain']._data - self._output_table['gain_error'] = self._output_table['gain_error']._data - self._output_table.meta['npixel'] = self.npixels - self._output_table.meta['comments'] = f'Produced with NectarGain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' -""" - - class FlatFieldSingleSPEMaker(FlatFieldSPEMaker) : + """class to perform fit of the SPE signal with all free parameters""" + __parameters_file = 'parameters_signal.yaml' __fit_array = None __reduced_name = "FlatFieldSingleSPE" @@ -429,5 +415,21 @@ def display(self,pixels_id,**kwargs) : del fig,ax +class FlatFieldSingleStdSPEMaker(FlatFieldSingleSPEMaker): + """class to perform fit of the SPE signal with n and pp fixed""" + __parameters_file = 'parameters_signalStd.yaml' + + def __init__(self,charge,counts,*args,**kwargs) : + super().__init__(charge,counts,*args,**kwargs) + self.__fix_parameters() + + def __fix_parameters(self) : + """this method should be used to fix n and pp + """ + log.info("updating parameters by fixing pp and n") + pp = self._parameters["pp"] + pp.frozen = True + n = self._parameters["n"] + n.frozen = True \ No newline at end of file From 32b2ba74ea55885bdf2a9cc26ecca2a7d49667b2 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Sat, 9 Sep 2023 03:16:16 +0200 Subject: [PATCH 20/62] implementation of FF SPE gain maker at nominal voltage + update of user script --- .../makers/gain/FlatFieldSPEMakers.py | 137 ++++++++++++++---- .../ggrolleron/gain_SPEfit_computation.py | 11 +- 2 files changed, 117 insertions(+), 31 deletions(-) diff --git a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py index 1aba276d..d7d49f73 100644 --- a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py @@ -6,7 +6,7 @@ import numpy as np import astropy.units as u -from astropy.table import Column +from astropy.table import Column,QTable import copy import os @@ -42,10 +42,7 @@ from .utils import UtilsMinuit,weight_gaussian,Statistics,MPE2 - - -__all__ = ["FlatFieldSingleSPEMaker","FlatFieldSingleStdSPEMaker"] - +__all__ = ["FlatFieldSingleHHVSPEMaker","FlatFieldSingleHHVStdSPEMaker"] class FlatFieldSPEMaker(GainMaker) : @@ -90,7 +87,7 @@ def read_param_from_yaml(self,parameters_file,only_update = False) : self._parameters.append(eval(f"self.__{name}")) @staticmethod - def _update_parameters(parameters,charge,counts) : + def _update_parameters(parameters,charge,counts,**kwargs) : coeff_ped,coeff_mean = __class__._get_mean_gaussian_fit(charge,counts) pedestal = parameters['pedestal'] pedestal.value = coeff_ped[1] @@ -176,12 +173,15 @@ def _update_table_from_parameters(self) : self._results.add_column(Column(data = np.empty((self.npixels),dtype = np.float64),name = f"{param.name}_error",unit = param.unit)) -class FlatFieldSingleSPEMaker(FlatFieldSPEMaker) : + +class FlatFieldSingleHHVSPEMaker(FlatFieldSPEMaker) : """class to perform fit of the SPE signal with all free parameters""" __parameters_file = 'parameters_signal.yaml' __fit_array = None __reduced_name = "FlatFieldSingleSPE" + __nproc_default = 8 + __chunksize_default = 1 #constructors def __init__(self,charge,counts,*args,**kwargs) : @@ -288,7 +288,7 @@ def _make_fit_array_from_parameters(self, pixels_id = None, **kwargs) : for _id in pixels_id : i = np.where(self.pixels_id == _id)[0][0] - parameters = __class__._update_parameters(self.parameters,self._charge[i].data[~self._charge[i].mask],self._counts[i].data[~self._charge[i].mask]) + parameters = __class__._update_parameters(self.parameters,self._charge[i].data[~self._charge[i].mask],self._counts[i].data[~self._charge[i].mask],pixel_id=_id,**kwargs) minuitParameters = UtilsMinuit.make_minuit_par_kwargs(parameters) minuit_kwargs = {parname : minuitParameters['values'][parname] for parname in minuitParameters['values']} log.info(f'creation of fit instance for pixel : {_id}') @@ -315,7 +315,11 @@ def run_fit(i) : log.info("Finished") return {f"values_{i}" : _values, f"errors_{i}" : _errors} - def make(self,pixels_id = None, display = True,**kwargs) : + def make(self, + pixels_id = None, + multiproc = True, + display = True, + **kwargs) : log.info("running maker") log.info('checking asked pixels id') if pixels_id is None : @@ -338,18 +342,30 @@ def make(self,pixels_id = None, display = True,**kwargs) : ) log.info("running fits") - t = time.time() - with Pool(8) as pool: - result = pool.starmap_async(__class__.run_fit, - [(i,) for i in range(npix)]) - result.wait() - try : - res = result.get() - except Exception as e : - log.error(e,exc_info=True) - raise e - log.debug(res) - log.info(f'time for multiproc with starmap_async execution is {time.time() - t:.2e} sec') + if multiproc : + nproc = kwargs.get("nproc",__class__.__nproc_default) + chunksize = kwargs.get("chunksize",max(__class__.__chunksize_default,npix//(nproc*10))) + log.info(f"pooling with nproc {nproc}, chunksize {chunksize}") + + t = time.time() + with Pool(nproc) as pool: + result = pool.starmap_async(__class__.run_fit, + [(i,) for i in range(npix)], + chunksize=chunksize) + result.wait() + try : + res = result.get() + except Exception as e : + log.error(e,exc_info=True) + raise e + log.debug(res) + log.info(f'time for multiproc with starmap_async execution is {time.time() - t:.2e} sec') + else : + log.info("running in mono-cpu") + t = time.time() + res = [__class__.run_fit(i) for i in range(npix)] + log.debug(res) + log.info(f'time for singleproc execution is {time.time() - t:.2e} sec') log.info("filling result table from fits results") self._fill_results_table_from_dict(res,pixels_id) @@ -388,9 +404,9 @@ def plot_single(pixel_id,charge,counts,pp,resolution,gain,gain_error,n,pedestal, ax.legend(fontsize=18) return fig,ax - def display(self,pixels_id,**kwargs) : figpath = kwargs.get('figpath',f"/tmp/NectarGain_pid{os.getpid()}") + os.makedirs(figpath,exist_ok = True) for _id in pixels_id : index = np.argmax(self._results['pixels_id'] == _id) fig,ax = __class__.plot_single( @@ -407,22 +423,23 @@ def display(self,pixels_id,**kwargs) : self._results['luminosity'][index].value, self._results['likelihood'][index], ) - - os.makedirs(figpath,exist_ok = True) fig.savefig(f"{figpath}/fit_SPE_pixel{_id}.pdf") fig.clf() plt.close(fig) del fig,ax -class FlatFieldSingleStdSPEMaker(FlatFieldSingleSPEMaker): + +class FlatFieldSingleHHVStdSPEMaker(FlatFieldSingleHHVSPEMaker): """class to perform fit of the SPE signal with n and pp fixed""" __parameters_file = 'parameters_signalStd.yaml' +#constructors def __init__(self,charge,counts,*args,**kwargs) : super().__init__(charge,counts,*args,**kwargs) self.__fix_parameters() +#methods def __fix_parameters(self) : """this method should be used to fix n and pp """ @@ -431,5 +448,73 @@ def __fix_parameters(self) : pp.frozen = True n = self._parameters["n"] n.frozen = True + + + +class FlatFieldSingleNominalSPEMaker(FlatFieldSingleHHVSPEMaker): + """class to perform fit of the SPE signal at nominal voltage from fitted data obtained with 1400V run + Thus, n, pp and res are fixed""" + __parameters_file = 'parameters_signal_fromHHVFit.yaml' + +#constructors + def __init__(self, charge, counts, nectarGainSPEresult : str, same_luminosity : bool = True, *args, **kwargs): + super().__init__(charge, counts, *args, **kwargs) + self.__fix_parameters(same_luminosity) + self.__same_luminosity = same_luminosity + self.__nectarGainSPEresult = self._read_SPEresult(nectarGainSPEresult) + +#getters and setters + @property + def nectarGainSPEresult(self) : return copy.deepcopy(self.__nectarGainSPEresult) + + @property + def same_luminosity(self) : return copy.deepcopy(self.__same_luminosity) + +#methods + def _read_SPEresult(self,nectarGainSPEresult : str) : + table = QTable.read(nectarGainSPEresult,format = "ascii.ecsv") + argsort = [] + mask = [] + for _id in self._pixels_id : + if _id in table['pixels_id'] : + argsort.append(np.where(_id==table['pixels_id'])[0][0]) + mask.append(True) + else : + mask.append(True) + self._pixels_id = self._pixels_id[np.array(mask)] + return table[np.array(argsort)] + + def __fix_parameters(self, same_luminosity : bool) : + """this method should be used to fix n, pp and res + """ + log.info("updating parameters by fixing pp, n and res") + pp = self._parameters["pp"] + pp.frozen = True + n = self._parameters["n"] + n.frozen = True + resolution = self._parameters["resolution"] + resolution.frozen = True + if same_luminosity : + luminosity = self._parameters["luminosity"] + luminosity.frozen = True + + def _make_fit_array_from_parameters(self, pixels_id = None, **kwargs) : + return super()._make_fit_array_from_parameters(self, pixels_id = pixels_id, nectarGainSPEresult = self.__nectarGainSPEresult, **kwargs) + + @staticmethod + def _update_parameters(parameters,charge,counts,pixel_id,nectarGainSPEresult,**kwargs) : + param = super()._update_parameters(parameters,charge,counts,**kwargs) + luminosity = param["luminosity"] + resolution = param["resolution"] + pp = param["pp"] + n = param["n"] + + index = np.where(pixel_id == nectarGainSPEresult["pixels_id"])[0][0] + + resolution.value = nectarGainSPEresult[index]["resolution"].value + pp.value = nectarGainSPEresult[index]["pp"].value + n.value = nectarGainSPEresult[index]["n"].value - \ No newline at end of file + if luminosity.frozen : + luminosity.value = nectarGainSPEresult[index]["luminosity"].value + return param \ No newline at end of file diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py index 2fc0177e..2129ce0a 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py @@ -14,7 +14,7 @@ #import seaborn as sns from nectarchain.calibration.container import ChargeContainer -from nectarchain.calibration.NectarGain import NectarGainSPESingleSignalStd,NectarGainSPESingleSignal +from nectarchain.calibration.makers.gain.FlatFieldSPEMakers import FlatFieldSingleHHVSPEMaker,FlatFieldSingleHHVStdSPEMaker parser = argparse.ArgumentParser( prog = 'gain_SPEfit_computation.py', @@ -96,15 +96,16 @@ def main(args) : charge_run_1400V = ChargeContainer.from_file(f"{os.environ.get('NECTARCAMDATA')}/charges/{args.chargeExtractorPath}/",args.run_number) if args.free_pp_n : - gain_Std = NectarGainSPESingleSignal(signal = charge_run_1400V) + gain_Std = FlatFieldSingleHHVSPEMaker(signal = charge_run_1400V) else : - gain_Std = NectarGainSPESingleSignalStd(signal = charge_run_1400V) + gain_Std = FlatFieldSingleHHVStdSPEMaker(signal = charge_run_1400V) + t = time.time() - gain_Std.run(pixel = args.pixels, multiproc = args.multiproc, nproc = args.nproc, chunksize = args.chunksize, figpath = figpath+f"/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}") + gain_Std.make(pixels_id = args.pixels, multiproc = args.multiproc, nproc = args.nproc, chunksize = args.chunksize, figpath = figpath+f"/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}") log.info(f"fit time = {time.time() - t } sec") gain_Std.save(f"{os.environ.get('NECTARCAMDATA')}/../SPEfit/data/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}/",overwrite = args.overwrite) - conv_rate = len(gain_Std._output_table[gain_Std._output_table['is_valid']])/gain_Std.npixels if args.pixels is None else len(gain_Std._output_table[gain_Std._output_table['is_valid']])/len(args.pixels) + conv_rate = len(gain_Std._results[gain_Std._results['is_valid']])/gain_Std.npixels if args.pixels is None else len(gain_Std._results[gain_Std._results['is_valid']])/len(args.pixels) log.info(f"convergence rate : {conv_rate}") if __name__ == "__main__": From 9ee64ca4e4962b45ad89e36ad0f9a9cb720986ff Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Sat, 9 Sep 2023 04:08:36 +0200 Subject: [PATCH 21/62] bugfix user script updated : FF SPE at VVH voltage (ie for single run) --- .../makers/gain/FlatFieldSPEMakers.py | 17 +++++++++------ .../ggrolleron/gain_SPEfit_computation.py | 21 +++++++++++++++---- .../ggrolleron/gain_SPEfit_computation.sh | 8 +++---- 3 files changed, 32 insertions(+), 14 deletions(-) diff --git a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py index d7d49f73..fbaf2d7e 100644 --- a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py @@ -32,6 +32,7 @@ from inspect import signature +from numba import njit, prange from .gainMakers import GainMaker @@ -88,7 +89,7 @@ def read_param_from_yaml(self,parameters_file,only_update = False) : @staticmethod def _update_parameters(parameters,charge,counts,**kwargs) : - coeff_ped,coeff_mean = __class__._get_mean_gaussian_fit(charge,counts) + coeff_ped,coeff_mean = __class__._get_mean_gaussian_fit(charge,counts,**kwargs) pedestal = parameters['pedestal'] pedestal.value = coeff_ped[1] pedestal.min = coeff_ped[1] - coeff_ped[2] @@ -111,7 +112,7 @@ def _update_parameters(parameters,charge,counts,**kwargs) : return parameters @staticmethod - def _get_mean_gaussian_fit(charge, counts ,extension = ""): + def _get_mean_gaussian_fit(charge, counts ,extension = "",**kwargs): #charge = charge_in.data[~histo_in.mask] #histo = histo_in.data[~histo_in.mask] windows_lenght = __class__._Windows_lenght @@ -121,7 +122,7 @@ def _get_mean_gaussian_fit(charge, counts ,extension = ""): peak_max = np.argmax(histo_smoothed[peaks[0]]) peak_pos,peak_value = charge[peaks[0][peak_max]], counts[peaks[0][peak_max]] coeff, _ = curve_fit(weight_gaussian, charge[:peaks[0][peak_max]], histo_smoothed[:peaks[0][peak_max]],p0 = [peak_value,peak_pos,1]) - if log.getEffectiveLevel() == logging.DEBUG : + if log.getEffectiveLevel() == logging.DEBUG and kwargs.get("display",False) : log.debug('plotting figures with prefit parameters computation') fig,ax = plt.subplots(1,1,figsize = (5,5)) ax.errorbar(charge,counts,np.sqrt(counts),zorder=0,fmt=".",label = "data") @@ -146,7 +147,7 @@ def _get_mean_gaussian_fit(charge, counts ,extension = ""): peak_pos_mean,peak_value_mean = charge[mask][peaks_mean[0][peak_max_mean]], histo_smoothed[mask][peaks_mean[0][peak_max_mean]] mask = (charge > ((coeff[1]+peak_pos_mean)/2)) * (charge < (peak_pos_mean + (peak_pos_mean-coeff[1])/2)) coeff_mean, _ = curve_fit(weight_gaussian, charge[mask], histo_smoothed[mask],p0 = [peak_value_mean,peak_pos_mean,1]) - if log.getEffectiveLevel() == logging.DEBUG : + if log.getEffectiveLevel() == logging.DEBUG and kwargs.get("display",False) : log.debug('plotting figures with prefit parameters computation') fig,ax = plt.subplots(1,1,figsize = (5,5)) ax.errorbar(charge,counts,np.sqrt(counts),zorder=0,fmt=".",label = "data") @@ -211,7 +212,7 @@ def __init__(self,charge,counts,*args,**kwargs) : @classmethod def create_from_chargeContainer(cls, signal : ChargeContainer,**kwargs) : histo = signal.histo_hg(autoscale = True) - return cls(charge = histo[1],counts = histo[0],pixels_id = signal.pixels_id) + return cls(charge = histo[1],counts = histo[0],pixels_id = signal.pixels_id,**kwargs) #getters and setters @property @@ -277,6 +278,7 @@ def Chi2(pedestal,pp,luminosity,resolution,mean,n,pedestalWidth) : return __class__._NG_Likelihood_Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity,charge,counts,**kwargs) return Chi2 + #@njit(parallel=True,nopython = True) def _make_fit_array_from_parameters(self, pixels_id = None, **kwargs) : if pixels_id is None : npix = self.npixels @@ -287,6 +289,8 @@ def _make_fit_array_from_parameters(self, pixels_id = None, **kwargs) : fit_array = np.empty((npix),dtype = np.object_) for _id in pixels_id : + #for j in prange(len(pixels_id)) : + # _id = pixels_id[j] i = np.where(self.pixels_id == _id)[0][0] parameters = __class__._update_parameters(self.parameters,self._charge[i].data[~self._charge[i].mask],self._counts[i].data[~self._charge[i].mask],pixel_id=_id,**kwargs) minuitParameters = UtilsMinuit.make_minuit_par_kwargs(parameters) @@ -326,7 +330,8 @@ def make(self, npix = self.npixels else : log.debug('checking that asked pixels id are in data') - mask = np.array([_id in self.pixels_id for _id in pixels_id]) + pixels_id = np.asarray(pixels_id) + mask = np.array([_id in self.pixels_id for _id in pixels_id],dtype = bool) if False in mask : log.debug(f"The following pixels are not in data : {pixels_id[~mask]}") pixels_id = pixels_id[mask] diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py index 2129ce0a..56724a54 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py @@ -38,7 +38,13 @@ help='tag for voltage specifcication (1400V or nominal), used to setup the output path. See help for more details' ) -#output figures path extension +#output figures and path extension +parser.add_argument('--display', + action='store_true', + default=False, + help='whether to save plot or not' + ) + parser.add_argument('--output_fig_tag', type = str, default='', @@ -96,13 +102,20 @@ def main(args) : charge_run_1400V = ChargeContainer.from_file(f"{os.environ.get('NECTARCAMDATA')}/charges/{args.chargeExtractorPath}/",args.run_number) if args.free_pp_n : - gain_Std = FlatFieldSingleHHVSPEMaker(signal = charge_run_1400V) + gain_Std = FlatFieldSingleHHVSPEMaker.create_from_chargeContainer(signal = charge_run_1400V) else : - gain_Std = FlatFieldSingleHHVStdSPEMaker(signal = charge_run_1400V) + gain_Std = FlatFieldSingleHHVStdSPEMaker.create_from_chargeContainer(signal = charge_run_1400V) t = time.time() - gain_Std.make(pixels_id = args.pixels, multiproc = args.multiproc, nproc = args.nproc, chunksize = args.chunksize, figpath = figpath+f"/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}") + gain_Std.make(pixels_id = args.pixels, + multiproc = args.multiproc, + display = args.display, + nproc = args.nproc, + chunksize = args.chunksize, + figpath = figpath+f"/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}" + ) + log.info(f"fit time = {time.time() - t } sec") gain_Std.save(f"{os.environ.get('NECTARCAMDATA')}/../SPEfit/data/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}/",overwrite = args.overwrite) conv_rate = len(gain_Std._results[gain_Std._results['is_valid']])/gain_Std.npixels if args.pixels is None else len(gain_Std._results[gain_Std._results['is_valid']])/len(args.pixels) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh index bf6e8446..932d6721 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh @@ -1,13 +1,13 @@ #HOW TO : #to perform SPE fit of a HHV run -python gain_SPEfit_computation.py -r 2634 --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 +python gain_SPEfit_computation.py -r 2634 --display --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 #to perform SPE fit of a HHV run letting n and pp parameters free -python gain_SPEfit_computation.py -r 2634 --voltage_tag "1400V" --free_pp_n --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 +python gain_SPEfit_computation.py -r 2634 --display --voltage_tag "1400V" --free_pp_n --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 #to perform SPE fit of a run at nominal voltage -python gain_SPEfit_computation.py -r 3936 --voltage_tag "nominal" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 +python gain_SPEfit_computation.py -r 3936 --display --voltage_tag "nominal" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 #to perform SPE fit of a HHV run -python gain_SPEfit_computation.py -r 3942 --output_fig_tag "testMyRes" --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 \ No newline at end of file +python gain_SPEfit_computation.py -r 3942 --display --output_fig_tag "testMyRes" --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 --verbosity info --overwrite \ No newline at end of file From 17e800cccb631c41dc8dd438670bacb37c91f93c Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Sat, 9 Sep 2023 04:24:16 +0200 Subject: [PATCH 22/62] update for nominal voltage data FF SPE fit from HHV pre-computed one --- .../gain_SPEfit_combined_computation.py | 19 +++++++++----- .../gain_SPEfit_combined_computation.sh | 8 +++--- .../ggrolleron/gain_SPEfit_computation.py | 25 +++++-------------- .../ggrolleron/gain_SPEfit_computation.sh | 8 +++--- 4 files changed, 27 insertions(+), 33 deletions(-) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py index c349e0b2..99cf1679 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py @@ -13,7 +13,7 @@ #import seaborn as sns from nectarchain.calibration.container import ChargeContainer -from nectarchain.calibration.NectarGain import NectarGainSPESingleSignalfromHHVFit +from nectarchain.calibration.makers.gain.FlatFieldSPEMakers import FlatFieldSingleNominalSPEMaker parser = argparse.ArgumentParser( prog = 'gain_SPEfit_combined_computation.py', @@ -97,7 +97,7 @@ def main(args) : - figpath = f"{os.environ.get('NECTARCHAIN_FIGURES')}/" + figpath = f"{os.environ.get('NECTARCHAIN_FIGURES',f'/tmp/nectarchain_log/{os.getpid()}/figure')}/" figpath_ext = "" if args.output_fig_tag == "" else f"-{args.output_fig_tag}" @@ -108,15 +108,22 @@ def main(args) : if args.combined : raise NotImplementedError("combined fit not implemented yet") else : - gain_Std = NectarGainSPESingleSignalfromHHVFit(signal = charge_run, + gain_Std = FlatFieldSingleNominalSPEMaker.create_from_chargeContainer(signal = charge_run, nectarGainSPEresult=args.VVH_fitted_results, same_luminosity=args.same_luminosity ) t = time.time() - gain_Std.run(pixel = args.pixels, multiproc = args.multiproc, nproc = args.nproc, chunksize = args.chunksize, figpath = figpath+f"/{multipath}nominal-prefitCombinedSPE{args.SPE_fit_results_tag}-SPEStd-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}") + gain_Std.make(pixels_id = args.pixels, + multiproc = args.multiproc, + display = args.display, + nproc = args.nproc, + chunksize = args.chunksize, + figpath = figpath+f"/{multipath}nominal-prefitCombinedSPE{args.SPE_fit_results_tag}-SPEStd-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}" + ) + log.info(f"fit time = {time.time() - t } sec") gain_Std.save(f"{os.environ.get('NECTARCAMDATA')}/../SPEfit/data/{multipath}nominal-prefitCombinedSPE{args.SPE_fit_results_tag}-SPEStd-{args.run_number}-{args.chargeExtractorPath}/",overwrite = args.overwrite) - log.info(f"convergence rate : {len(gain_Std._output_table[gain_Std._output_table['is_valid']])/gain_Std.npixels}") + log.info(f"convergence rate : {len(gain_Std._results[gain_Std._results['is_valid']])/gain_Std.npixels}") if __name__ == "__main__": args = parser.parse_args() @@ -128,7 +135,7 @@ def main(args) : elif args.verbosity == "debug" : logginglevel = logging.DEBUG - os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures") + os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG','/tmp/nectarchain_log')}/{os.getpid()}/figures") logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',force = True, level=logginglevel,filename = f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/{Path(__file__).stem}_{os.getpid()}.log") log = logging.getLogger(__name__) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.sh b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.sh index 9e796f30..609a4138 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.sh +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.sh @@ -1,9 +1,9 @@ #HOW TO : #to perform SPE fit of run 2633 (supposed to be taken at nominal voltage) from SPE fit performed on 2634 (taken at 1400V) -python gain_SPEfit_combined_computation.py -r 2633 --SPE_fit_results_tag 2634 --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 --same_luminosity --VVH_fitted_results "/data/users/ggroller/NECTARCAM/SPEfit/data/MULTI-1400V-SPEStd-2634-LocalPeakWindowSum_4-12/output_table.ecsv" +python gain_SPEfit_combined_computation.py -r 2633 --display --SPE_fit_results_tag 2634 --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 --same_luminosity --VVH_fitted_results "/data/users/ggroller/NECTARCAM/SPEfit/data/MULTI-1400V-SPEStd-2634-LocalPeakWindowSum_4-12/output_table.ecsv" #to perform a joint SPE fit of run 2633 (supposed to be taken at nominal voltage) and run 2634 (1400V). -python gain_SPEfit_combined_computation.py -r 2633 --SPE_fit_results_tag 2634 --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 --same_luminosity --combined +python gain_SPEfit_combined_computation.py -r 2633 --display --SPE_fit_results_tag 2634 --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 --same_luminosity --combined -#to perform SPE fit of run 3936 (supposed to be taken at nominal voltage) from SPE fit performed on 2634 (taken at 1400V) (luminosity can not be supposed the same) -python gain_SPEfit_combined_computation.py -r 3936 --SPE_fit_results_tag 3942 --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 --VVH_fitted_results "/data/users/ggroller/NECTARCAM/SPEfit/data/MULTI-1400V-SPEStd-3942-LocalPeakWindowSum_4-12/output_table.ecsv" +#to perform SPE fit of run 3936 (supposed to be taken at nominal voltage) from SPE fit performed on 3942 (taken at 1400V) (luminosity can not be supposed the same) +python gain_SPEfit_combined_computation.py -r 3936 --display --SPE_fit_results_tag 3942 --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 --VVH_fitted_results "/data/users/ggroller/NECTARCAM/SPEfit/data/MULTI-1400V-SPEStd-3942-LocalPeakWindowSum_4-12/output_table.ecsv" diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py index 56724a54..5988807e 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py @@ -38,13 +38,7 @@ help='tag for voltage specifcication (1400V or nominal), used to setup the output path. See help for more details' ) -#output figures and path extension -parser.add_argument('--display', - action='store_true', - default=False, - help='whether to save plot or not' - ) - +#output figures path extension parser.add_argument('--output_fig_tag', type = str, default='', @@ -93,7 +87,7 @@ def main(args) : - figpath = f"{os.environ.get('NECTARCHAIN_FIGURES')}/" + figpath = f"{os.environ.get('NECTARCHAIN_FIGURES',f'/tmp/nectarchain_log/{os.getpid()}/figure')}/" figpath_ext = "" if args.output_fig_tag == "" else f"-{args.output_fig_tag}" multipath = "MULTI-" if args.multiproc else "" @@ -102,20 +96,13 @@ def main(args) : charge_run_1400V = ChargeContainer.from_file(f"{os.environ.get('NECTARCAMDATA')}/charges/{args.chargeExtractorPath}/",args.run_number) if args.free_pp_n : - gain_Std = FlatFieldSingleHHVSPEMaker.create_from_chargeContainer(signal = charge_run_1400V) + gain_Std = FlatFieldSingleHHVSPEMaker(signal = charge_run_1400V) else : - gain_Std = FlatFieldSingleHHVStdSPEMaker.create_from_chargeContainer(signal = charge_run_1400V) + gain_Std = FlatFieldSingleHHVStdSPEMaker(signal = charge_run_1400V) t = time.time() - gain_Std.make(pixels_id = args.pixels, - multiproc = args.multiproc, - display = args.display, - nproc = args.nproc, - chunksize = args.chunksize, - figpath = figpath+f"/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}" - ) - + gain_Std.make(pixels_id = args.pixels, multiproc = args.multiproc, nproc = args.nproc, chunksize = args.chunksize, figpath = figpath+f"/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}") log.info(f"fit time = {time.time() - t } sec") gain_Std.save(f"{os.environ.get('NECTARCAMDATA')}/../SPEfit/data/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}/",overwrite = args.overwrite) conv_rate = len(gain_Std._results[gain_Std._results['is_valid']])/gain_Std.npixels if args.pixels is None else len(gain_Std._results[gain_Std._results['is_valid']])/len(args.pixels) @@ -131,7 +118,7 @@ def main(args) : elif args.verbosity == "debug" : logginglevel = logging.DEBUG - os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures") + os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG','tmp/nectarchain_log')}/{os.getpid()}/figures") logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',force = True, level=logginglevel,filename = f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/{Path(__file__).stem}_{os.getpid()}.log") log = logging.getLogger(__name__) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh index 932d6721..bf6e8446 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh @@ -1,13 +1,13 @@ #HOW TO : #to perform SPE fit of a HHV run -python gain_SPEfit_computation.py -r 2634 --display --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 +python gain_SPEfit_computation.py -r 2634 --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 #to perform SPE fit of a HHV run letting n and pp parameters free -python gain_SPEfit_computation.py -r 2634 --display --voltage_tag "1400V" --free_pp_n --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 +python gain_SPEfit_computation.py -r 2634 --voltage_tag "1400V" --free_pp_n --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 #to perform SPE fit of a run at nominal voltage -python gain_SPEfit_computation.py -r 3936 --display --voltage_tag "nominal" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 +python gain_SPEfit_computation.py -r 3936 --voltage_tag "nominal" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 #to perform SPE fit of a HHV run -python gain_SPEfit_computation.py -r 3942 --display --output_fig_tag "testMyRes" --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 --verbosity info --overwrite \ No newline at end of file +python gain_SPEfit_computation.py -r 3942 --output_fig_tag "testMyRes" --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 \ No newline at end of file From 0139b7de14c5aa24c4fa341741171d25033f2381 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Sat, 9 Sep 2023 04:43:38 +0200 Subject: [PATCH 23/62] -bugfix update for nominal FF SPE fit --- .../makers/gain/FlatFieldSPEMakers.py | 10 ++++---- .../gain_SPEfit_combined_computation.py | 7 +++++- .../ggrolleron/gain_SPEfit_computation.py | 24 ++++++++++++++----- .../ggrolleron/gain_SPEfit_computation.sh | 8 +++---- 4 files changed, 34 insertions(+), 15 deletions(-) diff --git a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py index fbaf2d7e..413f7497 100644 --- a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py @@ -288,15 +288,15 @@ def _make_fit_array_from_parameters(self, pixels_id = None, **kwargs) : fit_array = np.empty((npix),dtype = np.object_) - for _id in pixels_id : + for i,_id in enumerate(pixels_id) : #for j in prange(len(pixels_id)) : # _id = pixels_id[j] - i = np.where(self.pixels_id == _id)[0][0] - parameters = __class__._update_parameters(self.parameters,self._charge[i].data[~self._charge[i].mask],self._counts[i].data[~self._charge[i].mask],pixel_id=_id,**kwargs) + index = np.where(self.pixels_id == _id)[0][0] + parameters = __class__._update_parameters(self.parameters,self._charge[index].data[~self._charge[index].mask],self._counts[index].data[~self._charge[index].mask],pixel_id=_id,**kwargs) minuitParameters = UtilsMinuit.make_minuit_par_kwargs(parameters) minuit_kwargs = {parname : minuitParameters['values'][parname] for parname in minuitParameters['values']} log.info(f'creation of fit instance for pixel : {_id}') - fit_array[i] = Minuit(__class__.cost(self._charge[i].data[~self._charge[i].mask],self._counts[i].data[~self._charge[i].mask]),**minuit_kwargs) + fit_array[i] = Minuit(__class__.cost(self._charge[index].data[~self._charge[index].mask],self._counts[index].data[~self._charge[index].mask]),**minuit_kwargs) log.debug('fit created') fit_array[i].errordef = Minuit.LIKELIHOOD fit_array[i].strategy = 0 @@ -438,6 +438,7 @@ def display(self,pixels_id,**kwargs) : class FlatFieldSingleHHVStdSPEMaker(FlatFieldSingleHHVSPEMaker): """class to perform fit of the SPE signal with n and pp fixed""" __parameters_file = 'parameters_signalStd.yaml' + __reduced_name = "FlatFieldSingleStdSPE" #constructors def __init__(self,charge,counts,*args,**kwargs) : @@ -460,6 +461,7 @@ class FlatFieldSingleNominalSPEMaker(FlatFieldSingleHHVSPEMaker): """class to perform fit of the SPE signal at nominal voltage from fitted data obtained with 1400V run Thus, n, pp and res are fixed""" __parameters_file = 'parameters_signal_fromHHVFit.yaml' + __reduced_name = "FlatFieldSingleNominalSPE" #constructors def __init__(self, charge, counts, nectarGainSPEresult : str, same_luminosity : bool = True, *args, **kwargs): diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py index 99cf1679..51326243 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py @@ -31,7 +31,12 @@ help='to force overwrite files on disk' ) -#output figures path extension +#output figures and path extension +parser.add_argument('--display', + action='store_true', + default=False, + help='whether to save plot or not' + ) parser.add_argument('--output_fig_tag', type = str, default='', diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py index 5988807e..3eb4f6cb 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py @@ -38,7 +38,12 @@ help='tag for voltage specifcication (1400V or nominal), used to setup the output path. See help for more details' ) -#output figures path extension +#output figures and path extension +parser.add_argument('--display', + action='store_true', + default=False, + help='whether to save plot or not' + ) parser.add_argument('--output_fig_tag', type = str, default='', @@ -87,7 +92,7 @@ def main(args) : - figpath = f"{os.environ.get('NECTARCHAIN_FIGURES',f'/tmp/nectarchain_log/{os.getpid()}/figure')}/" + figpath = f"{os.environ.get('NECTARCHAIN_FIGURES')}/" figpath_ext = "" if args.output_fig_tag == "" else f"-{args.output_fig_tag}" multipath = "MULTI-" if args.multiproc else "" @@ -96,13 +101,20 @@ def main(args) : charge_run_1400V = ChargeContainer.from_file(f"{os.environ.get('NECTARCAMDATA')}/charges/{args.chargeExtractorPath}/",args.run_number) if args.free_pp_n : - gain_Std = FlatFieldSingleHHVSPEMaker(signal = charge_run_1400V) + gain_Std = FlatFieldSingleHHVSPEMaker.create_from_chargeContainer(signal = charge_run_1400V) else : - gain_Std = FlatFieldSingleHHVStdSPEMaker(signal = charge_run_1400V) + gain_Std = FlatFieldSingleHHVStdSPEMaker.create_from_chargeContainer(signal = charge_run_1400V) t = time.time() - gain_Std.make(pixels_id = args.pixels, multiproc = args.multiproc, nproc = args.nproc, chunksize = args.chunksize, figpath = figpath+f"/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}") + gain_Std.make(pixels_id = args.pixels, + multiproc = args.multiproc, + display = args.display, + nproc = args.nproc, + chunksize = args.chunksize, + figpath = figpath+f"/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}" + ) + log.info(f"fit time = {time.time() - t } sec") gain_Std.save(f"{os.environ.get('NECTARCAMDATA')}/../SPEfit/data/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}/",overwrite = args.overwrite) conv_rate = len(gain_Std._results[gain_Std._results['is_valid']])/gain_Std.npixels if args.pixels is None else len(gain_Std._results[gain_Std._results['is_valid']])/len(args.pixels) @@ -118,7 +130,7 @@ def main(args) : elif args.verbosity == "debug" : logginglevel = logging.DEBUG - os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG','tmp/nectarchain_log')}/{os.getpid()}/figures") + os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures") logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',force = True, level=logginglevel,filename = f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/{Path(__file__).stem}_{os.getpid()}.log") log = logging.getLogger(__name__) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh index bf6e8446..932d6721 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.sh @@ -1,13 +1,13 @@ #HOW TO : #to perform SPE fit of a HHV run -python gain_SPEfit_computation.py -r 2634 --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 +python gain_SPEfit_computation.py -r 2634 --display --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 #to perform SPE fit of a HHV run letting n and pp parameters free -python gain_SPEfit_computation.py -r 2634 --voltage_tag "1400V" --free_pp_n --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 +python gain_SPEfit_computation.py -r 2634 --display --voltage_tag "1400V" --free_pp_n --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 #to perform SPE fit of a run at nominal voltage -python gain_SPEfit_computation.py -r 3936 --voltage_tag "nominal" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 +python gain_SPEfit_computation.py -r 3936 --display --voltage_tag "nominal" --chargeExtractorPath LocalPeakWindowSum_4-12 --overwrite --multiproc --nproc 6 --chunksize 50 #to perform SPE fit of a HHV run -python gain_SPEfit_computation.py -r 3942 --output_fig_tag "testMyRes" --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 \ No newline at end of file +python gain_SPEfit_computation.py -r 3942 --display --output_fig_tag "testMyRes" --voltage_tag "1400V" --chargeExtractorPath LocalPeakWindowSum_4-12 --multiproc --nproc 6 --chunksize 2 -p 0 1 2 3 4 5 6 7 8 9 10 11 12 --verbosity info --overwrite \ No newline at end of file From 177e577605c702f3485a4b874f20b04284ece24d Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 11 Sep 2023 23:10:32 +0200 Subject: [PATCH 24/62] put I/O method in mother class --- src/nectarchain/calibration/makers/core.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/nectarchain/calibration/makers/core.py b/src/nectarchain/calibration/makers/core.py index e0ed8a0e..887c3dc7 100644 --- a/src/nectarchain/calibration/makers/core.py +++ b/src/nectarchain/calibration/makers/core.py @@ -3,6 +3,8 @@ logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') log = logging.getLogger(__name__) log.handlers = logging.getLogger('__main__').handlers +import os +from pathlib import Path from abc import ABC, ABCMeta, abstractmethod @@ -17,14 +19,15 @@ class CalibrationMaker(ABC) : """mother class for all the calibration makers that can be defined to compute calibration coeficients from data """ + _reduced_name = "Calibration" #constructors def __new__(cls,*args,**kwargs) : return super(CalibrationMaker,cls).__new__(cls) - def __init__(self,*args,**kwargs) -> None: + def __init__(self,pixels_id,*args,**kwargs) -> None: super().__init__() - self.__pixels_id = kwargs.get("pixels_id", None) + self.__pixels_id = pixels_id self.__results = QTable() self.__results.add_column(Column(self.__pixels_id,"pixels_id",unit = u.dimensionless_unscaled)) self.__results.meta['npixels'] = self.npixels @@ -35,6 +38,13 @@ def __init__(self,*args,**kwargs) -> None: def make(self,*args,**kwargs) : pass +#I/O method + def save(self,path,**kwargs) : + path = Path(path) + os.makedirs(path,exist_ok = True) + log.info(f'data saved in {path}') + self._results.write(f"{path}/results_{self._reduced_name}.ecsv", format='ascii.ecsv',overwrite = kwargs.get("overwrite",False)) + #getters and setters @property def _pixels_id(self) : return self.__pixels_id From 41c6a9269ea16ae46f082adfe3df16a0ca5def9b Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 11 Sep 2023 23:11:16 +0200 Subject: [PATCH 25/62] add low gain in contructor (only high gain before) --- .../calibration/makers/gain/gainMakers.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/nectarchain/calibration/makers/gain/gainMakers.py b/src/nectarchain/calibration/makers/gain/gainMakers.py index abe8d3bf..20264e7a 100644 --- a/src/nectarchain/calibration/makers/gain/gainMakers.py +++ b/src/nectarchain/calibration/makers/gain/gainMakers.py @@ -12,7 +12,7 @@ from ..core import CalibrationMaker -__all__ = ["GainMaker"] +__all__ = ["high_gainMaker"] class GainMaker(CalibrationMaker) : """mother class for of the gain calibration @@ -21,18 +21,27 @@ class GainMaker(CalibrationMaker) : #constructors def __init__(self,*args,**kwargs) -> None: super().__init__(*args,**kwargs) - self.__gain = np.empty((self.npixels),dtype = np.float64) - self._results.add_column(Column(data = self.__gain,name = "gain",unit = u.dimensionless_unscaled)) - self._results.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"gain_error",unit = u.dimensionless_unscaled)) + self.__high_gain = np.empty((self.npixels),dtype = np.float64) + self.__low_gain = np.empty((self.npixels),dtype = np.float64) + self._results.add_column(Column(data = self.__high_gain,name = "high_gain",unit = u.dimensionless_unscaled)) + self._results.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"high_gain_error",unit = u.dimensionless_unscaled)) + self._results.add_column(Column(data = self.__low_gain,name = "low_gain",unit = u.dimensionless_unscaled)) + self._results.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"low_gain_error",unit = u.dimensionless_unscaled)) self._results.add_column(Column(np.zeros((self.npixels),dtype = bool),"is_valid",unit = u.dimensionless_unscaled)) #getters and setters @property - def _gain(self) : return self.__gain - @_gain.setter - def _gain(self,value) : self.__gain = value + def _high_gain(self) : return self.__high_gain + @_high_gain.setter + def _high_gain(self,value) : self.__high_gain = value @property - def gain(self) : return copy(self.__gain) + def high_gain(self) : return copy(self.__high_gain) + @property + def _low_gain(self) : return self.__low_gain + @_low_gain.setter + def _low_gain(self,value) : self.__low_gain = value + @property + def low_gain(self) : return copy(self.__low_gain) \ No newline at end of file From 10793f0460ecea680aac2bdc9f47daa138fad957 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 11 Sep 2023 23:23:10 +0200 Subject: [PATCH 26/62] typo bugfiox --- src/nectarchain/calibration/makers/gain/gainMakers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nectarchain/calibration/makers/gain/gainMakers.py b/src/nectarchain/calibration/makers/gain/gainMakers.py index 20264e7a..7cd9fbea 100644 --- a/src/nectarchain/calibration/makers/gain/gainMakers.py +++ b/src/nectarchain/calibration/makers/gain/gainMakers.py @@ -12,7 +12,7 @@ from ..core import CalibrationMaker -__all__ = ["high_gainMaker"] +__all__ = ["GainMaker"] class GainMaker(CalibrationMaker) : """mother class for of the gain calibration From 090844c86cfd26866f2ca212c819fdb502f3471d Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 11 Sep 2023 23:26:10 +0200 Subject: [PATCH 27/62] move gain utils in one single folder --- .../calibration/makers/gain/utils/__init__.py | 2 + .../calibration/makers/gain/utils/error.py | 23 ++ .../calibration/makers/gain/utils/utils.py | 299 ++++++++++++++++++ 3 files changed, 324 insertions(+) create mode 100644 src/nectarchain/calibration/makers/gain/utils/__init__.py create mode 100644 src/nectarchain/calibration/makers/gain/utils/error.py create mode 100644 src/nectarchain/calibration/makers/gain/utils/utils.py diff --git a/src/nectarchain/calibration/makers/gain/utils/__init__.py b/src/nectarchain/calibration/makers/gain/utils/__init__.py new file mode 100644 index 00000000..b4996a25 --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/utils/__init__.py @@ -0,0 +1,2 @@ +from .error import * +from .utils import * \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/utils/error.py b/src/nectarchain/calibration/makers/gain/utils/error.py new file mode 100644 index 00000000..a136e7cc --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/utils/error.py @@ -0,0 +1,23 @@ +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + + +class DifferentPixelsID(Exception) : + def __init__(self,message) : + self.__message = message + @property + def message(self) : return self.__message + +class PedestalValueError(ValueError) : + def __init__(self,message) : + self.__message = message + @property + def message(self) : return self.__message + +class MeanValueError(ValueError) : + def __init__(self,message) : + self.__message = message + @property + def message(self) : return self.__message diff --git a/src/nectarchain/calibration/makers/gain/utils/utils.py b/src/nectarchain/calibration/makers/gain/utils/utils.py new file mode 100644 index 00000000..d1016a12 --- /dev/null +++ b/src/nectarchain/calibration/makers/gain/utils/utils.py @@ -0,0 +1,299 @@ +import logging +import math + +import numpy as np +from iminuit import Minuit +from scipy import interpolate, signal +from scipy.special import gammainc +from scipy.stats import norm,chi2 + +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +__all__ = ['UtilsMinuit','Multiprocessing'] + +from ..parameters import Parameters + +class Multiprocessing() : + @staticmethod + def custom_error_callback(error): + log.error(f'Got an error: {error}') + log.error(error,exc_info=True) + +class Statistics() : + @staticmethod + def chi2_pvalue(ndof : int, likelihood : float) : + return 1 - chi2(df = ndof).cdf(likelihood) + +class UtilsMinuit() : + @staticmethod + def make_minuit_par_kwargs(parameters : Parameters): + """Create *Parameter Keyword Arguments* for the `Minuit` constructor. + + updated for Minuit >2.0 + """ + names = parameters.parnames + kwargs = {"names": names,"values" : {}} + + for parameter in parameters.parameters: + kwargs["values"][parameter.name] = parameter.value + min_ = None if np.isnan(parameter.min) else parameter.min + max_ = None if np.isnan(parameter.max) else parameter.max + error = 0.1 if np.isnan(parameter.error) else parameter.error + kwargs[f"limit_{parameter.name}"] = (min_, max_) + kwargs[f"error_{parameter.name}"] = error + if parameter.frozen : + kwargs[f"fix_{parameter.name}"] = True + return kwargs + + @staticmethod + def set_minuit_parameters_limits_and_errors(m : Minuit,parameters : dict) : + """function to set minuit parameter limits and errors with Minuit >2.0 + + Args: + m (Minuit): a Minuit instance + parameters (dict): dict containing parameters names, limits errors and values + """ + for name in parameters["names"] : + m.limits[name] = parameters[f"limit_{name}"] + m.errors[name] = parameters[f"error_{name}"] + if parameters.get(f"fix_{name}",False) : + m.fixed[name] = True + + +# Usefull fucntions for the fit +def gaussian(x, mu, sig): + #return (1./(sig*np.sqrt(2*math.pi)))*np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.))) + return norm.pdf(x,loc = mu,scale = sig) + +def weight_gaussian(x,N, mu, sig) : + return N * gaussian(x, mu, sig) + +def doubleGauss(x,sig1,mu2,sig2,p): + return p *2 *gaussian(x, 0, sig1) + (1-p) * gaussian(x, mu2, sig2) + +def PMax(r): + """p_{max} in equation 6 in Caroff et al. (2019) + + Args: + r (float): SPE resolution + + Returns: + float : p_{max} + """ + if r > np.sqrt((np.pi -2 )/ 2) : + pmax = np.pi/(2 * (r**2 + 1)) + else : + pmax = np.pi*r**2/(np.pi*r**2 + np.pi - 2*r**2 - 2) + return pmax + +def ax(p,res): + """a in equation 4 in Caroff et al. (2019) + + Args: + p (float): proportion of the low charge component (2 gaussians model) + res (float): SPE resolution + + Returns: + float : a + """ + return ((2/np.pi)*p**2-p/(res**2+1)) + +def bx(p,mu2): + """b in equation 4 in Caroff et al. (2019) + + Args: + p (float): proportion of the low charge component (2 gaussians model) + mu2 (float): position of the high charge Gaussian + + Returns: + float : b + """ + return (np.sqrt(2/np.pi)*2*p*(1-p)*mu2) + +def cx(sig2,mu2,res,p): + """c in equation 4 in Caroff et al. (2019) + Note : There is a typo in the article 1-p**2 -> (1-p)**2 + + Args: + sig2 (float): width of the high charge Gaussian + mu2 (float): position of the high charge Gaussian + res (float): SPE resolution + p (float): proportion of the low charge component (2 gaussians model) + + Returns: + float : c + """ + return (1-p)**2*mu2**2 - (1-p)*(sig2**2+mu2**2)/(res**2+1) + +def delta(p,res,sig2,mu2): + """well known delta in 2nd order polynom + + Args: + p (_type_): _description_ + res (_type_): _description_ + sig2 (_type_): _description_ + mu2 (_type_): _description_ + + Returns: + float : b**2 - 4*a*c + """ + return bx(p,mu2)*bx(p,mu2) - 4*ax(p,res)*cx(sig2,mu2,res,p) + +def ParamU(p,r): + """d in equation 6 in Caroff et al. (2019) + + Args: + p (float): proportion of the low charge component (2 gaussians model) + r (float): SPE resolution + + Returns: + float : d + """ + return ((8*(1-p)**2*p**2)/np.pi - 4*(2*p**2/np.pi - p/(r**2+1))*((1-p)**2-(1-p)/(r**2+1))) + +def ParamS(p,r): + """e in equation 6 in Caroff et al. (2019) + + Args: + p (float): proportion of the low charge component (2 gaussians model) + r (float): SPE resolution + + Returns: + float : e + """ + e = (4*(2*p**2/np.pi - p/(r**2+1))*(1-p))/(r**2+1) + return e + +def SigMin(p,res,mu2): + """sigma_{high,min} in equation 6 in Caroff et al. (2019) + + Args: + p (float): proportion of the low charge component (2 gaussians model) + res (float): SPE resolution + mu2 (float): position of the high charge Gaussian + + Returns: + float : sigma_{high,min} + """ + return mu2*np.sqrt((-ParamU(p,res)+(bx(p,mu2)**2/mu2**2))/(ParamS(p,res))) + +def SigMax(p,res,mu2): + """sigma_{high,min} in equation 6 in Caroff et al. (2019) + + Args: + p (float): proportion of the low charge component (2 gaussians model) + res (float): SPE resolution + mu2 (float): position of the high charge Gaussian + + Returns: + float : sigma_{high,min} + """ + temp = (-ParamU(p,res))/(ParamS(p,res)) + if temp < 0 : + err = ValueError("-d/e must be < 0") + log.error(err,exc_info=True) + raise err + else : + return mu2*np.sqrt(temp) + +def sigma1(p,res,sig2,mu2): + """sigma_{low} in equation 5 in Caroff et al. (2019) + + Args: + sig2 (float): width of the high charge Gaussian + mu2 (float): position of the high charge Gaussian + res (float): SPE resolution + p (float): proportion of the low charge component (2 gaussians model) + + Returns: + float : sigma_{low} + """ + return (-bx(p,mu2)+np.sqrt(delta(p,res,sig2,mu2)))/(2*ax(p,res)) + +def sigma2(n,p,res,mu2): + """sigma_{high} in equation 7 in Caroff et al. (2019) + + Args: + n (float): parameter n in equation + mu2 (float): position of the high charge Gaussian + res (float): SPE resolution + p (float): proportion of the low charge component (2 gaussians model) + + Returns: + float : sigma_{high} + """ + if ((-ParamU(p,res)+(bx(p,mu2)**2/mu2**2))/(ParamS(p,res)) > 0): + return SigMin(p,res,mu2)+n*(SigMax(p,res,mu2)-SigMin(p,res,mu2)) + else: + return n*SigMax(p,res,mu2) + + # The real final model callign all the above for luminosity (lum) + PED, wil return probability of number of Spe +def MPE2(x,pp,res,mu2,n,muped,sigped,lum,**kwargs): + log.debug(f"pp = {pp}, res = {res}, mu2 = {mu2}, n = {n}, muped = {muped}, sigped = {sigped}, lum = {lum}") + f = 0 + ntotalPE = kwargs.get("ntotalPE",0) + if ntotalPE == 0 : + #about 1sec + for i in range(1000): + if (gammainc(i+1,lum) < 1e-5): + ntotalPE = i + break + #print(ntotalPE) + #about 8 sec, 1 sec by nPEPDF call + #for i in range(ntotalPE): + # f = f + ((lum**i)/math.factorial(i)) * np.exp(-lum) * nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) + + f = np.sum([(lum**i)/math.factorial(i) * np.exp(-lum) * nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) for i in range(ntotalPE)],axis = 0) # 10 % faster + return f + +# Fnal model shape/function (for one SPE) +def doubleGaussConstrained(x,pp,res,mu2,n): + p = pp*PMax(res) + sig2 = sigma2(n,p,res,mu2) + sig1 = sigma1(p,res,sig2,mu2) + return doubleGauss(x,sig1,mu2,sig2,p) + +# Get the gain from the parameters model +def Gain(pp,res,mu2,n): + """analytic gain computatuon + + Args: + mu2 (float): position of the high charge Gaussian + res (float): SPE resolution + pp (float): p' in equation 7 in Caroff et al. (2019) + n (float): n in equation 7 in Caroff et al. (2019) + + Returns: + float : gain + """ + p = pp*PMax(res) + sig2 = sigma2(n,p,res,mu2) + return (1-p)*mu2 + 2*p*sigma1(p,res,sig2,mu2)/np.sqrt(2*np.pi) + +def nPEPDF(x,pp,res,mu2,n,muped,sigped,nph,size_charge): + allrange = np.linspace(-1 * size_charge,size_charge,size_charge*2) + spe = [] + #about 2 sec this is the main pb + #for i in range(len(allrange)): + # if (allrange[i]>=0): + # spe.append(doubleGaussConstrained(allrange[i],pp,res,mu2,n)) + # else: + # spe.append(0) + + spe = doubleGaussConstrained(allrange,pp,res,mu2,n) * (allrange>=0 * np.ones(allrange.shape)) #100 times faster + + # ~ plt.plot(allrange,spe) + #npe = semi_gaussian(allrange, muped, sigped) + npe = gaussian(allrange, 0, sigped) + # ~ plt.plot(allrange,npe) + # ~ plt.show() + for i in range(nph): + #npe = np.convolve(npe,spe,"same") + npe = signal.fftconvolve(npe,spe,"same") + # ~ plt.plot(allrange,npe) + # ~ plt.show() + fff = interpolate.UnivariateSpline(allrange,npe,ext=1,k=3,s=0) + norm = np.trapz(fff(allrange),allrange) + return fff(x-muped)/norm \ No newline at end of file From 8596d2440cd6a1fb98838bf0242c58618cc75b64 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 11 Sep 2023 23:29:23 +0200 Subject: [PATCH 28/62] bugfix nominal voltage fit the fitting process continues although the initial parameters computation failed --- .../makers/gain/FlatFieldSPEMakers.py | 71 ++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py index 413f7497..c19d3e9c 100644 --- a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py @@ -42,6 +42,8 @@ from .utils import UtilsMinuit,weight_gaussian,Statistics,MPE2 +from .utils import MeanValueError,PedestalValueError + __all__ = ["FlatFieldSingleHHVSPEMaker","FlatFieldSingleHHVStdSPEMaker"] @@ -89,26 +91,31 @@ def read_param_from_yaml(self,parameters_file,only_update = False) : @staticmethod def _update_parameters(parameters,charge,counts,**kwargs) : - coeff_ped,coeff_mean = __class__._get_mean_gaussian_fit(charge,counts,**kwargs) - pedestal = parameters['pedestal'] - pedestal.value = coeff_ped[1] - pedestal.min = coeff_ped[1] - coeff_ped[2] - pedestal.max = coeff_ped[1] + coeff_ped[2] - log.debug(f"pedestal updated : {pedestal}") - pedestalWidth = parameters["pedestalWidth"] - pedestalWidth.value = pedestal.max - pedestal.value - pedestalWidth.max = 3 * pedestalWidth.value - log.debug(f"pedestalWidth updated : {pedestalWidth.value}") try : - if (coeff_mean[1] - pedestal.value < 0) or ((coeff_mean[1] - coeff_mean[2]) - pedestal.max < 0) : raise Exception("mean gaussian fit not good") + coeff_ped,coeff_mean = __class__._get_mean_gaussian_fit(charge,counts,**kwargs) + pedestal = parameters['pedestal'] + pedestal.value = coeff_ped[1] + pedestal.min = coeff_ped[1] - coeff_ped[2] + pedestal.max = coeff_ped[1] + coeff_ped[2] + log.debug(f"pedestal updated : {pedestal}") + pedestalWidth = parameters["pedestalWidth"] + pedestalWidth.value = pedestal.max - pedestal.value + pedestalWidth.max = 3 * pedestalWidth.value + log.debug(f"pedestalWidth updated : {pedestalWidth.value}") + + if (coeff_mean[1] - pedestal.value < 0) or ((coeff_mean[1] - coeff_mean[2]) - pedestal.max < 0) : raise MeanValueError("mean gaussian fit not good") mean = parameters['mean'] mean.value = coeff_mean[1] - pedestal.value mean.min = (coeff_mean[1] - coeff_mean[2]) - pedestal.max mean.max = (coeff_mean[1] + coeff_mean[2]) - pedestal.min log.debug(f"mean updated : {mean}") - except Exception as e : + except MeanValueError as e : log.warning(e,exc_info=True) log.warning("mean parameters limits and starting value not changed") + except Exception as e : + log.warning(e, exc_info = True) + log.warning("pedestal and mean parameters limits and starting value not changed") + return parameters @staticmethod @@ -180,7 +187,7 @@ class FlatFieldSingleHHVSPEMaker(FlatFieldSPEMaker) : __parameters_file = 'parameters_signal.yaml' __fit_array = None - __reduced_name = "FlatFieldSingleSPE" + _reduced_name = "FlatFieldSingleSPE" __nproc_default = 8 __chunksize_default = 1 @@ -225,13 +232,6 @@ def counts(self) : return copy.deepcopy(self.__counts) @property def _counts(self) : return self.__counts -#I/O method - def save(self,path,**kwargs) : - path = Path(path) - os.makedirs(path,exist_ok = True) - log.info(f'data saved in {path}') - self._results.write(f"{path}/results_{self.__reduced_name}.ecsv", format='ascii.ecsv',overwrite = kwargs.get("overwrite",False)) - #methods def _fill_results_table_from_dict(self,dico,pixels_id) : chi2_sig = signature(__class__.cost(self._charge,self._counts)) @@ -248,9 +248,9 @@ def _fill_results_table_from_dict(self,dico,pixels_id) : self._results[key][index] = values[j] self._results[f"{key}_error"][index] = errors[j] if key == 'mean' : - self._gain[index] = values[j] - self._results[f"gain_error"][index] = [errors[j],errors[j]] - self._results[f"gain"][index] = values[j] + self._high_gain[index] = values[j] + self._results[f"high_gain_error"][index] = [errors[j],errors[j]] + self._results[f"high_gain"][index] = values[j] self._results['is_valid'][index] = True self._results["likelihood"][index] = __class__.__fit_array[i].fcn(__class__.__fit_array[i].values) ndof = self._counts.data[index][~self._counts.mask[index]].shape[0] - __class__.__fit_array[i].nfit @@ -327,6 +327,7 @@ def make(self, log.info("running maker") log.info('checking asked pixels id') if pixels_id is None : + pixels_id = self.pixels_id npix = self.npixels else : log.debug('checking that asked pixels id are in data') @@ -343,7 +344,9 @@ def make(self, else : log.info("creation of the fits instance array") __class__.__fit_array = self._make_fit_array_from_parameters( - pixels_id = pixels_id + pixels_id = pixels_id, + display = display, + **kwargs ) log.info("running fits") @@ -420,8 +423,8 @@ def display(self,pixels_id,**kwargs) : self._counts[index], self._results['pp'][index].value, self._results['resolution'][index].value, - self._results['gain'][index].value, - self._results['gain_error'][index].value.mean(), + self._results['high_gain'][index].value, + self._results['high_gain_error'][index].value.mean(), self._results['n'][index].value, self._results['pedestal'][index].value, self._results['pedestalWidth'][index].value, @@ -438,7 +441,7 @@ def display(self,pixels_id,**kwargs) : class FlatFieldSingleHHVStdSPEMaker(FlatFieldSingleHHVSPEMaker): """class to perform fit of the SPE signal with n and pp fixed""" __parameters_file = 'parameters_signalStd.yaml' - __reduced_name = "FlatFieldSingleStdSPE" + _reduced_name = "FlatFieldSingleStdSPE" #constructors def __init__(self,charge,counts,*args,**kwargs) : @@ -461,14 +464,16 @@ class FlatFieldSingleNominalSPEMaker(FlatFieldSingleHHVSPEMaker): """class to perform fit of the SPE signal at nominal voltage from fitted data obtained with 1400V run Thus, n, pp and res are fixed""" __parameters_file = 'parameters_signal_fromHHVFit.yaml' - __reduced_name = "FlatFieldSingleNominalSPE" + _reduced_name = "FlatFieldSingleNominalSPE" #constructors - def __init__(self, charge, counts, nectarGainSPEresult : str, same_luminosity : bool = True, *args, **kwargs): + def __init__(self, charge, counts, nectarGainSPEresult : str, same_luminosity : bool = False, *args, **kwargs): super().__init__(charge, counts, *args, **kwargs) self.__fix_parameters(same_luminosity) self.__same_luminosity = same_luminosity self.__nectarGainSPEresult = self._read_SPEresult(nectarGainSPEresult) + if len(self.__nectarGainSPEresult) == 0 : + log.warning("The intersection between pixels id from the data and those valid from the SPE fit result is empty") #getters and setters @property @@ -480,6 +485,7 @@ def same_luminosity(self) : return copy.deepcopy(self.__same_luminosity) #methods def _read_SPEresult(self,nectarGainSPEresult : str) : table = QTable.read(nectarGainSPEresult,format = "ascii.ecsv") + table = table[table["is_valid"]] argsort = [] mask = [] for _id in self._pixels_id : @@ -487,12 +493,12 @@ def _read_SPEresult(self,nectarGainSPEresult : str) : argsort.append(np.where(_id==table['pixels_id'])[0][0]) mask.append(True) else : - mask.append(True) + mask.append(False) self._pixels_id = self._pixels_id[np.array(mask)] return table[np.array(argsort)] def __fix_parameters(self, same_luminosity : bool) : - """this method should be used to fix n, pp and res + """this method should be used to fix n, pp, res and possibly luminosity """ log.info("updating parameters by fixing pp, n and res") pp = self._parameters["pp"] @@ -502,11 +508,12 @@ def __fix_parameters(self, same_luminosity : bool) : resolution = self._parameters["resolution"] resolution.frozen = True if same_luminosity : + log.info("fixing luminosity") luminosity = self._parameters["luminosity"] luminosity.frozen = True def _make_fit_array_from_parameters(self, pixels_id = None, **kwargs) : - return super()._make_fit_array_from_parameters(self, pixels_id = pixels_id, nectarGainSPEresult = self.__nectarGainSPEresult, **kwargs) + return super()._make_fit_array_from_parameters(pixels_id = pixels_id, nectarGainSPEresult = self.__nectarGainSPEresult, **kwargs) @staticmethod def _update_parameters(parameters,charge,counts,pixel_id,nectarGainSPEresult,**kwargs) : From 77f12ab071423cb84dafd940b5819eebc569a98d Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 11 Sep 2023 23:30:06 +0200 Subject: [PATCH 29/62] Photostatistic implementation done --- .../makers/gain/PhotoStatisticMakers.py | 288 +++++++++--------- .../calibration/makers/gain/__init__.py | 2 +- .../ggrolleron/gain_PhotoStat_computation.py | 20 +- .../ggrolleron/gain_PhotoStat_computation.sh | 2 +- 4 files changed, 155 insertions(+), 157 deletions(-) diff --git a/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py b/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py index d0d4edeb..d3efdae2 100644 --- a/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py +++ b/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py @@ -14,19 +14,112 @@ import os from datetime import date from pathlib import Path - -from . import GainMaker +import copy from ctapipe_io_nectarcam import constants from ...container import ChargeContainer +from .gainMakers import GainMaker + + +__all__ = ["PhotoStatisticMaker"] + +class PhotoStatisticMaker(GainMaker): + _reduced_name = "PhotoStatistic" -__all__ = ["PhotoStatGainFFandPed"] +#constructors + def __init__(self, + FFcharge_hg, + FFcharge_lg, + Pedcharge_hg, + Pedcharge_lg, + coefCharge_FF_Ped, + SPE_resolution, + *args, + **kwargs + ) : + super().__init__(*args,**kwargs) + + self.__coefCharge_FF_Ped = coefCharge_FF_Ped + + self.__FFcharge_hg = FFcharge_hg + self.__FFcharge_lg = FFcharge_lg + + self.__Pedcharge_hg = Pedcharge_hg + self.__Pedcharge_lg = Pedcharge_lg + + if isinstance(SPE_resolution,np.ndarray) and len(SPE_resolution) == self.npixels : + self.__SPE_resolution = SPE_resolution + elif isinstance(SPE_resolution,list) and len(SPE_resolution) == self.npixels : + self.__SPE_resolution = np.array(SPE_resolution) + elif isinstance(SPE_resolution,float) : + self.__SPE_resolution = SPE_resolution * np.ones((self.npixels)) + else : + e = TypeError("SPE_resolution must be a float, a numpy.ndarray or list instance") + raise e + + self.__check_shape() + + + @classmethod + def create_from_chargeContainer(cls, + FFcharge : ChargeContainer, + Pedcharge : ChargeContainer, + coefCharge_FF_Ped, + SPE_resolution, + **kwargs) : + if isinstance(SPE_resolution , str) or isinstance(SPE_resolution , Path) : + SPE_resolution,SPE_pixels_id = __class__.__readSPE(SPE_resolution) + else : + SPE_pixels_id = None + + kwargs_init = __class__.__get_charges_FF_Ped_reshaped(FFcharge, + Pedcharge, + SPE_resolution, + SPE_pixels_id) + + kwargs.update(kwargs_init) + return cls(coefCharge_FF_Ped = coefCharge_FF_Ped, **kwargs) + + @classmethod + def create_from_run_numbers(cls, FFrun : int, Pedrun : int, SPE_resolution : str, **kwargs) : + FFkwargs = __class__.__readFF(FFrun, **kwargs) + Pedkwargs = __class__.__readPed(Pedrun, **kwargs) + kwargs.update(FFkwargs) + kwargs.update(Pedkwargs) + return cls.create_from_chargeContainer(SPE_resolution = SPE_resolution, **kwargs) + +#methods + @staticmethod + def __readSPE(SPEresults) : + log.info(f'reading SPE resolution from {SPEresults}') + table = QTable.read(SPEresults) + table.sort('pixels_id') + return table['resolution'][table['is_valid']].value,table['pixels_id'][table['is_valid']].value -class PhotoStatisticMaker(ABC): + @staticmethod + def __get_charges_FF_Ped_reshaped( FFcharge : ChargeContainer, Pedcharge : ChargeContainer, SPE_resolution, SPE_pixels_id) : + log.info("reshape of SPE, Ped and FF data with intersection of pixel ids") + out = {} + + FFped_intersection = np.intersect1d(Pedcharge.pixels_id,FFcharge.pixels_id) + if not(SPE_pixels_id is None) : + SPEFFPed_intersection = np.intersect1d(FFped_intersection,SPE_pixels_id) + mask_SPE = np.array([SPE_pixels_id[i] in SPEFFPed_intersection for i in range(len(SPE_pixels_id))],dtype = bool) + out["SPE_resolution"] = SPE_resolution[mask_SPE] + + out["pixels_id"] = SPEFFPed_intersection + out["FFcharge_hg"] = FFcharge.select_charge_hg(SPEFFPed_intersection) + out["FFcharge_lg"] = FFcharge.select_charge_lg(SPEFFPed_intersection) + out["Pedcharge_hg"] = Pedcharge.select_charge_hg(SPEFFPed_intersection) + out["Pedcharge_lg"] = Pedcharge.select_charge_lg(SPEFFPed_intersection) + + log.info(f"data have {len(SPEFFPed_intersection)} pixels in common") + return out - def _readFF(self,FFRun,maxevents: int = None,**kwargs) : + @staticmethod + def __readFF(FFRun,**kwargs) : log.info('reading FF data') method = kwargs.get('method','FullWaveformSum') FFchargeExtractorWindowLength = kwargs.get('FFchargeExtractorWindowLength',None) @@ -36,226 +129,119 @@ def _readFF(self,FFRun,maxevents: int = None,**kwargs) : log.error(e,exc_info=True) raise e else : - self.__coefCharge_FF_Ped = FFchargeExtractorWindowLength / constants.N_SAMPLES + coefCharge_FF_Ped = FFchargeExtractorWindowLength / constants.N_SAMPLES else : - self.__coefCharge_FF_Ped = 1 + coefCharge_FF_Ped = 1 if isinstance(FFRun,int) : try : - self.FFcharge = ChargeContainer.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{method}",FFRun) + FFcharge = ChargeContainer.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{method}",FFRun) log.info(f'charges have ever been computed for FF run {FFRun}') except Exception as e : log.error("charge have not been yet computed") raise e - - #log.info(f'loading waveforms for FF run {FFRun}') - #FFwaveforms = WaveformsContainer(FFRun,maxevents) - #FFwaveforms.load_wfs() - #if method != 'std' : - # log.info(f'computing charge for FF run {FFRun} with following method : {method}') - # self.FFcharge = ChargeContainer.from_waveforms(FFwaveforms,method = method) - #else : - # log.info(f'computing charge for FF run {FFRun} with std method') - # self.FFcharge = ChargeContainer.from_waveforms(FFwaveforms) - #log.debug('writting on disk charge for further works') - #os.makedirs(f"{os.environ['NECTARCAMDATA']}/charges/{method}",exist_ok = True) - #self.FFcharge.write(f"{os.environ['NECTARCAMDATA']}/charges/{method}",overwrite = True) - - elif isinstance(FFRun,ChargeContainer): - self.FFcharge = FFRun else : e = TypeError("FFRun must be int or ChargeContainer") log.error(e,exc_info = True) raise e + return {"FFcharge" : FFcharge, "coefCharge_FF_Ped" : coefCharge_FF_Ped} - def _readPed(self,PedRun,maxevents: int = None,**kwargs) : + @staticmethod + def __readPed(PedRun,**kwargs) : log.info('reading Ped data') method = 'FullWaveformSum'#kwargs.get('method','std') if isinstance(PedRun,int) : try : - self.Pedcharge = ChargeContainer.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{method}",PedRun) + Pedcharge = ChargeContainer.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{method}",PedRun) log.info(f'charges have ever been computed for Ped run {PedRun}') except Exception as e : log.error("charge have not been yet computed") raise e - - #log.info(f'loading waveforms for Ped run {PedRun}') - #Pedwaveforms = WaveformsContainer(PedRun,maxevents) - #Pedwaveforms.load_wfs() - #if method != 'std' : - # log.info(f'computing charge for Ped run {PedRun} with following method : {method}') - # self.Pedcharge = ChargeContainer.from_waveforms(Pedwaveforms,method = method) - #else : - # log.info(f'computing charge for Ped run {PedRun} with std method') - # self.Pedcharge = ChargeContainer.from_waveforms(Pedwaveforms) - #log.debug('writting on disk charge for further works') - #os.makedirs(f"{os.environ['NECTARCAMDATA']}/charges/{method}",exist_ok = True) - #self.Pedcharge.write(f"{os.environ['NECTARCAMDATA']}/charges/{method}",overwrite = True) - - elif isinstance(PedRun,ChargeContainer): - self.Pedcharge = PedRun else : e = TypeError("PedRun must be int or ChargeContainer") log.error(e,exc_info = True) raise e + return {"Pedcharge" : Pedcharge} - def _readSPE(self,SPEresults) : - log.info(f'reading SPE resolution from {SPEresults}') - table = QTable.read(SPEresults) - table.sort('pixel') - self.SPEResolution = table['resolution'] - self.SPEGain = table['gain'] - self.SPEGain_error = table['gain_error'] - self._SPEvalid = table['is_valid'] - self._SPE_pixels_id = np.array(table['pixel'].value,dtype = np.uint16) - - def _reshape_all(self) : - log.info("reshape of SPE, Ped and FF data with intersection of pixel ids") - FFped_intersection = np.intersect1d(self.Pedcharge.pixels_id,self.FFcharge.pixels_id) - SPEFFPed_intersection = np.intersect1d(FFped_intersection,self._SPE_pixels_id[self._SPEvalid]) - self._pixels_id = SPEFFPed_intersection - log.info(f"data have {len(self._pixels_id)} pixels in common") - - self._FFcharge_hg = self.FFcharge.select_charge_hg(SPEFFPed_intersection) - self._FFcharge_lg = self.FFcharge.select_charge_lg(SPEFFPed_intersection) - - self._Pedcharge_hg = self.Pedcharge.select_charge_hg(SPEFFPed_intersection) - self._Pedcharge_lg = self.Pedcharge.select_charge_lg(SPEFFPed_intersection) - - #self._mask_FF = np.array([self.FFcharge.pixels_id[i] in SPEFFPed_intersection for i in range(self.FFcharge.npixels)],dtype = bool) - #self._mask_Ped = np.array([self.Pedcharge.pixels_id[i] in SPEFFPed_intersection for i in range(self.Pedcharge.npixels)],dtype = bool) - self._mask_SPE = np.array([self._SPE_pixels_id[i] in SPEFFPed_intersection for i in range(len(self._SPE_pixels_id))],dtype = bool) - - - - def create_output_table(self) : - self._output_table = QTable() - self._output_table.meta['npixel'] = self.npixels - self._output_table.meta['comments'] = f'Produced with NectarGain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' - - self._output_table.add_column(Column(np.ones((self.npixels),dtype = bool),"is_valid",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(self.pixels_id,"pixel",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),"high gain",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"high gain error",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),"low gain",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"low gain error",unit = u.dimensionless_unscaled)) + def __check_shape(self) : + try : + self.__FFcharge_hg[0] * self.__FFcharge_lg[0] * self.__Pedcharge_hg[0] * self.__Pedcharge_lg[0] * self.__SPE_resolution * self._pixels_id + except Exception as e : + log.error(e,exc_info = True) + raise e - def run(self,**kwargs): + def make(self,**kwargs): log.info('running photo statistic method') + self._results["high_gain"] = self.gainHG + self._results["low_gain"] = self.gainLG + #self._results["is_valid"] = self._SPEvalid - self._output_table["high gain"] = self.gainHG - self._output_table["low gain"] = self.gainLG - #self._output_table["is_valid"] = self._SPEvalid - - def save(self,path,**kwargs) : - path = Path(path) - os.makedirs(path,exist_ok = True) - log.info(f'data saved in {path}') - self._output_table.write(f"{path}/output_table.ecsv", format='ascii.ecsv',overwrite = kwargs.get("overwrite",False)) - def plot_correlation(self) : - mask = (self._output_table["high gain"]>20) * (self.SPEGain[self._mask_SPE]>0) * (self._output_table["high gain"]<80) * self._output_table['is_valid'] - a, b, r, p_value, std_err = linregress(self._output_table["high gain"][mask], self.SPEGain[self._mask_SPE][mask],'greater') - x = np.linspace(self._output_table["high gain"][mask].min(),self._output_table["high gain"][mask].max(),1000) + def plot_correlation(photoStat_gain,SPE_gain) : + mask = (photoStat_gain>20) * (SPE_gain>0) * (photoStat_gain<80) + a, b, r, p_value, std_err = linregress(photoStat_gain[mask], SPE_gain[mask],'greater') + x = np.linspace(photoStat_gain[mask].min(),photoStat_gain[mask].max(),1000) y = lambda x: a * x + b with quantity_support() : fig,ax = plt.subplots(1,1,figsize=(8, 6)) - ax.scatter(self._output_table["high gain"][mask],self.SPEGain[self._mask_SPE][mask],marker =".") + ax.scatter(photoStat_gain[mask],SPE_gain[mask],marker =".") ax.plot(x,y(x),color = 'red', label = f"linear fit,\n a = {a:.2e},\n b = {b:.2e},\n r = {r:.2e},\n p_value = {p_value:.2e},\n std_err = {std_err:.2e}") ax.plot(x,x,color = 'black',label = "y = x") ax.set_xlabel("Gain Photo stat (ADC)", size=15) ax.set_ylabel("Gain SPE fit (ADC)", size=15) #ax.set_xlim(xmin = 0) #ax.set_ylim(ymin = 0) - ax.legend(fontsize=15) return fig +#getters and setters @property - def npixels(self) : return self._pixels_id.shape[0] - - @property - def pixels_id(self) : return self._pixels_id + def SPE_resolution(self) : return copy.deepcopy(self.__SPE_resolution) @property - def sigmaPedHG(self) : return np.std(self._Pedcharge_hg ,axis = 0) * np.sqrt(self.__coefCharge_FF_Ped) + def sigmaPedHG(self) : return np.std(self.__Pedcharge_hg ,axis = 0) * np.sqrt(self.__coefCharge_FF_Ped) @property - def sigmaChargeHG(self) : return np.std(self._FFcharge_hg - self.meanPedHG, axis = 0) + def sigmaChargeHG(self) : return np.std(self.__FFcharge_hg - self.meanPedHG, axis = 0) @property - def meanPedHG(self) : return np.mean(self._Pedcharge_hg ,axis = 0) * self.__coefCharge_FF_Ped + def meanPedHG(self) : return np.mean(self.__Pedcharge_hg ,axis = 0) * self.__coefCharge_FF_Ped @property - def meanChargeHG(self) : return np.mean(self._FFcharge_hg - self.meanPedHG, axis = 0) + def meanChargeHG(self) : return np.mean(self.__FFcharge_hg - self.meanPedHG, axis = 0) @property def BHG(self) : - min_events = np.min((self._FFcharge_hg.shape[0],self._Pedcharge_hg.shape[0])) - upper = (np.power(self._FFcharge_hg.mean(axis = 1)[:min_events] - self._Pedcharge_hg.mean(axis = 1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeHG.mean(),2)).mean(axis = 0) + min_events = np.min((self.__FFcharge_hg.shape[0],self.__Pedcharge_hg.shape[0])) + upper = (np.power(self.__FFcharge_hg.mean(axis = 1)[:min_events] - self.__Pedcharge_hg.mean(axis = 1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeHG.mean(),2)).mean(axis = 0) lower = np.power(self.meanChargeHG.mean(),2)#np.power(self.meanChargeHG,2)#np.power(self.meanChargeHG.mean(),2) return np.sqrt(upper/lower) @property def gainHG(self) : return ((np.power(self.sigmaChargeHG,2) - np.power(self.sigmaPedHG,2) - np.power(self.BHG * self.meanChargeHG,2)) - /(self.meanChargeHG * (1 + np.power(self.SPEResolution[self._mask_SPE],2)))) + /(self.meanChargeHG * (1 + np.power(self.SPE_resolution,2)))) @property - def sigmaPedLG(self) : return np.std(self._Pedcharge_lg ,axis = 0) * np.sqrt(self.__coefCharge_FF_Ped) + def sigmaPedLG(self) : return np.std(self.__Pedcharge_lg ,axis = 0) * np.sqrt(self.__coefCharge_FF_Ped) @property - def sigmaChargeLG(self) : return np.std(self._FFcharge_lg - self.meanPedLG,axis = 0) + def sigmaChargeLG(self) : return np.std(self.__FFcharge_lg - self.meanPedLG,axis = 0) @property - def meanPedLG(self) : return np.mean(self._Pedcharge_lg,axis = 0) * self.__coefCharge_FF_Ped + def meanPedLG(self) : return np.mean(self.__Pedcharge_lg,axis = 0) * self.__coefCharge_FF_Ped @property - def meanChargeLG(self) : return np.mean(self._FFcharge_lg - self.meanPedLG,axis = 0) + def meanChargeLG(self) : return np.mean(self.__FFcharge_lg - self.meanPedLG,axis = 0) @property def BLG(self) : - min_events = np.min((self._FFcharge_lg.shape[0],self._Pedcharge_lg.shape[0])) - upper = (np.power(self._FFcharge_lg.mean(axis = 1)[:min_events] - self._Pedcharge_lg.mean(axis = 1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeLG.mean(),2)).mean(axis = 0) + min_events = np.min((self.__FFcharge_lg.shape[0],self.__Pedcharge_lg.shape[0])) + upper = (np.power(self.__FFcharge_lg.mean(axis = 1)[:min_events] - self.__Pedcharge_lg.mean(axis = 1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeLG.mean(),2)).mean(axis = 0) lower = np.power(self.meanChargeLG.mean(),2) #np.power(self.meanChargeLG,2) #np.power(self.meanChargeLG.mean(),2) return np.sqrt(upper/lower) @property def gainLG(self) : return ((np.power(self.sigmaChargeLG,2) - np.power(self.sigmaPedLG,2) - np.power(self.BLG * self.meanChargeLG,2)) - /(self.meanChargeLG * (1 + np.power(self.SPEResolution[self._mask_SPE],2)))) - - - -class PhotoStatGainFFandPed(PhotoStatGain): - def __init__(self, FFRun, PedRun, SPEresults : str, maxevents : int = None, **kwargs) : - self._readFF(FFRun,maxevents,**kwargs) - self._readPed(PedRun,maxevents,**kwargs) - - """ - if self.FFcharge.charge_hg.shape[1] != self.Pedcharge.charge_hg.shape[1] : - e = Exception("Ped run and FF run must have the same number of pixels") - log.error(e,exc_info = True) - raise e - """ - - self._readSPE(SPEresults) - ##need to implement reshape of SPE results with FF and Ped pixels ids - self._reshape_all() - - """ - if (self.FFcharge.pixels_id.shape[0] != self._SPE_pixels_id.shape[0]) : - e = Exception("Ped run and FF run must have the same number of pixels as SPE fit results") - log.error(e,exc_info = True) - raise e - - if (self.FFcharge.pixels_id != self.Pedcharge.pixels_id).any() or (self.FFcharge.pixels_id != self._SPE_pixels_id).any() or (self.Pedcharge.pixels_id != self._SPE_pixels_id).any() : - e = DifferentPixelsID("Ped run, FF run and SPE run need to have same pixels id") - log.error(e,exc_info = True) - raise e - else : - self._pixels_id = self.FFcharge.pixels_id - """ - self.create_output_table() - + /(self.meanChargeLG * (1 + np.power(self.SPE_resolution,2)))) -class PhotoStatisticMaker(GainMaker) : diff --git a/src/nectarchain/calibration/makers/gain/__init__.py b/src/nectarchain/calibration/makers/gain/__init__.py index 0897bc0b..a4100bfb 100644 --- a/src/nectarchain/calibration/makers/gain/__init__.py +++ b/src/nectarchain/calibration/makers/gain/__init__.py @@ -1,3 +1,3 @@ from .FlatFieldSPEMakers import * #from .WhiteTargetSPEMakers import * -#from .PhotoStatisticMakers import * \ No newline at end of file +from .PhotoStatisticMakers import * \ No newline at end of file diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py index 3ad357ab..1e5ed511 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py @@ -12,7 +12,10 @@ import argparse -from nectarchain.calibration.NectarGain import PhotoStatGainFFandPed +from astropy.table import QTable + +from nectarchain.calibration.makers.gain.PhotoStatisticMakers import PhotoStatisticMaker + parser = argparse.ArgumentParser( prog = 'gain_PhotoStat_computation.py', @@ -77,14 +80,23 @@ def main(args) : figpath = os.environ.get('NECTARCHAIN_FIGURES') - photoStat_FFandPed = PhotoStatGainFFandPed(args.FF_run_number, args.ped_run_number, SPEresults = args.SPE_fit_results,method = args.chargeExtractorPath, FFchargeExtractorWindowLength = args.FFchargeExtractorWindowLength) - photoStat_FFandPed.run() + photoStat_FFandPed = PhotoStatisticMaker.create_from_run_numbers( + FFrun = args.FF_run_number, + Pedrun = args.ped_run_number, + SPE_resolution = args.SPE_fit_results, + method = args.chargeExtractorPath, + FFchargeExtractorWindowLength = args.FFchargeExtractorWindowLength + ) + photoStat_FFandPed.make() photoStat_FFandPed.save(f"{os.environ.get('NECTARCAMDATA')}/../PhotoStat/data/PhotoStat-FF{args.FF_run_number}-ped{args.ped_run_number}-SPEres{args.SPE_fit_results_tag}-{args.chargeExtractorPath}/",overwrite = args.overwrite) log.info(f"BF^2 HG : {np.power(np.mean(photoStat_FFandPed.BHG),2)}") log.info(f"BF^2 LG : {np.power(np.mean(photoStat_FFandPed.BLG),2)}") if args.correlation : - fig = photoStat_FFandPed.plot_correlation() + table = QTable.read(args.SPE_fit_results,format = 'ascii.ecsv') + table.sort('pixels_id') + mask = np.array([pix in photoStat_FFandPed.pixels_id for pix in table['pixels_id'].value],dtype = bool) + fig = PhotoStatisticMaker.plot_correlation(photoStat_FFandPed.results['high_gain'],table['high_gain'][mask]) os.makedirs(f"{figpath}/PhotoStat-FF{args.FF_run_number}-ped{args.ped_run_number}-{args.chargeExtractorPath}/",exist_ok=True) fig.savefig(f"{figpath}/PhotoStat-FF{args.FF_run_number}-ped{args.ped_run_number}-{args.chargeExtractorPath}/correlation_PhotoStat_SPE{args.SPE_fit_results_tag}.pdf") diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.sh b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.sh index aa4c8dc8..e6809ed3 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.sh +++ b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.sh @@ -2,4 +2,4 @@ #to perform photo-statistic high and low gain computation with pedestal run 2609, flat field run 2609 and SPE fit result from run 2634 (1400V run) python gain_PhotoStat_computation.py -p 2609 -f 2608 --chargeExtractorPath LocalPeakWindowSum_4-12 --FFchargeExtractorWindowLength 16 --correlation --overwrite --SPE_fit_results "/data/users/ggroller/NECTARCAM/SPEfit/data/MULTI-1400V-SPEStd-2634-LocalPeakWindowSum_4-12/output_table.ecsv" --SPE_fit_results_tag VVH2634 -python gain_PhotoStat_computation.py -p 3938 -f 3937 --chargeExtractorPath LocalPeakWindowSum_4-12 --FFchargeExtractorWindowLength 16 --correlation --overwrite --SPE_fit_results "/data/users/ggroller/NECTARCAM/SPEfit/data/MULTI-1400V-SPEStd-2634-LocalPeakWindowSum_4-12/output_table.ecsv" --SPE_fit_results_tag VVH2634 +python gain_PhotoStat_computation.py -p 3938 -f 3937 --chargeExtractorPath LocalPeakWindowSum_4-12 --FFchargeExtractorWindowLength 16 --correlation --overwrite --SPE_fit_results "/data/users/ggroller/NECTARCAM/SPEfit/data/MULTI-1400V-SPEStd-3942-LocalPeakWindowSum_4-12/results_FlatFieldSingleStdSPE.ecsv" --SPE_fit_results_tag VVH3942 From 043601dd2c664ed4add140da8a3e6a970fa968d1 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 11 Sep 2023 23:40:49 +0200 Subject: [PATCH 30/62] cleaning : import, type, etc --- src/nectarchain/calibration/makers/core.py | 2 +- .../calibration/makers/flatfieldMakers.py | 6 +- .../makers/gain/FlatFieldSPEMakers.py | 24 +- .../makers/gain/PhotoStatisticMakers.py | 8 +- .../makers/gain/WhiteTargetSPEMakers.py | 15 + .../calibration/makers/gain/gainMakers.py | 2 - .../calibration/makers/gain/parameters.py | 12 +- .../makers/gain/parameters_ped.yaml | 7 - .../calibration/makers/gain/utils.py | 298 ------------------ .../calibration/makers/gain/utils/utils.py | 2 +- .../calibration/makers/pedestalMakers.py | 5 +- 11 files changed, 44 insertions(+), 337 deletions(-) delete mode 100644 src/nectarchain/calibration/makers/gain/parameters_ped.yaml delete mode 100644 src/nectarchain/calibration/makers/gain/utils.py diff --git a/src/nectarchain/calibration/makers/core.py b/src/nectarchain/calibration/makers/core.py index 887c3dc7..0213f723 100644 --- a/src/nectarchain/calibration/makers/core.py +++ b/src/nectarchain/calibration/makers/core.py @@ -35,7 +35,7 @@ def __init__(self,pixels_id,*args,**kwargs) -> None: #methods @abstractmethod - def make(self,*args,**kwargs) : + def make(self,*args,**kwargs) : pass #I/O method diff --git a/src/nectarchain/calibration/makers/flatfieldMakers.py b/src/nectarchain/calibration/makers/flatfieldMakers.py index f2cf419d..4df1e325 100644 --- a/src/nectarchain/calibration/makers/flatfieldMakers.py +++ b/src/nectarchain/calibration/makers/flatfieldMakers.py @@ -1,4 +1,3 @@ -import sys import logging logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') log = logging.getLogger(__name__) @@ -10,8 +9,7 @@ __all__ = ["FlatfieldMaker"] class FlatfieldMaker(CalibrationMaker) : - def __init__(self,**kwargs) : - super().__init__(**kwargs) - + def __init__(self,*args,**kwargs) -> None: + super().__init__(*args,**kwargs) def make(self) : raise NotImplementedError("The computation of the flatfield calibration is not yet implemented, feel free to contribute !:)") \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py index c19d3e9c..b84bedb5 100644 --- a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py @@ -13,7 +13,6 @@ import yaml import time -from pathlib import Path import matplotlib.pyplot as plt from matplotlib.patches import Rectangle @@ -32,8 +31,6 @@ from inspect import signature -from numba import njit, prange - from .gainMakers import GainMaker from ...container import ChargeContainer @@ -53,7 +50,7 @@ class FlatFieldSPEMaker(GainMaker) : _Order = 2 #constructors - def __init__(self,*args,**kwargs) : + def __init__(self,*args,**kwargs) -> None: super().__init__(*args,**kwargs) self.__parameters = Parameters() @@ -66,7 +63,7 @@ def parameters(self) : return copy.deepcopy(self.__parameters) def _parameters(self) : return self.__parameters #methods - def read_param_from_yaml(self,parameters_file,only_update = False) : + def read_param_from_yaml(self,parameters_file,only_update = False) -> None: with open(f"{os.path.dirname(os.path.abspath(__file__))}/{parameters_file}") as parameters : param = yaml.safe_load(parameters) if only_update : @@ -174,7 +171,7 @@ def _get_mean_gaussian_fit(charge, counts ,extension = "",**kwargs): return coeff, coeff_mean - def _update_table_from_parameters(self) : + def _update_table_from_parameters(self) -> None: for param in self._parameters.parameters : if not(param.name in self._results.colnames) : self._results.add_column(Column(data = np.empty((self.npixels),dtype = np.float64),name = param.name,unit = param.unit)) @@ -192,13 +189,10 @@ class FlatFieldSingleHHVSPEMaker(FlatFieldSPEMaker) : __chunksize_default = 1 #constructors - def __init__(self,charge,counts,*args,**kwargs) : + def __init__(self,charge,counts,*args,**kwargs) -> None: super().__init__(*args,**kwargs) self.__charge = charge self.__counts = counts - self.__mask_fitted_pixel = np.zeros((self.__charge.shape[0]),dtype = bool) - - self.__pedestal = Parameter(name = "pedestal", value = (np.min(self.__charge) + np.sum(self.__charge * self.__counts)/(np.sum(self.__counts)))/2, @@ -233,7 +227,7 @@ def counts(self) : return copy.deepcopy(self.__counts) def _counts(self) : return self.__counts #methods - def _fill_results_table_from_dict(self,dico,pixels_id) : + def _fill_results_table_from_dict(self,dico,pixels_id) -> None: chi2_sig = signature(__class__.cost(self._charge,self._counts)) for i in range(len(pixels_id)) : values = dico[i].get(f"values_{i}",None) @@ -444,12 +438,12 @@ class FlatFieldSingleHHVStdSPEMaker(FlatFieldSingleHHVSPEMaker): _reduced_name = "FlatFieldSingleStdSPE" #constructors - def __init__(self,charge,counts,*args,**kwargs) : + def __init__(self,charge,counts,*args,**kwargs) -> None: super().__init__(charge,counts,*args,**kwargs) self.__fix_parameters() #methods - def __fix_parameters(self) : + def __fix_parameters(self) -> None: """this method should be used to fix n and pp """ log.info("updating parameters by fixing pp and n") @@ -467,7 +461,7 @@ class FlatFieldSingleNominalSPEMaker(FlatFieldSingleHHVSPEMaker): _reduced_name = "FlatFieldSingleNominalSPE" #constructors - def __init__(self, charge, counts, nectarGainSPEresult : str, same_luminosity : bool = False, *args, **kwargs): + def __init__(self, charge, counts, nectarGainSPEresult : str, same_luminosity : bool = False, *args, **kwargs) -> None: super().__init__(charge, counts, *args, **kwargs) self.__fix_parameters(same_luminosity) self.__same_luminosity = same_luminosity @@ -497,7 +491,7 @@ def _read_SPEresult(self,nectarGainSPEresult : str) : self._pixels_id = self._pixels_id[np.array(mask)] return table[np.array(argsort)] - def __fix_parameters(self, same_luminosity : bool) : + def __fix_parameters(self, same_luminosity : bool) -> None: """this method should be used to fix n, pp, res and possibly luminosity """ log.info("updating parameters by fixing pp, n and res") diff --git a/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py b/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py index d3efdae2..2c29b78c 100644 --- a/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py +++ b/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py @@ -38,7 +38,7 @@ def __init__(self, SPE_resolution, *args, **kwargs - ) : + ) -> None: super().__init__(*args,**kwargs) self.__coefCharge_FF_Ped = coefCharge_FF_Ped @@ -162,14 +162,14 @@ def __readPed(PedRun,**kwargs) : raise e return {"Pedcharge" : Pedcharge} - def __check_shape(self) : + def __check_shape(self) -> None: try : self.__FFcharge_hg[0] * self.__FFcharge_lg[0] * self.__Pedcharge_hg[0] * self.__Pedcharge_lg[0] * self.__SPE_resolution * self._pixels_id except Exception as e : log.error(e,exc_info = True) raise e - def make(self,**kwargs): + def make(self,**kwargs)-> None: log.info('running photo statistic method') self._results["high_gain"] = self.gainHG self._results["low_gain"] = self.gainLG @@ -191,7 +191,7 @@ def plot_correlation(photoStat_gain,SPE_gain) : #ax.set_xlim(xmin = 0) #ax.set_ylim(ymin = 0) ax.legend(fontsize=15) - return fig + return fig #getters and setters @property diff --git a/src/nectarchain/calibration/makers/gain/WhiteTargetSPEMakers.py b/src/nectarchain/calibration/makers/gain/WhiteTargetSPEMakers.py index e69de29b..0e455b1a 100644 --- a/src/nectarchain/calibration/makers/gain/WhiteTargetSPEMakers.py +++ b/src/nectarchain/calibration/makers/gain/WhiteTargetSPEMakers.py @@ -0,0 +1,15 @@ +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +from .gainMakers import GainMaker + + +__all__ = ["FlatfieldMaker"] + +class WhiteTargetSPEMaker(GainMaker) : + def __init__(self,*args,**kwargs) -> None: + super().__init__(*args,**kwargs) + def make(self) : + raise NotImplementedError("The computation of the white target calibration is not yet implemented, feel free to contribute !:)") \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/gainMakers.py b/src/nectarchain/calibration/makers/gain/gainMakers.py index 7cd9fbea..a9adac8c 100644 --- a/src/nectarchain/calibration/makers/gain/gainMakers.py +++ b/src/nectarchain/calibration/makers/gain/gainMakers.py @@ -1,10 +1,8 @@ -import sys import logging logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') log = logging.getLogger(__name__) log.handlers = logging.getLogger('__main__').handlers -import os import numpy as np from copy import copy from astropy.table import Column diff --git a/src/nectarchain/calibration/makers/gain/parameters.py b/src/nectarchain/calibration/makers/gain/parameters.py index b01e48f8..11c6e8d9 100644 --- a/src/nectarchain/calibration/makers/gain/parameters.py +++ b/src/nectarchain/calibration/makers/gain/parameters.py @@ -10,7 +10,15 @@ __all__ = ["Parameter","Parameters"] class Parameter() : - def __init__(self, name, value, min = np.nan, max = np.nan, error = np.nan, unit = u.dimensionless_unscaled, frozen : bool = False): + def __init__(self, + name, + value, + min = np.nan, + max = np.nan, + error = np.nan, + unit = u.dimensionless_unscaled, + frozen : bool = False + )-> None: self.__name = name self.__value = value self.__error = error @@ -67,7 +75,7 @@ def __init__(self,parameters_liste : list = []) -> None: self.__parameters = copy.deepcopy(parameters_liste) - def append(self,parameter : Parameter) : + def append(self,parameter : Parameter) -> None: self.__parameters.append(parameter) def __getitem__(self,key) : diff --git a/src/nectarchain/calibration/makers/gain/parameters_ped.yaml b/src/nectarchain/calibration/makers/gain/parameters_ped.yaml deleted file mode 100644 index e1fab9db..00000000 --- a/src/nectarchain/calibration/makers/gain/parameters_ped.yaml +++ /dev/null @@ -1,7 +0,0 @@ -{ - pedestalWidth : { - value : 50, - min : 1, - max : 150 - } -} \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/utils.py b/src/nectarchain/calibration/makers/gain/utils.py deleted file mode 100644 index 44686680..00000000 --- a/src/nectarchain/calibration/makers/gain/utils.py +++ /dev/null @@ -1,298 +0,0 @@ -import logging -import math - -import numpy as np -from iminuit import Minuit -from scipy import interpolate, signal -from scipy.special import gammainc -from scipy.stats import norm,chi2 - -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') -log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers - -__all__ = ['UtilsMinuit','Multiprocessing'] -from .parameters import Parameters - -class Multiprocessing() : - @staticmethod - def custom_error_callback(error): - log.error(f'Got an error: {error}') - log.error(error,exc_info=True) - -class Statistics() : - @staticmethod - def chi2_pvalue(ndof : int, likelihood : float) : - return 1 - chi2(df = ndof).cdf(likelihood) - -class UtilsMinuit() : - @staticmethod - def make_minuit_par_kwargs(parameters : Parameters): - """Create *Parameter Keyword Arguments* for the `Minuit` constructor. - - updated for Minuit >2.0 - """ - names = parameters.parnames - kwargs = {"names": names,"values" : {}} - - for parameter in parameters.parameters: - kwargs["values"][parameter.name] = parameter.value - min_ = None if np.isnan(parameter.min) else parameter.min - max_ = None if np.isnan(parameter.max) else parameter.max - error = 0.1 if np.isnan(parameter.error) else parameter.error - kwargs[f"limit_{parameter.name}"] = (min_, max_) - kwargs[f"error_{parameter.name}"] = error - if parameter.frozen : - kwargs[f"fix_{parameter.name}"] = True - return kwargs - - @staticmethod - def set_minuit_parameters_limits_and_errors(m : Minuit,parameters : dict) : - """function to set minuit parameter limits and errors with Minuit >2.0 - - Args: - m (Minuit): a Minuit instance - parameters (dict): dict containing parameters names, limits errors and values - """ - for name in parameters["names"] : - m.limits[name] = parameters[f"limit_{name}"] - m.errors[name] = parameters[f"error_{name}"] - if parameters.get(f"fix_{name}",False) : - m.fixed[name] = True - - -# Usefull fucntions for the fit -def gaussian(x, mu, sig): - #return (1./(sig*np.sqrt(2*math.pi)))*np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.))) - return norm.pdf(x,loc = mu,scale = sig) - -def weight_gaussian(x,N, mu, sig) : - return N * gaussian(x, mu, sig) - -def doubleGauss(x,sig1,mu2,sig2,p): - return p *2 *gaussian(x, 0, sig1) + (1-p) * gaussian(x, mu2, sig2) - -def PMax(r): - """p_{max} in equation 6 in Caroff et al. (2019) - - Args: - r (float): SPE resolution - - Returns: - float : p_{max} - """ - if r > np.sqrt((np.pi -2 )/ 2) : - pmax = np.pi/(2 * (r**2 + 1)) - else : - pmax = np.pi*r**2/(np.pi*r**2 + np.pi - 2*r**2 - 2) - return pmax - -def ax(p,res): - """a in equation 4 in Caroff et al. (2019) - - Args: - p (float): proportion of the low charge component (2 gaussians model) - res (float): SPE resolution - - Returns: - float : a - """ - return ((2/np.pi)*p**2-p/(res**2+1)) - -def bx(p,mu2): - """b in equation 4 in Caroff et al. (2019) - - Args: - p (float): proportion of the low charge component (2 gaussians model) - mu2 (float): position of the high charge Gaussian - - Returns: - float : b - """ - return (np.sqrt(2/np.pi)*2*p*(1-p)*mu2) - -def cx(sig2,mu2,res,p): - """c in equation 4 in Caroff et al. (2019) - Note : There is a typo in the article 1-p**2 -> (1-p)**2 - - Args: - sig2 (float): width of the high charge Gaussian - mu2 (float): position of the high charge Gaussian - res (float): SPE resolution - p (float): proportion of the low charge component (2 gaussians model) - - Returns: - float : c - """ - return (1-p)**2*mu2**2 - (1-p)*(sig2**2+mu2**2)/(res**2+1) - -def delta(p,res,sig2,mu2): - """well known delta in 2nd order polynom - - Args: - p (_type_): _description_ - res (_type_): _description_ - sig2 (_type_): _description_ - mu2 (_type_): _description_ - - Returns: - float : b**2 - 4*a*c - """ - return bx(p,mu2)*bx(p,mu2) - 4*ax(p,res)*cx(sig2,mu2,res,p) - -def ParamU(p,r): - """d in equation 6 in Caroff et al. (2019) - - Args: - p (float): proportion of the low charge component (2 gaussians model) - r (float): SPE resolution - - Returns: - float : d - """ - return ((8*(1-p)**2*p**2)/np.pi - 4*(2*p**2/np.pi - p/(r**2+1))*((1-p)**2-(1-p)/(r**2+1))) - -def ParamS(p,r): - """e in equation 6 in Caroff et al. (2019) - - Args: - p (float): proportion of the low charge component (2 gaussians model) - r (float): SPE resolution - - Returns: - float : e - """ - e = (4*(2*p**2/np.pi - p/(r**2+1))*(1-p))/(r**2+1) - return e - -def SigMin(p,res,mu2): - """sigma_{high,min} in equation 6 in Caroff et al. (2019) - - Args: - p (float): proportion of the low charge component (2 gaussians model) - res (float): SPE resolution - mu2 (float): position of the high charge Gaussian - - Returns: - float : sigma_{high,min} - """ - return mu2*np.sqrt((-ParamU(p,res)+(bx(p,mu2)**2/mu2**2))/(ParamS(p,res))) - -def SigMax(p,res,mu2): - """sigma_{high,min} in equation 6 in Caroff et al. (2019) - - Args: - p (float): proportion of the low charge component (2 gaussians model) - res (float): SPE resolution - mu2 (float): position of the high charge Gaussian - - Returns: - float : sigma_{high,min} - """ - temp = (-ParamU(p,res))/(ParamS(p,res)) - if temp < 0 : - err = ValueError("-d/e must be < 0") - log.error(err,exc_info=True) - raise err - else : - return mu2*np.sqrt(temp) - -def sigma1(p,res,sig2,mu2): - """sigma_{low} in equation 5 in Caroff et al. (2019) - - Args: - sig2 (float): width of the high charge Gaussian - mu2 (float): position of the high charge Gaussian - res (float): SPE resolution - p (float): proportion of the low charge component (2 gaussians model) - - Returns: - float : sigma_{low} - """ - return (-bx(p,mu2)+np.sqrt(delta(p,res,sig2,mu2)))/(2*ax(p,res)) - -def sigma2(n,p,res,mu2): - """sigma_{high} in equation 7 in Caroff et al. (2019) - - Args: - n (float): parameter n in equation - mu2 (float): position of the high charge Gaussian - res (float): SPE resolution - p (float): proportion of the low charge component (2 gaussians model) - - Returns: - float : sigma_{high} - """ - if ((-ParamU(p,res)+(bx(p,mu2)**2/mu2**2))/(ParamS(p,res)) > 0): - return SigMin(p,res,mu2)+n*(SigMax(p,res,mu2)-SigMin(p,res,mu2)) - else: - return n*SigMax(p,res,mu2) - - # The real final model callign all the above for luminosity (lum) + PED, wil return probability of number of Spe -def MPE2(x,pp,res,mu2,n,muped,sigped,lum,**kwargs): - log.debug(f"pp = {pp}, res = {res}, mu2 = {mu2}, n = {n}, muped = {muped}, sigped = {sigped}, lum = {lum}") - f = 0 - ntotalPE = kwargs.get("ntotalPE",0) - if ntotalPE == 0 : - #about 1sec - for i in range(1000): - if (gammainc(i+1,lum) < 1e-5): - ntotalPE = i - break - #print(ntotalPE) - #about 8 sec, 1 sec by nPEPDF call - #for i in range(ntotalPE): - # f = f + ((lum**i)/math.factorial(i)) * np.exp(-lum) * nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) - - f = np.sum([(lum**i)/math.factorial(i) * np.exp(-lum) * nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) for i in range(ntotalPE)],axis = 0) # 10 % faster - return f - -# Fnal model shape/function (for one SPE) -def doubleGaussConstrained(x,pp,res,mu2,n): - p = pp*PMax(res) - sig2 = sigma2(n,p,res,mu2) - sig1 = sigma1(p,res,sig2,mu2) - return doubleGauss(x,sig1,mu2,sig2,p) - -# Get the gain from the parameters model -def Gain(pp,res,mu2,n): - """analytic gain computatuon - - Args: - mu2 (float): position of the high charge Gaussian - res (float): SPE resolution - pp (float): p' in equation 7 in Caroff et al. (2019) - n (float): n in equation 7 in Caroff et al. (2019) - - Returns: - float : gain - """ - p = pp*PMax(res) - sig2 = sigma2(n,p,res,mu2) - return (1-p)*mu2 + 2*p*sigma1(p,res,sig2,mu2)/np.sqrt(2*np.pi) - -def nPEPDF(x,pp,res,mu2,n,muped,sigped,nph,size_charge): - allrange = np.linspace(-1 * size_charge,size_charge,size_charge*2) - spe = [] - #about 2 sec this is the main pb - #for i in range(len(allrange)): - # if (allrange[i]>=0): - # spe.append(doubleGaussConstrained(allrange[i],pp,res,mu2,n)) - # else: - # spe.append(0) - - spe = doubleGaussConstrained(allrange,pp,res,mu2,n) * (allrange>=0 * np.ones(allrange.shape)) #100 times faster - - # ~ plt.plot(allrange,spe) - #npe = semi_gaussian(allrange, muped, sigped) - npe = gaussian(allrange, 0, sigped) - # ~ plt.plot(allrange,npe) - # ~ plt.show() - for i in range(nph): - #npe = np.convolve(npe,spe,"same") - npe = signal.fftconvolve(npe,spe,"same") - # ~ plt.plot(allrange,npe) - # ~ plt.show() - fff = interpolate.UnivariateSpline(allrange,npe,ext=1,k=3,s=0) - norm = np.trapz(fff(allrange),allrange) - return fff(x-muped)/norm \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/gain/utils/utils.py b/src/nectarchain/calibration/makers/gain/utils/utils.py index d1016a12..54825935 100644 --- a/src/nectarchain/calibration/makers/gain/utils/utils.py +++ b/src/nectarchain/calibration/makers/gain/utils/utils.py @@ -11,7 +11,7 @@ log = logging.getLogger(__name__) log.handlers = logging.getLogger('__main__').handlers -__all__ = ['UtilsMinuit','Multiprocessing'] +#__all__ = ['UtilsMinuit','Multiprocessing'] from ..parameters import Parameters diff --git a/src/nectarchain/calibration/makers/pedestalMakers.py b/src/nectarchain/calibration/makers/pedestalMakers.py index b99d90fd..5936d76c 100644 --- a/src/nectarchain/calibration/makers/pedestalMakers.py +++ b/src/nectarchain/calibration/makers/pedestalMakers.py @@ -1,4 +1,3 @@ -import sys import logging logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') log = logging.getLogger(__name__) @@ -9,8 +8,8 @@ __all__ = ["PedestalMaker"] class PedestalMaker(CalibrationMaker) : - def __init__(self,**kwargs) : - super().__init__(**kwargs) + def __init__(self,*args,**kwargs) -> None: + super().__init__(*args,**kwargs) def make(self) : raise NotImplementedError("The computation of the pedestal calibration is not yet implemented, feel free to contribute !:)") From c0e93587e120d19bda44468565d1c6be8c4dacd3 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Tue, 12 Sep 2023 00:05:31 +0200 Subject: [PATCH 31/62] remove old SPE fit implementation --- .../NectarGain/PhotoStat/PhotoStat.py | 281 ---- .../NectarGain/PhotoStat/__init__.py | 1 - .../NectarGain/SPEfit/NectarGAIN_old.py | 418 ------ .../NectarGain/SPEfit/NectarGainSPE.py | 236 ---- .../SPEfit/NectarGainSPE_combined.py | 285 ---- .../SPEfit/NectarGainSPE_singlerun.py | 1170 ----------------- .../calibration/NectarGain/SPEfit/__init__.py | 2 - .../NectarGain/SPEfit/parameters.py | 104 -- .../NectarGain/SPEfit/parameters_ped.yaml | 7 - .../NectarGain/SPEfit/parameters_signal.yaml | 32 - .../SPEfit/parameters_signalStd.yaml | 32 - .../SPEfit/parameters_signal_combined.yaml | 42 - .../SPEfit/parameters_signal_fromHHVFit.yaml | 32 - .../calibration/NectarGain/SPEfit/utils.py | 298 ----- .../calibration/NectarGain/__init__.py | 2 - .../calibration/NectarGain/utils/__init__.py | 1 - .../calibration/NectarGain/utils/error.py | 11 - 17 files changed, 2954 deletions(-) delete mode 100644 src/nectarchain/calibration/NectarGain/PhotoStat/PhotoStat.py delete mode 100644 src/nectarchain/calibration/NectarGain/PhotoStat/__init__.py delete mode 100644 src/nectarchain/calibration/NectarGain/SPEfit/NectarGAIN_old.py delete mode 100644 src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE.py delete mode 100644 src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_combined.py delete mode 100644 src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py delete mode 100644 src/nectarchain/calibration/NectarGain/SPEfit/__init__.py delete mode 100644 src/nectarchain/calibration/NectarGain/SPEfit/parameters.py delete mode 100644 src/nectarchain/calibration/NectarGain/SPEfit/parameters_ped.yaml delete mode 100644 src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal.yaml delete mode 100644 src/nectarchain/calibration/NectarGain/SPEfit/parameters_signalStd.yaml delete mode 100644 src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_combined.yaml delete mode 100644 src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_fromHHVFit.yaml delete mode 100644 src/nectarchain/calibration/NectarGain/SPEfit/utils.py delete mode 100644 src/nectarchain/calibration/NectarGain/__init__.py delete mode 100644 src/nectarchain/calibration/NectarGain/utils/__init__.py delete mode 100644 src/nectarchain/calibration/NectarGain/utils/error.py diff --git a/src/nectarchain/calibration/NectarGain/PhotoStat/PhotoStat.py b/src/nectarchain/calibration/NectarGain/PhotoStat/PhotoStat.py deleted file mode 100644 index 2dae67d5..00000000 --- a/src/nectarchain/calibration/NectarGain/PhotoStat/PhotoStat.py +++ /dev/null @@ -1,281 +0,0 @@ -import numpy as np -from scipy.stats import linregress -from matplotlib import pyplot as plt -import astropy.units as u -from astropy.visualization import quantity_support -from astropy.table import QTable,Column -import os -from datetime import date -from pathlib import Path - -import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') -log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers - -from ctapipe_io_nectarcam import constants - -from ...container import ChargeContainer,WaveformsContainer - -from ..utils.error import DifferentPixelsID - -from abc import ABC - -__all__ = ["PhotoStatGainFFandPed"] - -class PhotoStatGain(ABC): - - def _readFF(self,FFRun,maxevents: int = None,**kwargs) : - log.info('reading FF data') - method = kwargs.get('method','FullWaveformSum') - FFchargeExtractorWindowLength = kwargs.get('FFchargeExtractorWindowLength',None) - if method != 'FullWaveformSum' : - if FFchargeExtractorWindowLength is None : - e = Exception(f"we have to specify FFchargeExtractorWindowLength argument if charge extractor method is not FullwaveformSum") - log.error(e,exc_info=True) - raise e - else : - self.__coefCharge_FF_Ped = FFchargeExtractorWindowLength / constants.N_SAMPLES - else : - self.__coefCharge_FF_Ped = 1 - if isinstance(FFRun,int) : - try : - self.FFcharge = ChargeContainer.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{method}",FFRun) - log.info(f'charges have ever been computed for FF run {FFRun}') - except Exception as e : - log.error("charge have not been yet computed") - raise e - - #log.info(f'loading waveforms for FF run {FFRun}') - #FFwaveforms = WaveformsContainer(FFRun,maxevents) - #FFwaveforms.load_wfs() - #if method != 'std' : - # log.info(f'computing charge for FF run {FFRun} with following method : {method}') - # self.FFcharge = ChargeContainer.from_waveforms(FFwaveforms,method = method) - #else : - # log.info(f'computing charge for FF run {FFRun} with std method') - # self.FFcharge = ChargeContainer.from_waveforms(FFwaveforms) - #log.debug('writting on disk charge for further works') - #os.makedirs(f"{os.environ['NECTARCAMDATA']}/charges/{method}",exist_ok = True) - #self.FFcharge.write(f"{os.environ['NECTARCAMDATA']}/charges/{method}",overwrite = True) - - elif isinstance(FFRun,ChargeContainer): - self.FFcharge = FFRun - else : - e = TypeError("FFRun must be int or ChargeContainer") - log.error(e,exc_info = True) - raise e - - def _readPed(self,PedRun,maxevents: int = None,**kwargs) : - log.info('reading Ped data') - method = 'FullWaveformSum'#kwargs.get('method','std') - if isinstance(PedRun,int) : - try : - self.Pedcharge = ChargeContainer.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{method}",PedRun) - log.info(f'charges have ever been computed for Ped run {PedRun}') - except Exception as e : - log.error("charge have not been yet computed") - raise e - - #log.info(f'loading waveforms for Ped run {PedRun}') - #Pedwaveforms = WaveformsContainer(PedRun,maxevents) - #Pedwaveforms.load_wfs() - #if method != 'std' : - # log.info(f'computing charge for Ped run {PedRun} with following method : {method}') - # self.Pedcharge = ChargeContainer.from_waveforms(Pedwaveforms,method = method) - #else : - # log.info(f'computing charge for Ped run {PedRun} with std method') - # self.Pedcharge = ChargeContainer.from_waveforms(Pedwaveforms) - #log.debug('writting on disk charge for further works') - #os.makedirs(f"{os.environ['NECTARCAMDATA']}/charges/{method}",exist_ok = True) - #self.Pedcharge.write(f"{os.environ['NECTARCAMDATA']}/charges/{method}",overwrite = True) - - elif isinstance(PedRun,ChargeContainer): - self.Pedcharge = PedRun - else : - e = TypeError("PedRun must be int or ChargeContainer") - log.error(e,exc_info = True) - raise e - - def _readSPE(self,SPEresults) : - log.info(f'reading SPE resolution from {SPEresults}') - table = QTable.read(SPEresults) - table.sort('pixel') - self.SPEResolution = table['resolution'] - self.SPEGain = table['gain'] - self.SPEGain_error = table['gain_error'] - self._SPEvalid = table['is_valid'] - self._SPE_pixels_id = np.array(table['pixel'].value,dtype = np.uint16) - - def _reshape_all(self) : - log.info("reshape of SPE, Ped and FF data with intersection of pixel ids") - FFped_intersection = np.intersect1d(self.Pedcharge.pixels_id,self.FFcharge.pixels_id) - SPEFFPed_intersection = np.intersect1d(FFped_intersection,self._SPE_pixels_id[self._SPEvalid]) - self._pixels_id = SPEFFPed_intersection - log.info(f"data have {len(self._pixels_id)} pixels in common") - - self._FFcharge_hg = self.FFcharge.select_charge_hg(SPEFFPed_intersection) - self._FFcharge_lg = self.FFcharge.select_charge_lg(SPEFFPed_intersection) - - self._Pedcharge_hg = self.Pedcharge.select_charge_hg(SPEFFPed_intersection) - self._Pedcharge_lg = self.Pedcharge.select_charge_lg(SPEFFPed_intersection) - - #self._mask_FF = np.array([self.FFcharge.pixels_id[i] in SPEFFPed_intersection for i in range(self.FFcharge.npixels)],dtype = bool) - #self._mask_Ped = np.array([self.Pedcharge.pixels_id[i] in SPEFFPed_intersection for i in range(self.Pedcharge.npixels)],dtype = bool) - self._mask_SPE = np.array([self._SPE_pixels_id[i] in SPEFFPed_intersection for i in range(len(self._SPE_pixels_id))],dtype = bool) - - - - def create_output_table(self) : - self._output_table = QTable() - self._output_table.meta['npixel'] = self.npixels - self._output_table.meta['comments'] = f'Produced with NectarGain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' - - self._output_table.add_column(Column(np.ones((self.npixels),dtype = bool),"is_valid",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(self.pixels_id,"pixel",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),"high gain",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"high gain error",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),"low gain",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"low gain error",unit = u.dimensionless_unscaled)) - - def run(self,**kwargs): - log.info('running photo statistic method') - - self._output_table["high gain"] = self.gainHG - self._output_table["low gain"] = self.gainLG - #self._output_table["is_valid"] = self._SPEvalid - - def save(self,path,**kwargs) : - path = Path(path) - os.makedirs(path,exist_ok = True) - log.info(f'data saved in {path}') - self._output_table.write(f"{path}/output_table.ecsv", format='ascii.ecsv',overwrite = kwargs.get("overwrite",False)) - - def plot_correlation(self) : - mask = (self._output_table["high gain"]>20) * (self.SPEGain[self._mask_SPE]>0) * (self._output_table["high gain"]<80) * self._output_table['is_valid'] - a, b, r, p_value, std_err = linregress(self._output_table["high gain"][mask], self.SPEGain[self._mask_SPE][mask],'greater') - x = np.linspace(self._output_table["high gain"][mask].min(),self._output_table["high gain"][mask].max(),1000) - y = lambda x: a * x + b - with quantity_support() : - fig,ax = plt.subplots(1,1,figsize=(8, 6)) - ax.scatter(self._output_table["high gain"][mask],self.SPEGain[self._mask_SPE][mask],marker =".") - ax.plot(x,y(x),color = 'red', label = f"linear fit,\n a = {a:.2e},\n b = {b:.2e},\n r = {r:.2e},\n p_value = {p_value:.2e},\n std_err = {std_err:.2e}") - ax.plot(x,x,color = 'black',label = "y = x") - ax.set_xlabel("Gain Photo stat (ADC)", size=15) - ax.set_ylabel("Gain SPE fit (ADC)", size=15) - #ax.set_xlim(xmin = 0) - #ax.set_ylim(ymin = 0) - - ax.legend(fontsize=15) - return fig - - @property - def npixels(self) : return self._pixels_id.shape[0] - - @property - def pixels_id(self) : return self._pixels_id - - @property - def sigmaPedHG(self) : return np.std(self._Pedcharge_hg ,axis = 0) * np.sqrt(self.__coefCharge_FF_Ped) - - @property - def sigmaChargeHG(self) : return np.std(self._FFcharge_hg - self.meanPedHG, axis = 0) - - @property - def meanPedHG(self) : return np.mean(self._Pedcharge_hg ,axis = 0) * self.__coefCharge_FF_Ped - - @property - def meanChargeHG(self) : return np.mean(self._FFcharge_hg - self.meanPedHG, axis = 0) - - @property - def BHG(self) : - min_events = np.min((self._FFcharge_hg.shape[0],self._Pedcharge_hg.shape[0])) - upper = (np.power(self._FFcharge_hg.mean(axis = 1)[:min_events] - self._Pedcharge_hg.mean(axis = 1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeHG.mean(),2)).mean(axis = 0) - lower = np.power(self.meanChargeHG.mean(),2)#np.power(self.meanChargeHG,2)#np.power(self.meanChargeHG.mean(),2) - return np.sqrt(upper/lower) - - @property - def gainHG(self) : - return ((np.power(self.sigmaChargeHG,2) - np.power(self.sigmaPedHG,2) - np.power(self.BHG * self.meanChargeHG,2)) - /(self.meanChargeHG * (1 + np.power(self.SPEResolution[self._mask_SPE],2)))) - - - @property - def sigmaPedLG(self) : return np.std(self._Pedcharge_lg ,axis = 0) * np.sqrt(self.__coefCharge_FF_Ped) - - @property - def sigmaChargeLG(self) : return np.std(self._FFcharge_lg - self.meanPedLG,axis = 0) - - @property - def meanPedLG(self) : return np.mean(self._Pedcharge_lg,axis = 0) * self.__coefCharge_FF_Ped - - @property - def meanChargeLG(self) : return np.mean(self._FFcharge_lg - self.meanPedLG,axis = 0) - - @property - def BLG(self) : - min_events = np.min((self._FFcharge_lg.shape[0],self._Pedcharge_lg.shape[0])) - upper = (np.power(self._FFcharge_lg.mean(axis = 1)[:min_events] - self._Pedcharge_lg.mean(axis = 1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeLG.mean(),2)).mean(axis = 0) - lower = np.power(self.meanChargeLG.mean(),2) #np.power(self.meanChargeLG,2) #np.power(self.meanChargeLG.mean(),2) - return np.sqrt(upper/lower) - - @property - def gainLG(self) : return ((np.power(self.sigmaChargeLG,2) - np.power(self.sigmaPedLG,2) - np.power(self.BLG * self.meanChargeLG,2)) - /(self.meanChargeLG * (1 + np.power(self.SPEResolution[self._mask_SPE],2)))) - - - -class PhotoStatGainFFandPed(PhotoStatGain): - def __init__(self, FFRun, PedRun, SPEresults : str, maxevents : int = None, **kwargs) : - self._readFF(FFRun,maxevents,**kwargs) - self._readPed(PedRun,maxevents,**kwargs) - - """ - if self.FFcharge.charge_hg.shape[1] != self.Pedcharge.charge_hg.shape[1] : - e = Exception("Ped run and FF run must have the same number of pixels") - log.error(e,exc_info = True) - raise e - """ - - self._readSPE(SPEresults) - ##need to implement reshape of SPE results with FF and Ped pixels ids - self._reshape_all() - - """ - if (self.FFcharge.pixels_id.shape[0] != self._SPE_pixels_id.shape[0]) : - e = Exception("Ped run and FF run must have the same number of pixels as SPE fit results") - log.error(e,exc_info = True) - raise e - - if (self.FFcharge.pixels_id != self.Pedcharge.pixels_id).any() or (self.FFcharge.pixels_id != self._SPE_pixels_id).any() or (self.Pedcharge.pixels_id != self._SPE_pixels_id).any() : - e = DifferentPixelsID("Ped run, FF run and SPE run need to have same pixels id") - log.error(e,exc_info = True) - raise e - else : - self._pixels_id = self.FFcharge.pixels_id - """ - self.create_output_table() - - - -class PhotoStatGainFF(PhotoStatGain): - def __init__(self, FFRun, PedRun, SPEresults : str, maxevents : int = None, **kwargs) : - e = NotImplementedError("PhotoStatGainFF is not yet implemented") - log.error(e, exc_info = True) - raise e - self._readFF(FFRun,maxevents,**kwargs) - - self._readSPE(SPEresults) - - """ - if self.FFcharge.pixels_id != self._SPE_pixels_id : - e = DifferentPixelsID("Ped run, FF run and SPE run need to have same pixels id") - log.error(e,exc_info = True) - raise e - else : - self._pixels_id = self.FFcharge.pixels_id - """ - - self._reshape_all() - - self.create_output_table() \ No newline at end of file diff --git a/src/nectarchain/calibration/NectarGain/PhotoStat/__init__.py b/src/nectarchain/calibration/NectarGain/PhotoStat/__init__.py deleted file mode 100644 index 4edb081c..00000000 --- a/src/nectarchain/calibration/NectarGain/PhotoStat/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .PhotoStat import * \ No newline at end of file diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGAIN_old.py b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGAIN_old.py deleted file mode 100644 index 1c8fca26..00000000 --- a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGAIN_old.py +++ /dev/null @@ -1,418 +0,0 @@ -import math -import numpy as np -from scipy import optimize, interpolate -from matplotlib import pyplot as plt -from scipy import signal -from scipy.special import gammainc -from iminuit import Minuit -import random - - - - -# Gain's class -class NectarSPEGain(): - - def __init__(self): - self.histoPed = [] - self.chargePed = [] - self.histoSPE = [] - self.chargeSPE = [] - self.histoPedHHV = [] - self.chargePedHHV = [] - self.histoSPEHHV = [] - self.chargeSPEHHV = [] - self.dataType = "" - ###### parameters model ######### - self.pedestalMean = 0 - self.pedestalWidth = 0 - self.pedestalMeanSPE = 0 - self.pedestalWidthSPE = 0 - self.Luminosity = 0 - self.pp = 0 - self.resolution = 0 - self.meanUp = 0 - self.n = 0 - self.pedestalMeanHHV = 0 - self.pedestalWidthHHV = 0 - self.pedestalMeanSPEHHV = 0 - self.pedestalWidthSPEHHV = 0 - self.LuminosityHHV = 0 - self.ppHHV = 0 - self.resolutionHHV = 0 - self.meanUpHHV = 0 - self.nHHV = 0 - self.gain = 0 - - self.pedestalMeanUp = 0 - self.pedestalWidthUp = 0 - self.pedestalMeanSPEUp = 0 - self.pedestalWidthSPEUp = 0 - self.LuminosityUp = 0 - self.ppUp = 0 - self.resolutionUp = 0 - self.meanUpUp = 0 - self.nUp = 0 - self.pedestalMeanHHVUp = 0 - self.pedestalWidthHHVUp = 0 - self.pedestalMeanSPEHHVUp = 0 - self.pedestalWidthSPEHHVUp = 0 - self.LuminosityHHVUp = 0 - self.ppHHVUp = 0 - self.resolutionHHVUp = 0 - self.meanUpHHVUp = 0 - self.nHHVUp = 0 - self.gainUp = 0 - - self.pedestalMeanLow = 0 - self.pedestalWidthLow = 0 - self.pedestalMeanSPELow = 0 - self.pedestalWidthSPELow = 0 - self.LuminosityLow = 0 - self.ppLow = 0 - self.resolutionLow = 0 - self.meanUpLow = 0 - self.nLow = 0 - self.pedestalMeanHHVLow = 0 - self.pedestalWidthHHVLow = 0 - self.pedestalMeanSPEHHVLow = 0 - self.pedestalWidthSPEHHVLow = 0 - self.LuminosityHHVLow = 0 - self.ppHHVLow = 0 - self.resolutionHHVLow = 0 - self.meanUpHHVLow = 0 - self.nHHVLow = 0 - self.gainLow = 0 - - ####### data ####### - -# Fill histos as given by the user - def FillDataHisto(self,chargeSPE,dataSPE,chargePed = 0,dataPed = 0,chargePedHHV=0,dataPedHHV=0,chargeSPEHHV=0,dataSPEHHV=0): - self.histoPed = dataPed - self.chargePed = chargePed - self.histoSignal = dataSPE - self.chargeSignal = chargeSPE - self.histoPedHHV = dataPedHHV - self.chargePedHHV = chargePedHHV - self.histoSignalHHV = dataSPEHHV - self.chargeSignalHHV = chargeSPEHHV - self.dataType = "histo" - - ####### functions model ####### - -# Usefull fucntions for the fit - def gaussian(self,x, mu, sig): - return (1./(sig*np.sqrt(2*math.pi)))*np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.))) - - def doubleGauss(self,x,sig1,mu2,sig2,p): - return p *2 *self.gaussian(x, 0, sig1) + (1-p) * self.gaussian(x, mu2, sig2) - - def PMax(self,r): - if (np.pi*r**2/(np.pi*r**2 + np.pi - 2*r**2 - 2) <= 1): - return np.pi*r**2/(np.pi*r**2 + np.pi - 2*r**2 - 2) - else: - return 1 - - def ax(self,p,res): - return ((2/np.pi)*p**2-p/(res**2+1)) - - def bx(self,p,mu2): - return (np.sqrt(2/np.pi)*2*p*(1-p)*mu2) - - def cx(self,sig2,mu2,res,p): - return (1-p)**2*mu2**2 - (1-p)*(sig2**2+mu2**2)/(res**2+1) - - def delta(self,p,res,sig2,mu2): - return self.bx(p,mu2)*self.bx(p,mu2) - 4*self.ax(p,res)*self.cx(sig2,mu2,res,p) - - def ParamU(self,p,r): - return ((8*(1-p)**2*p**2)/np.pi - 4*(2*p**2/np.pi - p/(r**2+1))*((1-p)**2-(1-p)/(r**2+1))) - - def ParamS(self,p,r): - return (4*(2*p**2/np.pi - p/(r**2+1))*(1-p))/(r**2+1) - - def SigMin(self,p,res,mu2): - return mu2*np.sqrt((-self.ParamU(p,res)+(self.bx(p,mu2)**2/mu2**2))/(self.ParamS(p,res))) - - def SigMax(self,p,res,mu2): - return mu2*np.sqrt((-self.ParamU(p,res))/(self.ParamS(p,res))) - - def sigma1(self,p,res,sig2,mu2): - return (-self.bx(p,mu2)+np.sqrt(self.delta(p,res,sig2,mu2)))/(2*self.ax(p,res)) - - def sigma2(self,n,p,res,mu2): - if ((-self.ParamU(p,res)+(self.bx(p,mu2)**2/mu2**2))/(self.ParamS(p,res)) > 0): - return self.SigMin(p,res,mu2)+n*(self.SigMax(p,res,mu2)-self.SigMin(p,res,mu2)) - else: - return n*self.SigMax(p,res,mu2) - -# Final model shape/function (for one SPE) - def doubleGaussConstrained(self,x,pp,res,mu2,n): - p = pp*self.PMax(res) - sig2 = self.sigma2(n,p,res,mu2) - sig1 = self.sigma1(p,res,sig2,mu2) - return self.doubleGauss(x,sig1,mu2,sig2,p) - -# Get the gain from the parameters model - def Gain(self,pp,res,mu2,n): - p = pp*self.PMax(res) - sig2 = self.sigma2(n,p,res,mu2) - return (1-p)*mu2 + 2*p*self.sigma1(p,res,sig2,mu2)/np.sqrt(2*np.pi) - - #def nPEPDF(x,pp,res,mu2,n,muped,sigped,nph): - # allrange = np.linspace(-1000,1000,2000) - # spe = doubleGaussConstrained(allrange,pp,res,mu2,n) - # ped = gaussian(allrange, muped, sigped) - # for i in range(nph): - # npe = np.convolve(spe,ppp,"same") - -# The PDF model for one/N SPE convoluted with the pedestal (0 SPE --> pedestal, 1 SPE --> 1 SPExPed, etc) - def nPEPDF(self,x,pp,res,mu2,n,muped,sigped,nph,size_charge): - allrange = np.linspace(-1 * size_charge,size_charge,size_charge*2) - spe = [] - for i in range(len(allrange)): - if (allrange[i]>=0): - spe.append(self.doubleGaussConstrained(allrange[i],pp,res,mu2,n)) - else: - spe.append(0) - # ~ plt.plot(allrange,spe) - #npe = semi_gaussian(allrange, muped, sigped) - npe = self.gaussian(allrange, 0, sigped) - # ~ plt.plot(allrange,npe) - # ~ plt.show() - for i in range(nph): - #npe = np.convolve(npe,spe,"same") - npe = signal.fftconvolve(npe,spe,"same") - # ~ plt.plot(allrange,npe) - # ~ plt.show() - fff = interpolate.UnivariateSpline(allrange,npe,ext=1,k=3,s=0) - norm = np.trapz(fff(allrange),allrange) - return fff(x-muped)/norm - -# The real final model callign all the above for luminosity (lum) + PED, wil return probability of number of Spe - def MPE2(self,x,pp,res,mu2,n,muped,sigped,lum): - f = 0 - ntotalPE = 0 - for i in range(1000): - if (gammainc(i+1,lum) < 1e-5): - ntotalPE = i - break - #print(ntotalPE) - for i in range(ntotalPE): - f = f + ((lum**i)/math.factorial(i)) * np.exp(-lum) * self.nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) - return f - - ####### Likelihood ######## - -# Not tested - def NG_LikelihoodPedestal_Unbinned(self,mean,sigma,charge): - Lik = 0 - for i in range(len(events)): - Lik = Lik-2.*math.log(self.gaussian(charge[i],mean,sigma)) - return Lik - - -# Not tested - def NG_LikelihoodSignal_Unbinned(self,pp,res,mu2,n,muped,sigped,lum,charge,nPrecision): - MaxCharge = np.maximum(charge)+1 - MinCharge = np.minimum(charge)-1 - ChargeTable = np.linspace(MinCharge,MaxCharge,nPrecision) - pdf = self.MPE2(ChargeTable,pp,res,mu2,n,muped,sigped,lum) - pdf_interpolated = interpolate.UnivariateSpline(ChargeTable,npe,ext=1,k=3,s=0) - Lik = 0 - for i in range(len(events)): - Lik = Lik-2*math.log(pdf_interpolated(charge[i])) - return Lik - -# Chi2 (used now) for the pedestal (for pure pedestal data) - def NG_LikelihoodPedestal_Chi2(self,mean,sigma,charge,nEvents): - Lik = 0 - Ntot = np.sum(nEvents) - for i in range(len(nEvents)): - if (nEvents[i] > 0): - Lik = Lik + ((self.gaussian(charge[i],mean,sigma)*Ntot - nEvents[i])**2)/nEvents[i] - return Lik - - -# Chi2 (used now) for signal ie luminosity (SPE) - def NG_LikelihoodSignal_Chi2(self,pp,res,mu2,n,muped,sigped,lum,charge,nEvents): - pdf = self.MPE2(charge,pp,res,mu2,n,muped,sigped,lum) - Ntot = np.sum(nEvents) - Lik = 0 - for i in range(len(nEvents)): - if (nEvents[i] > 0): - Lik = Lik + (pdf[i]*Ntot-nEvents[i])**2/nEvents[i] - return Lik - -# To call NG_LikelihoodSignal_Chi2 wit hthe data loaded, will all free paramters of the model (work only at 1000V) but not recommened (very complex to converg) better to use the fix mdoel (below) -# Ideal/final for 1000V - def Chi2Signal(self,pp,res,mu2,n,muped,sigped,lum): - return self.NG_LikelihoodSignal_Chi2(pp,res,mu2,n,muped,sigped,lum,self.chargeSignal,self.histoSignal) - -# To call NG_LikelihoodSignal_Chi2 wit hthe data loaded, not all free parameters (pp and n are fixed (as they are independent from the gain, should be the same for all pixels althogh unchecked), will be usefull for the combined fit with the 1000V -# Paramters fixed after Sami did a avergae on 50 pixels just have an idea and gain time - def Chi2SignalFixedModel(self,res,mu2,muped,sigped,lum): - return self.NG_LikelihoodSignal_Chi2(self.pp,res,mu2,self.n,muped,sigped,lum,self.chargeSignal,self.histoSignal) - -# Chi2 (used now) for the ped at 1000V (alwasy work) - def Chi2Ped(self,muped,sigped): - return self.NG_LikelihoodPedestal_Chi2(muped,sigped,self.chargePed,self.histoPed) - -# Chi2 (used now) for the signal (HHV: high high voltage ie 1400V) at 1400V -# Ideal/final for 1400V - def Chi2SignalHHV(self,pp,res,mu2,n,muped,sigped,lum): - return self.NG_LikelihoodSignal_Chi2(pp,res,mu2,n,muped,sigped,lum,self.chargeSignalHHV,self.histoSignalHHV) - -# Chi2 (used now) for the ped at 1400V (always work) -# Ideal for file with only pedestal/noise (not for us now)) - def Chi2PedHHV(self,muped,sigped): - return self.NG_LikelihoodPedestal_Chi2(muped,sigped,self.chargePedHHV,self.histoPedHHV) - -# Chi2 (not tested yet) for the combined pedestal + signal (at 1000V) -# If pp and n are free it will not converge, requires a pedestal run like above, ideal for 1000V file + noise/pedestal file -# Not for now - def Chi2CombiSignalAndPed(self,pp,res,mu2,n,muped,sigped,lum): - return self.Chi2Signal(self.pp,res,mu2,self.n,muped,sigped,lum)+self.Chi2Ped(muped,sigped) - #return self.Chi2Signal(pp,res,mu2,n,muped,sigped,lum)+self.Chi2Ped(muped,sigped) - -# Chi2 (not tested yet) for the combined pedestal + signal (at 1400V) -# Requires a pedestal file so not for now - def Chi2CombiSignalAndPedHHV(self,pp,res,mu2,n,muped,sigped,lum): - return self.Chi2SignalHHV(pp,res,mu2,n,muped,sigped,lum)+self.Chi2PedHHV(muped,sigped) - -# Chi2 (not tested yet) for the combined pedestal + signal (1000V) + signal (1400V) -# Requires a pedestal file so not for now -# Ideal/final for ped + 1000V + 1400V - def Chi2AllCombined(self,pp,res,mu2,mu2HHV,n,muped,mupedHHV,sigped,lum,lumHHV): - return self.Chi2CombiSignalAndPed(pp,res,mu2,n,muped,sigped,lum)+self.Chi2CombiSignalAndPedHHV(pp,res,mu2HHV,n,mupedHHV,sigped,lum) - -# Chi2 (not tested yet) for the combined signal (1000V) + signal (1400V) -# Ideal/final for 1000V + 1400V - def Chi2AllNoPedCombined(self,pp,res,mu2,mu2HHV,n,muped,mupedHHV,sigped,lum,lumHHV): - return self.Chi2Signal(pp,res,mu2,n,muped,sigped,lum)+self.Chi2SignalHHV(pp,res,mu2HHV,n,mupedHHV,sigped,lum) - - ####### Compute Start Parameters ###### - -# "Smart" values to start the fit - def StartParameters(self): - self.pedestalMean = (np.min(self.chargePed) + np.sum(self.chargePed*self.histoPed)/np.sum(self.histoPed))/2. - self.pedestalMeanLow = np.min(self.chargePed) - self.pedestalMeanUp = np.sum(self.chargePed*self.histoPed)/np.sum(self.histoPed) - #self.pedestalWidth = np.sqrt(np.sum(self.chargePed**2 * self.histoPed)/np.sum(self.histoPed)-self.pedestalMean**2) - #self.pedestalWidth = 16 - self.pedestalWidth = 50 - #self.pedestalWidthLow = self.pedestalWidth-3 - self.pedestalWidthLow = 1 - self.pedestalWidthUp = self.pedestalWidth+100 - print("pedestal mean ",self.pedestalMean," width ", self.pedestalWidth) - self.pedestalMeanSPE = self.pedestalMean - self.pedestalMeanSPELow = self.pedestalMeanSPE-60 - self.pedestalMeanSPEUp =self.pedestalMeanSPE+60 - self.pedestalWidthSPE = self.pedestalWidth - self.pedestalWidthSPELow = self.pedestalWidthSPE-10 - self.pedestalWidthSPEUp = self.pedestalWidthSPE+10 - self.Luminosity = 1. - self.LuminosityLow = 0.01 - #self.LuminosityUp = 2. - self.LuminosityUp = 5. - self.pp = 0.3735 - self.resolution = 0.5 - self.resolutionLow = 0.3 - self.resolutionUp = 0.7 - #self.meanUp = 50. - self.meanUp = 500. - #self.meanUpLow = 20. - self.meanUpLow = 400. - #self.meanUpUp = 100. - self.meanUpUp = 600. - self.n = 0.708 - self.pedestalMeanHHV = self.pedestalMean - self.pedestalWidthHHV = self.pedestalWidth - self.pedestalMeanSPEHHV = self.pedestalMean - self.pedestalWidthSPEHHV = self.pedestalWidth - self.LuminosityHHV = 1. - self.ppHHV = 0.3735 - self.resolutionHHV = 0.5 - self.meanUpHHV = 300. - self.nHHV = 0.708 - - ####### Fit minuit ####### - -# To fit with iminuit for Signal only, create functions for all the cases above - - def fitSignalOnly(self,ID = 0): - self.StartParameters() - parName = ["res","mu2","muped","sigped","lum"] - parValues = [self.resolution,self.meanUp,self.pedestalMean,self.pedestalWidth,self.Luminosity] - LimitLow = [self.resolutionLow,self.meanUpLow,self.pedestalMeanLow,self.pedestalWidthLow,self.LuminosityLow] - LimitUp = [self.resolutionUp,self.meanUpUp,self.pedestalMeanUp,self.pedestalWidthUp,self.LuminosityUp] - - parameters = make_minuit_par_kwargs(parValues,parName,LimitLow,LimitUp) - m = Minuit(self.Chi2SignalFixedModel,**parameters['values']) - m.print_level = 2 - set_minuit_parameters_limits_and_errors(m,parameters) - m.strategy = 2 - print(m.values) - results = m.migrad(ncall=4000000) - m.hesse() - print(m.values) - print(m.errors) - print("Reconstructed gain is ", self.Gain(self.pp,m.values[0],m.values[1],self.n)) - gainGenerated = [] - for i in range(1000): - gainGenerated.append(self.Gain(self.pp,random.gauss(m.values[0], m.errors[0]),random.gauss(m.values[1], m.errors[1]),self.n)) - print("Uncertainty is ", np.std(gainGenerated)) - plt.figure(figsize=(8, 6)) - plt.errorbar(self.chargeSignal,self.histoSignal,np.sqrt(self.histoSignal),zorder=0,fmt=".",label = "data") - plt.plot(self.chargeSignal,np.trapz(self.histoSignal,self.chargeSignal)*self.MPE2(self.chargeSignal,self.pp,m.values[0],m.values[1],self.n,m.values[2],m.values[3],m.values[4]),zorder=1,linewidth=2,label = "MPE model fit \n gain = "+str(round(self.Gain(self.pp,m.values[0],m.values[1],self.n),2))+" +/- " + str(round(np.std(gainGenerated),2)) + " ADC/pe") - plt.xticks(size = 15) - plt.yticks(size = 15) - plt.xlabel("Charge (ADC)", size=15) - plt.ylabel("Events", size=15) - #plt.plot(self.chargeSignal,self.MPE2(self.chargeSignal,self.pp,m.values[0],m.values[1],self.n,m.values[2],m.values[3],m.values[4]),linewidth=2) - #print(np.trapz(self.MPE2(self.chargeSignal,self.pp,m.values[0],m.values[1],self.n,m.values[2],m.values[3],m.values[4]),self.chargeSignal)) - plt.legend(fontsize=15) - #plt.show() - return self.Gain(self.pp,m.values[0],m.values[1],self.n),np.std(gainGenerated),m.values,m.errors - - -# To fit with iminuit for Signal 1400V only with pp and n free paramters, create functions for all the cases above - def fitSignalOnly1400V(self): - self.StartParameters() - # need to adapt free parameters - parName = ["pp","res","mu2","n", "muped","sigped","lum"] - parValues = [self.pp,self.resolution,self.meanUp,self.n,self.pedestalMean,self.pedestalWidth,self.Luminosity] - LimitLow = [0.2,self.resolutionLow,self.meanUpLow,0.5,self.pedestalMeanLow,self.pedestalWidthLow,self.LuminosityLow] - LimitUp = [0.8,self.resolutionUp,self.meanUpUp,0.9,self.pedestalMeanUp,self.pedestalWidthUp,self.LuminosityUp] - - test = make_minuit_par_kwargs(parValues,parName,LimitLow,LimitUp) - # the critical lign (here using fixed model) - m = Minuit(self.Chi2Signal,**test, print_level=2) - m.get_initial_param_states() - m.set_strategy(2) - print(m.values) - results = m.migrad(ncall=4000000) - m.hesse() - print(m.values) - print(m.errors) - gain = self.Gain(m.values[0],m.values[1],m.values[2],m.values[3]) - print(f"Reconstructed gain is {gain}") - gainGenerated = [] - for i in range(1000): - gainGenerated.append(self.Gain(random.gauss(m.values[0],m.errors[0]),random.gauss(m.values[1],m.errors[1]),random.gauss(m.values[2],m.errors[2]),random.gauss(m.values[3],m.errors[3]))) - print(gainGenerated) - print("Uncertainty is ", np.std(gainGenerated)) - plt.figure(figsize=(8, 6)) - plt.errorbar(self.chargeSignal,self.histoSignal,np.sqrt(self.histoSignal),zorder=0,fmt=".",label = "data") - plt.plot(self.chargeSignal,np.trapz(self.histoSignal,self.chargeSignal)*self.MPE2(self.chargeSignal,m.values[0],m.values[1],m.values[2],m.values[3],m.values[4],m.values[5],m.values[6]),zorder=1,linewidth=2,label = "MPE model fit \n gain = "+str(round(gain,2))+" +/- " + str(round(np.std(gainGenerated),2)) + " ADC/pe") - plt.xticks(size = 15) - plt.yticks(size = 15) - plt.xlabel("Charge (ADC)", size=15) - plt.ylabel("Events", size=15) - #plt.plot(self.chargeSignal,self.MPE2(self.chargeSignal,self.pp,m.values[0],m.values[1],self.n,m.values[2],m.values[3],m.values[4]),linewidth=2) - #print(np.trapz(self.MPE2(self.chargeSignal,self.pp,m.values[0],m.values[1],self.n,m.values[2],m.values[3],m.values[4]),self.chargeSignal)) - plt.legend(fontsize=15) - #plt.show() - return gain,np.std(gainGenerated),m.values,m.errors - - diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE.py b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE.py deleted file mode 100644 index 4f7c747c..00000000 --- a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE.py +++ /dev/null @@ -1,236 +0,0 @@ -import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') -log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers - -import copy -import numpy as np -import os -import yaml -import matplotlib.pyplot as plt -from matplotlib.patches import Rectangle -from matplotlib.colors import to_rgba - -from scipy.signal import find_peaks -from scipy.signal import savgol_filter -from scipy.optimize import curve_fit - -from abc import ABC, abstractclassmethod, abstractmethod - - -from iminuit import Minuit -from astropy.table import QTable - -import pandas as pd - -from .parameters import Parameters -from .utils import UtilsMinuit,weight_gaussian,Statistics - -__all__ = ['NectarGainSPE'] - -class NectarGainSPE_new(ABC) : - _Ncall = 4000000 - _Windows_lenght = 40 - _Order = 2 - - def __init__(self) : - #set parameters value for fit - self.__parameters = Parameters() - #output - self._output_table = QTable() - #self.create_output_table() #need to be done in the child class __init__ - - - - - - - - -class NectarGainSPE(ABC) : - _Ncall = 4000000 - _Windows_lenght = 40 - _Order = 2 - - def __init__(self) : - #set parameters value for fit - self.__parameters = Parameters() - - #output - self._output_table = QTable() - #self.create_output_table() #need to be done in the child class __init__ - - def fill_table(self,pixel : int, valid : bool,ndof : int,likelihood : float) : - self._output_table['is_valid'][pixel] = valid - for parameter in self._parameters.parameters : - self._output_table[parameter.name][pixel] = parameter.value - self._output_table[f'{parameter.name}_error'][pixel] = parameter.error - if valid : - self._output_table['pvalue'][pixel] = Statistics.chi2_pvalue(ndof,likelihood) - self._output_table['likelihood'][pixel] = likelihood - else : - self._output_table['pvalue'][pixel] = 0 - self._output_table['likelihood'][pixel] = 0 - - - def make_table(self,dictionary): - self._output_table = QTable.from_pandas(pd.DataFrame.from_dict(dictionary)) - - def _make_minuit_parameters(self) : - if log.getEffectiveLevel() == logging.DEBUG: - for parameter in self._parameters.parameters : - log.debug(parameter) - #create minuit parameters - self.__minuitParameters = UtilsMinuit.make_minuit_par_kwargs(self.__parameters.unfrozen) - - #ONLY KEEP STATIC METHOD NOW - #def _update_parameters_postfit(self,m : Minuit) : - # for i,name in enumerate(m.parameters) : - # tmp = self.__parameters[name] - # if tmp != [] : - # tmp.value = m.values[i] - # tmp.error = m.errors[i] - - @staticmethod - def _update_parameters_postfit(m : Minuit,parameters : Parameters) : - for i,name in enumerate(m.parameters) : - tmp = parameters[name] - if tmp != [] : - tmp.value = m.values[i] - tmp.error = m.errors[i] - - @staticmethod - def _make_output_dict_obs(m : Minuit, valid, pixels_id, parameters : Parameters, ndof : int) : - __class__._update_parameters_postfit(m,parameters) - output = {"is_valid" : valid, "pixel" : pixels_id} - for parameter in parameters.parameters : - output[parameter.name] = parameter.value - output[f"{parameter.name}_error"] = parameter.error - - output['likelihood'] = m.fval - output['pvalue'] = Statistics.chi2_pvalue(ndof,m.fval) - return output - - def read_param_from_yaml(self,parameters_file) : - with open(f"{os.path.dirname(os.path.abspath(__file__))}/{parameters_file}") as parameters : - param = yaml.safe_load(parameters) - for i,name in enumerate(self.__parameters.parnames) : - dico = param.get(name,False) - if dico : - self._parameters.parameters[i].value = dico.get('value') - self._parameters.parameters[i].min = dico.get("min",np.nan) - self._parameters.parameters[i].max = dico.get("max",np.nan) - - @staticmethod - def _get_mean_gaussian_fit(charge_in, histo_in ,extension = ""): - charge = charge_in.data[~histo_in.mask] - histo = histo_in.data[~histo_in.mask] - - windows_lenght = NectarGainSPE._Windows_lenght - order = NectarGainSPE._Order - histo_smoothed = savgol_filter(histo, windows_lenght, order) - - peaks = find_peaks(histo_smoothed,10) - peak_max = np.argmax(histo_smoothed[peaks[0]]) - peak_pos,peak_value = charge[peaks[0][peak_max]], histo[peaks[0][peak_max]] - - coeff, var_matrix = curve_fit(weight_gaussian, charge[:peaks[0][peak_max]], histo_smoothed[:peaks[0][peak_max]],p0 = [peak_value,peak_pos,1]) - - #nosw find SPE peak excluding pedestal data - mask = charge > coeff[1]+3*coeff[2] - peaks_mean = find_peaks(histo_smoothed[mask]) - - peak_max_mean = np.argmax(histo_smoothed[mask][peaks_mean[0]]) - peak_pos_mean,peak_value_mean = charge[mask][peaks_mean[0][peak_max_mean]], histo_smoothed[mask][peaks_mean[0][peak_max_mean]] - - mask = (charge > ((coeff[1]+peak_pos_mean)/2)) * (charge < (peak_pos_mean + (peak_pos_mean-coeff[1])/2)) - coeff_mean, var_matrix = curve_fit(weight_gaussian, charge[mask], histo_smoothed[mask],p0 = [peak_value_mean,peak_pos_mean,1]) - - if log.getEffectiveLevel() == logging.DEBUG : - log.debug('plotting figures with prefit parameters computation') - fig,ax = plt.subplots(1,1,figsize = (5,5)) - ax.errorbar(charge,histo,np.sqrt(histo),zorder=0,fmt=".",label = "data") - ax.plot(charge,histo_smoothed,label = f'smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})') - ax.plot(charge,weight_gaussian(charge,coeff_mean[0],coeff_mean[1],coeff_mean[2]),label = 'gaussian fit of the SPE') - ax.vlines(coeff_mean[1],0,peak_value,label = f'mean initial value = {coeff_mean[1] - coeff[1]:.0f}',color = "red") - ax.add_patch(Rectangle((coeff_mean[1]-coeff_mean[2], 0), 2 * coeff_mean[2], peak_value_mean,fc=to_rgba('red', 0.5))) - ax.set_xlim([peak_pos - 500,None]) - ax.set_xlabel("Charge (ADC)", size=15) - ax.set_ylabel("Events", size=15) - ax.legend(fontsize=7) - os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/",exist_ok=True) - fig.savefig(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/initialization_mean_pixel{extension}_{os.getpid()}.pdf") - fig.clf() - plt.close(fig) - del fig,ax - return coeff_mean,var_matrix - - @staticmethod - def _get_pedestal_gaussian_fit(charge_in, histo_in ,extension = "") : - #x = np.linspace(nectargain.charge[pixel].min(),nectargain.charge[pixel].max(),int(nectargain.charge[pixel].max()-nectargain.charge[pixel].min())) - #interp = interp1d(nectargain.charge[pixel],nectargain.histo[pixel]) - charge = charge_in.data[~histo_in.mask] - histo = histo_in.data[~histo_in.mask] - - windows_lenght = NectarGainSPE._Windows_lenght - order = NectarGainSPE._Order - histo_smoothed = savgol_filter(histo, windows_lenght, order) - - peaks = find_peaks(histo_smoothed,10) - peak_max = np.argmax(histo_smoothed[peaks[0]]) - peak_pos,peak_value = charge[peaks[0][peak_max]], histo[peaks[0][peak_max]] - - coeff, var_matrix = curve_fit(weight_gaussian, charge[:peaks[0][peak_max]], histo_smoothed[:peaks[0][peak_max]],p0 = [peak_value,peak_pos,1]) - - if log.getEffectiveLevel() == logging.DEBUG : - log.debug('plotting figures with prefit parameters computation') - fig,ax = plt.subplots(1,1,figsize = (5,5)) - ax.errorbar(charge,histo,np.sqrt(histo),zorder=0,fmt=".",label = "data") - ax.plot(charge,histo_smoothed,label = f'smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})') - ax.plot(charge,weight_gaussian(charge,coeff[0],coeff[1],coeff[2]),label = 'gaussian fit of the pedestal, left tail only') - ax.set_xlim([peak_pos - 500,None]) - ax.vlines(coeff[1],0,peak_value,label = f'pedestal initial value = {coeff[1]:.0f}',color = 'red') - ax.add_patch(Rectangle((coeff[1]-coeff[2], 0), 2 * coeff[2], peak_value,fc=to_rgba('red', 0.5))) - ax.set_xlabel("Charge (ADC)", size=15) - ax.set_ylabel("Events", size=15) - ax.legend(fontsize=7) - os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/",exist_ok=True) - fig.savefig(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/initialization_pedestal_pixel{extension}_{os.getpid()}.pdf") - fig.clf() - plt.close(fig) - del fig,ax - return coeff,var_matrix - - @abstractmethod - def create_output_table(self) : pass - - @abstractmethod - def save(self,path,**kwargs) : pass - - @abstractmethod - def run(self,pixel : int = None,**kwargs): pass - @abstractmethod - def _run_obs(self,pixel : int,**kwargs) : pass - @classmethod - @abstractmethod - def _run_obs_static(cls,it : int, funct, parameters : Parameters, pixels_id : int, charge : np.ndarray, histo : np.ndarray, **kwargs) : pass - - #@abstractmethod - #def _update_parameters_prefit(self,pixel : int) : pass - @classmethod - @abstractmethod - def _update_parameters_prefit_static(cls, it : int, parameters : Parameters, charge : np.ndarray, histo : np.ndarray,**kwargs) : pass - - - @abstractmethod - def Chi2(self,**kwargs) : pass - @abstractmethod - def Chi2_static(self,**kwargs) : pass - - - @property - def parameters(self) : return copy.deepcopy(self.__parameters) - @property - def _parameters(self) : return self.__parameters - @property - def _minuitParameters(self) : return self.__minuitParameters \ No newline at end of file diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_combined.py b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_combined.py deleted file mode 100644 index 89c59f24..00000000 --- a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_combined.py +++ /dev/null @@ -1,285 +0,0 @@ -import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') -log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers - -from tqdm import tqdm -import numpy as np -from astropy.table import Column -import astropy.units as u -from datetime import date -import random -import matplotlib.pyplot as plt -import os -from pathlib import Path - -from iminuit import Minuit - - -from .NectarGainSPE_singlerun import NectarGainSPESingle, NectarGainSPESingleSignalStd -from nectarchain.calibration.container import ChargeContainer -from .NectarGainSPE import NectarGainSPE -from .utils import UtilsMinuit,Gain,MPE2,Statistics - -__all__ = ["NectarGainSPECombinedNoPed"] - - -#class NectarGainSPESingleHHV(NectarGainSPESingle): -# """class to perform fit of the 1400V signal and pedestal""" - -#class NectarGainSPESingleCombined(NectarGainSPESingle): -# """class to perform fit of the 1400V and 1000V signal and pedestal""" - - - -class NectarGainSPECombinedNoPed(NectarGainSPE): - """class to perform fit of the 1400V and 1000V signal""" - __parameters_file = 'parameters_signal_combined.yaml' - - def __init__(self,signalHHV : ChargeContainer, signal : ChargeContainer, same_luminosity : bool = True, parameters_file = None, parameters_file_HHV = None,**kwargs) : - super().__init__(**kwargs) - - self.nectarGainHHV = NectarGainSPESingleSignalStd(signalHHV, parameters_file = parameters_file_HHV) - self.nectarGain = NectarGainSPESingleSignalStd(signal, parameters_file = parameters_file) - - if self.nectarGainHHV.npixels != self.nectarGain.npixels : - e = Exception("not same number of pixels in HHV and std run") - log.error(e,exc_info=True) - raise e - else : - self.__npixels = self.nectarGainHHV.npixels - - if not((self.nectarGainHHV.pixels_id == self.nectarGain.pixels_id).all()) : - e = Exception("not same pixels_id in HHV and std run") - log.error(e,exc_info=True) - raise e - else : - self.__pixels_id = self.nectarGainHHV.pixels_id - - self.__gain = np.empty((self.__npixels,3)) - self.__gainHHV = np.empty((self.__npixels,3)) - - #shared parameters - self.__pp = self.nectarGainHHV.pp - self.__resolution = self.nectarGainHHV.resolution - self.__n = self.nectarGainHHV.n - self.__pedestalWidth = self.nectarGainHHV.pedestalWidth - self._parameters.append(self.__pp) - self._parameters.append(self.__resolution) - self._parameters.append(self.__n) - self._parameters.append(self.__pedestalWidth) - if same_luminosity : - self.__luminosity = self.nectarGainHHV.luminosity - self._parameters.append(self.__luminosity) - - #others - if not(same_luminosity) : - self.__luminosity = self.nectarGain.luminosity - self.__luminosityHHV = self.nectarGain.luminosity - self.__luminosityHHV.name = "luminosityHHV" - self._parameters.append(self.__luminosity) - self._parameters.append(self.__luminosityHHV) - - self.__meanHHV = self.nectarGainHHV.mean - self.__meanHHV.name = "meanHHV" - self.__mean = self.nectarGain.mean - self.__pedestalHHV = self.nectarGainHHV.pedestal - self.__pedestalHHV.name = "pedestalHHV" - self.__pedestal = self.nectarGain.pedestal - self._parameters.append(self.__meanHHV) - self._parameters.append(self.__mean) - self._parameters.append(self.__pedestalHHV) - self._parameters.append(self.__pedestal) - - self.read_param_from_yaml(NectarGainSPECombinedNoPed.__parameters_file) - - log.info(self._parameters) - - self._make_minuit_parameters() - - self.create_output_table() - - def create_output_table(self) : - self._output_table.meta['npixel'] = self.npixels - self._output_table.meta['comments'] = f'Produced with NectarGain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' - - self._output_table.add_column(Column(np.zeros((self.npixels),dtype = bool),"is_valid",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(self.__pixels_id,"pixel",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),"gain",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"gain_error",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),"gainHHV",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"gainHHV_error",unit = u.dimensionless_unscaled)) - - for parameter in self._parameters.parameters : - self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),parameter.name,unit = parameter.unit)) - self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),f'{parameter.name}_error',unit = parameter.unit)) - - self._output_table.add_column(Column(np.zeros((self.npixels),dtype = np.float64),"likelihood",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.zeros((self.npixels),dtype = np.float64),"pvalue",unit = u.dimensionless_unscaled)) - - def Chi2(self,pixel : int): - def _Chi2(resolution,mean,meanHHV,pedestal,pedestalHHV,pedestalWidth,luminosity) : - return self.nectarGainHHV.Chi2(pixel)(resolution,meanHHV,pedestalHHV,pedestalWidth,luminosity) + self.nectarGain.Chi2(pixel)(resolution,mean,pedestal,pedestalWidth,luminosity) - return _Chi2 - - def save(self,path,**kwargs) : - path = Path(path) - os.makedirs(path,exist_ok = True) - log.info(f'data saved in {path}') - self._output_table.write(f"{path}/output_table.ecsv", format='ascii.ecsv',overwrite = kwargs.get("overwrite",False)) - - def run(self,pixel : int = None,**kwargs): - if pixel is None : - for i in tqdm(range(self.npixels)) : - if self.nectarGain.charge.mask[i].all() or self.nectarGainHHV.charge.mask[i].all() : - log.info(f'do not run fit on pixel {i} (pixel_id = {self.__pixels_id[i]}), it seems to be a broken pixel from charge computation') - else : - log.info(f"running SPE fit for pixel {i} (pixel_id = {self.__pixels_id[i]})") - self._run_obs(i,**kwargs) - else : - if not(isinstance(pixel,np.ndarray)) : - pixels = np.asarray([pixel],dtype = np.int16) - else : - pixels = pixel - for pixel in tqdm(pixels) : - if pixel >= self.npixels : - e = Exception(f"pixel must be < {self.npixels}") - log.error(e,exc_info=True) - raise e - else : - if self.nectarGain.charge.mask[pixel].all() or self.nectarGainHHV.charge.mask[pixel].all() : - log.info(f'do not run fit on pixel {pixel} (pixel_id = {self.__pixels_id[pixel]}), it seems to be a broken pixel from charge computation') - else : - log.info(f"running SPE fit for pixel {pixel} (pixel_id = {self.__pixels_id[pixel]})") - self._run_obs(pixel,**kwargs) - return 0 - - def _run_obs(self,pixel,**kwargs) : - self._update_parameters_prefit(pixel) - fit = Minuit(self.Chi2(pixel),**self._minuitParameters['values']) - UtilsMinuit.set_minuit_parameters_limits_and_errors(fit,self._minuitParameters) - log.info(f"Initial value of Likelihood = {self.Chi2(pixel)(**self._minuitParameters['values'])}") - log.info(f"Initial parameters value : {fit.values}") - #log.debug(self.Chi2(pixel)(0.5,500,14600,50,1)) - - if log.getEffectiveLevel() == logging.ERROR : - fit.print_level = 0 - if log.getEffectiveLevel() == logging.WARNING : - fit.print_level = 1 - if log.getEffectiveLevel() == logging.INFO : - fit.print_level = 2 - if log.getEffectiveLevel() == logging.DEBUG : - fit.print_level = 3 - - fit.strategy = 2 - fit.throw_nan = True - try : - log.info('migrad execution') - fit.migrad(ncall=super(NectarGainSPECombinedNoPed,self)._Ncall) - fit.hesse() - valid = fit.valid - except RuntimeError as e : - log.warning(e,exc_info = True) - log.info("change method : re-try with simplex") - fit.reset() - fit.throw_nan = True - try : - log.info('simplex execution') - fit.simplex(ncall=super(NectarGainSPECombinedNoPed,self)._Ncall) - fit.hesse() - valid = fit.valid - except Exception as e : - log.error(e,exc_info = True) - log.warning(f"skip pixel {pixel} (pixel_id : {self.pixels_id})") - valid = False - - except Exception as e : - log.error(e,exc_info = True) - raise e - - if valid : - log.info(f"fitted value : {fit.values}") - log.info(f"fitted errors : {fit.errors}") - self._update_parameters_postfit(fit) - - self.__gain[pixel,0] = Gain(self.__pp.value,self.__resolution.value,self.__mean.value,self.__n.value) - stat_gain = np.array([Gain(self.__pp.value,random.gauss(self.__resolution.value, self.__resolution.error),random.gauss(self.__mean.value, self.__mean.error),self.__n.value) for i in range(1000)]) - self.__gain[pixel,1] = self.__gain[pixel,0] - np.quantile(stat_gain,0.16) - self.__gain[pixel,2] = np.quantile(stat_gain,0.84) - self.__gain[pixel,0] - - self.__gainHHV[pixel,0] = Gain(self.__pp.value,self.__resolution.value,self.__meanHHV.value,self.__n.value) - stat_gain = np.array([Gain(self.__pp.value,random.gauss(self.__resolution.value, self.__resolution.error),random.gauss(self.__meanHHV.value, self.__meanHHV.error),self.__n.value) for i in range(1000)]) - self.__gainHHV[pixel,1] = self.__gainHHV[pixel,0] - np.quantile(stat_gain,0.16) - self.__gainHHV[pixel,2] = np.quantile(stat_gain,0.84) - self.__gainHHV[pixel,0] - - ndof = self.histo[pixel].data[~self.histo[pixel].mask].shape[0] - fit.nfit - self.fill_table(pixel,valid,ndof,fit.fval) - - log.info(f"Reconstructed gain is {self.__gain[pixel,0] - self.__gain[pixel,1]:.2f} < {self.__gain[pixel,0]:.2f} < {self.__gain[pixel,0] + self.__gain[pixel,2]:.2f}") - self._output_table['gain'][pixel] = self.__gain[pixel,0] - self._output_table['gain_error'][pixel][0] = self.__gain[pixel,1] - self._output_table['gain_error'][pixel][1] = self.__gain[pixel,2] - log.info(f"Reconstructed gainHHV is {self.__gainHHV[pixel,0] - self.__gainHHV[pixel,1]:.2f} < {self.__gainHHV[pixel,0]:.2f} < {self.__gainHHV[pixel,0] + self.__gainHHV[pixel,2]:.2f}") - self._output_table['gainHHV'][pixel] = self.__gainHHV[pixel,0] - self._output_table['gainHHV_error'][pixel][0] = self.__gainHHV[pixel,1] - self._output_table['gainHHV_error'][pixel][1] = self.__gainHHV[pixel,2] - - if kwargs.get('figpath',0) != 0 : - fig,ax = plt.subplots(1,2,figsize=(16, 6)) - ax[0].errorbar(self.nectarGain.charge[pixel],self.nectarGain.histo[pixel],np.sqrt(self.nectarGain.histo[pixel]),zorder=0,fmt=".",label = "data") - ax[0].plot(self.nectarGain.charge[pixel], - np.trapz(self.nectarGain.histo[pixel],self.nectarGain.charge[pixel])*MPE2(self.nectarGain.charge[pixel],self.__pp.value,self.__resolution.value,self.__mean.value,self.__n.value,self.__pedestal.value,self.__pedestalWidth.value,self.__luminosity.value), - zorder=1, - linewidth=2, - label = f"SPE model fit \n gain : {self.__gain[pixel,0] - self.__gain[pixel,1]:.2f} < {self.__gain[pixel,0]:.2f} < {self.__gain[pixel,0] + self.__gain[pixel,2]:.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval):.3f},\n likelihood = {fit.fval:.2f}") - ax[0].set_xlabel("Charge (ADC)", size=15) - ax[0].set_ylabel("Events", size=15) - ax[0].set_title(f"SPE fit pixel : {pixel} (pixel id : {self.pixels_id[pixel]})") - ax[0].legend(fontsize=12) - - ax[1].errorbar(self.nectarGainHHV.charge[pixel],self.nectarGainHHV.histo[pixel],np.sqrt(self.nectarGainHHV.histo[pixel]),zorder=0,fmt=".",label = "data") - ax[1].plot(self.nectarGainHHV.charge[pixel], - np.trapz(self.nectarGainHHV.histo[pixel],self.nectarGainHHV.charge[pixel])*MPE2(self.nectarGainHHV.charge[pixel],self.__pp.value,self.__resolution.value,self.__meanHHV.value,self.__n.value,self.__pedestalHHV.value,self.__pedestalWidth.value,self.__luminosity.value), - zorder=1, - linewidth=2, - label = f"SPE model fit \n gainHHV : {self.__gainHHV[pixel,0] - self.__gainHHV[pixel,1]:.2f} < {self.__gainHHV[pixel,0]:.2f} < {self.__gainHHV[pixel,0] + self.__gainHHV[pixel,2]:.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval):.3f},\n likelihood = {fit.fval:.2f}") - ax[1].set_xlabel("Charge (ADC)", size=15) - ax[1].set_ylabel("Events", size=15) - ax[1].set_title(f"SPE fit pixel : {pixel} (pixel id : {self.pixels_id[pixel]})") - ax[1].legend(fontsize=12) - - os.makedirs(kwargs.get('figpath'),exist_ok = True) - fig.savefig(f"{kwargs.get('figpath')}/fit_SPE_pixel{pixel}.pdf") - fig.clf() - plt.close(fig) - del fig,ax - else : - log.warning(f"fit {pixel} is not valid") - self.fill_table(pixel,valid,0,0) - - def _update_parameters_prefit(self,pixel) : - - coeff,var_matrix = NectarGainSPE._get_pedestal_gaussian_fit(self.nectarGain.charge, self.nectarGain.histo, pixel) - self.__pedestal.value = coeff[1] - self.__pedestal.min = coeff[1] - coeff[2] - self.__pedestal.max = coeff[1] + coeff[2] - self._minuitParameters['values']['pedestal'] = self.__pedestal.value - self._minuitParameters['limit_pedestal'] = (self.__pedestal.min,self.__pedestal.max) - log.debug(f"pedestal updated : {self.__pedestal.value}") - - coeff,var_matrix = NectarGainSPE._get_pedestal_gaussian_fit(self.nectarGain.charge, self.nectarGain.histo, pixel,"_HHV") - self.__pedestalHHV.value = coeff[1] - self.__pedestalHHV.min = coeff[1] - coeff[2] - self.__pedestalHHV.max = coeff[1] + coeff[2] - self._minuitParameters['values']['pedestalHHV'] = self.__pedestalHHV.value - self._minuitParameters['limit_pedestalHHV'] = (self.__pedestalHHV.min,self.__pedestalHHV.max) - log.debug(f"pedestalHHV updated : {self.__pedestalHHV.value}") - - def NG_Likelihood_Chi2(cls,**kwargs) : pass - - #run properties - @property - def pixels_id(self) : return self.__pixels_id - @property - def npixels(self) : return self.__npixels - diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py b/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py deleted file mode 100644 index fae81bc2..00000000 --- a/src/nectarchain/calibration/NectarGain/SPEfit/NectarGainSPE_singlerun.py +++ /dev/null @@ -1,1170 +0,0 @@ -import numpy as np -from matplotlib import pyplot as plt -from scipy.special import gammainc -from iminuit import Minuit -import random -import astropy.units as u -from astropy.table import Table,QTable,Column,MaskedColumn -import yaml -import os -from datetime import date -from pathlib import Path -from tqdm import tqdm -import matplotlib -matplotlib.use('AGG',force = True) - -import time - -import pandas as pd - -import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') -log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers - -import copy - -import multiprocessing as mlp -from multiprocessing.pool import ThreadPool as Pool - -from .parameters import Parameters,Parameter -from ...container import ChargeContainer -from .utils import UtilsMinuit,MPE2,Gain,gaussian,Multiprocessing,Statistics -from .NectarGainSPE import NectarGainSPE - -from abc import abstractclassmethod - -__all__ = ["NectarGainSPESingleSignalStd","NectarGainSPESingleSignal","NectarGainSPESinglePed","NectarGainSPESingleSignalfromHHVFit"] - - - -class NectarGainSPESingle(NectarGainSPE): - - def __init__(self,signal : ChargeContainer,**kwargs) : - log.info("initialisation of the SPE fit instance") - super().__init__(**kwargs) - - histo = signal.histo_hg(autoscale = True) - #access data - self.__charge = histo[1] - self.__histo = histo[0] - self.__mask_fitted_pixel = np.zeros((self.__charge.shape[0]),dtype = bool) - self.__pixels_id = signal.pixels_id - - self.__pedestal = Parameter(name = "pedestal", - value = (np.min(self.__charge) + np.sum(self.__charge * self.__histo)/(np.sum(self.__histo)))/2, - min = np.min(self.__charge), - max = np.sum(self.__charge*self.__histo)/np.sum(self.__histo), - unit = u.dimensionless_unscaled) - - - self._parameters.append(self.__pedestal) - - def _update_parameters() - - - def _make_fit_array_from_parameters(self) : - fit_array = np.empty((self.npixels),dtype = np.object_) - for i in range(len(fit_array)) : - parameters = self._update_parameters(parameters,charge[i].data[~charge[i].mask],counts[i].data[~charge[i].mask]) - minuitParameters = UtilsMinuit.make_minuit_par_kwargs(parameters) - #log.info('creation of fit') - fit_array[i] = Minuit(cost(charge[i].data[~charge[i].mask],counts[i].data[~charge[i].mask]), - pedestal = minuitParameters['values']['pedestal'], - pp = minuitParameters['values']['pp'], - luminosity = minuitParameters['values']['luminosity'], - resolution = minuitParameters['values']['resolution'], - mean = minuitParameters['values']['mean'], - n = minuitParameters['values']['n'], - pedestalWidth = minuitParameters['values']['pedestalWidth'] - ) - #log.info('fit created') - fit_array[i].errordef = Minuit.LIKELIHOOD - fit_array[i].strategy = 0 - fit_array[i].tol = 1e40 - fit_array[i].print_level = 1 - fit_array[i].throw_nan = True - UtilsMinuit.set_minuit_parameters_limits_and_errors(fit_array[i],minuitParameters) - #log.info(fit_array[i].values) - #log.info(fit_array[i].limits) - #log.info(fit_array[i].fixed) - return fit_array - - - -class NectarGainSPESingleSignal_new(NectarGainSPESingle_new) : - - """class to perform fit of the SPE signal with all free parameters""" - __parameters_file = 'parameters_signal.yaml' - #def __new__(cls) : - # print("NectarGainSPESingleSignal is not instanciable") - # return 0 - def __init__(self,signal : ChargeContainer,parameters_file = None,**kwargs) : - super().__init__(signal,**kwargs) - self.__gain = np.empty((self.charge.shape[0],3)) - #if parameters file is provided - if parameters_file is None : - parameters_file = NectarGainSPESingleSignal.__parameters_file - with open(f"{os.path.dirname(os.path.abspath(__file__))}/{parameters_file}") as parameters : - param = yaml.safe_load(parameters) - self.__pp = Parameter(name = "pp", - value = param["pp"]["value"], - min = param["pp"].get("min",np.nan), - max = param["pp"].get("max",np.nan), - unit = param["pp"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__pp) - - self.__luminosity = Parameter(name = "luminosity", - value = param["luminosity"]["value"], - min = param["luminosity"].get("min",np.nan), - max = param["luminosity"].get("max",np.nan), - unit = param["luminosity"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__luminosity) - - self.__resolution = Parameter(name = "resolution", - value = param["resolution"]["value"], - min = param["resolution"].get("min",np.nan), - max = param["resolution"].get("max",np.nan), - unit = param["resolution"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__resolution) - - self.__mean = Parameter(name = "mean", - value = param["mean"]["value"], - min = param["mean"].get("min",0), - max = param["mean"].get("max",np.nan), - unit = param["mean"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__mean) - - self.__n = Parameter(name = "n", - value = param["n"]["value"], - min = param["n"].get("min",np.nan), - max = param["n"].get("max",np.nan), - unit = param["n"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__n) - - self.__pedestalWidth = Parameter(name = "pedestalWidth", - value = param["pedestalWidth"]["value"], - min = param["pedestalWidth"].get("min",np.nan), - max = param["pedestalWidth"].get("max",np.nan), - unit = param["pedestalWidth"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__pedestalWidth) - - self._make_minuit_parameters() - - self.create_output_table() - - - @classmethod - def NG_Likelihood_Chi2(cls,pp,res,mu2,n,muped,sigped,lum,charge,histo,**kwargs): - pdf = MPE2(charge,pp,res,mu2,n,muped,sigped,lum,**kwargs) - #log.debug(f"pdf : {np.sum(pdf)}") - Ntot = np.sum(histo) - #log.debug(f'Ntot : {Ntot}') - mask = histo > 0 - Lik = np.sum(((pdf*Ntot-histo)[mask])**2/histo[mask]) #2 times faster - return Lik - - def Chi2(self,pixel : int): - def _Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity) : - #assert not(np.isnan(pp) or np.isnan(resolution) or np.isnan(mean) or np.isnan(n) or np.isnan(pedestal) or np.isnan(pedestalWidth) or np.isnan(luminosity)) - if self.__old_lum != luminosity : - for i in range(1000): - if (gammainc(i+1,luminosity) < 1e-5): - self.__old_ntotalPE = i - break - self.__old_lum = luminosity - kwargs = {"ntotalPE" : self.__old_ntotalPE} - - return self.NG_Likelihood_Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity,self.charge[pixel],self.histo[pixel],**kwargs) - return _Chi2 - - def Chi2_static(self,pixel : int) : - charge = copy.deepcopy(self.charge[pixel].data[~self.charge[pixel].mask]) - histo = copy.deepcopy(self.histo[pixel].data[~self.histo[pixel].mask]) - def _Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity) : - #assert not(np.isnan(pp) or np.isnan(resolution) or np.isnan(mean) or np.isnan(n) or np.isnan(pedestal) or np.isnan(pedestalWidth) or np.isnan(luminosity)) - ntotalPE = 0 - for i in range(1000): - if (gammainc(i+1,luminosity) < 1e-5): - ntotalPE = i - break - kwargs = {"ntotalPE" : ntotalPE} - return __class__.NG_Likelihood_Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity,charge,histo,**kwargs) - return _Chi2 - - @classmethod - def _update_parameters_prefit_static(cls,it : int, parameters : Parameters, charge : np.ndarray, histo : np.ndarray,**kwargs) : - super(__class__, cls)._update_parameters_prefit_static(it,parameters,charge,histo,**kwargs) - pedestal = parameters['pedestal'] - pedestalWidth = parameters["pedestalWidth"] - pedestalWidth.value = pedestal.max - pedestal.value - pedestalWidth.max = 3 * pedestalWidth.value - log.debug(f"pedestalWidth updated : {pedestalWidth}") - try : - coeff,var_matrix = NectarGainSPE._get_mean_gaussian_fit(charge,histo,f'{it}_nominal') - if (coeff[1] - pedestal.value < 0) or ((coeff[1] - coeff[2]) - pedestal.max < 0) : raise Exception("mean gaussian fit not good") - mean = parameters['mean'] - mean.value = coeff[1] - pedestal.value - mean.min = (coeff[1] - coeff[2]) - pedestal.max - mean.max = (coeff[1] + coeff[2]) - pedestal.min - log.debug(f"mean updated : {mean}") - except Exception as e : - log.warning(e,exc_info=True) - log.warning("mean parameters limits and starting value not changed") - - #fit parameters - @property - def pp(self) : return self.__pp - @property - def luminosity(self) : return self.__luminosity - @property - def mean(self) : return self.__mean - @property - def n(self) : return self.__n - @property - def resolution(self) : return self.__resolution - @property - def pedestalWidth(self) : return self.__pedestalWidth - @property - def gain(self) : return self.__gain - - - #intern parameters - @property - def _old_lum(self) : return self.__old_lum - @_old_lum.setter - def _old_lum(self,value) : self.__old_lum = value - @property - def _old_ntotalPE(self) : return self.__old_ntotalPE - @_old_ntotalPE.setter - def _old_ntotalPE(self,value) : self.__old_ntotalPE = value - - - - -class NectarGainSPESingle(NectarGainSPE): - _Ncall = 100 - _Nproc_Multiprocess = mlp.cpu_count() // 2 - _FitTol = 1e20 - _FitStrategy = 0 - - def __init__(self,signal : ChargeContainer,**kwargs) : - log.info("initialisation of the SPE fit instance") - super().__init__(**kwargs) - - histo = signal.histo_hg(autoscale = True) - #access data - self.__charge = histo[1] - self.__histo = histo[0] - self.__mask_fitted_pixel = np.zeros((self.__charge.shape[0]),dtype = bool) - self.__pixels_id = signal.pixels_id - - self.__pedestal = Parameter(name = "pedestal", - value = (np.min(self.__charge) + np.sum(self.__charge * self.__histo)/(np.sum(self.__histo)))/2, - min = np.min(self.__charge), - max = np.sum(self.__charge*self.__histo)/np.sum(self.__histo), - unit = u.dimensionless_unscaled) - - - self._parameters.append(self.__pedestal) - - def create_output_table(self) : - self._output_table.meta['npixel'] = self.npixels - self._output_table.meta['comments'] = f'Produced with NectarGain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' - - self._output_table.add_column(Column(np.zeros((self.npixels),dtype = bool),"is_valid",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(self.__pixels_id,"pixel",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),"gain",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"gain_error",unit = u.dimensionless_unscaled)) - - for parameter in self._parameters.parameters : - self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),parameter.name,unit = parameter.unit)) - self._output_table.add_column(Column(np.empty((self.npixels),dtype = np.float64),f'{parameter.name}_error',unit = parameter.unit)) - - self._output_table.add_column(Column(np.zeros((self.npixels),dtype = np.float64),"likelihood",unit = u.dimensionless_unscaled)) - self._output_table.add_column(Column(np.zeros((self.npixels),dtype = np.float64),"pvalue",unit = u.dimensionless_unscaled)) - - - def make_table_from_output_multi(self,list_dict : list) : - self._output_table = QTable.from_pandas(pd.DataFrame.from_dict(list_dict)) - for param in self._parameters.parameters : - self._output_table[param.name] = Column(self._output_table[param.name].value, param.name, unit=param.unit) - if 'gain' in self._output_table.colnames : - if isinstance(self._output_table['gain'],MaskedColumn) : - self._output_table['gain'] = self._output_table['gain']._data - self._output_table['gain_error'] = self._output_table['gain_error']._data - self._output_table.meta['npixel'] = self.npixels - self._output_table.meta['comments'] = f'Produced with NectarGain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' - - @staticmethod - def _run_fit(funct,parameters,pixels_id,prescan = False,**kwargs) : - minuitParameters = UtilsMinuit.make_minuit_par_kwargs(parameters.unfrozen) - fit = Minuit(funct,**minuitParameters['values']) - fit.errordef = Minuit.LIKELIHOOD - fit.tol = __class__._FitTol - #fit.precision = 1e-2 - - UtilsMinuit.set_minuit_parameters_limits_and_errors(fit,minuitParameters) - log.info(f"Initial value of Likelihood = {funct(**minuitParameters['values'])}") - log.info(f"Initial parameters value : {fit.values}") - #log.debug(self.Chi2(pixel)(0.5,500,14600,50,1)) - - if log.getEffectiveLevel() == logging.ERROR : - fit.print_level = 0 - if log.getEffectiveLevel() == logging.WARNING : - fit.print_level = 1 - if log.getEffectiveLevel() == logging.INFO : - fit.print_level = 2 - if log.getEffectiveLevel() == logging.DEBUG : - fit.print_level = 3 - - fit.strategy = __class__._FitStrategy - fit.throw_nan = True - if prescan : - log.info("let's do a fisrt brut force scan") - fit.scan() - try : - log.info('migrad execution') - t = time.time() - fit.migrad(ncall=__class__._Ncall) - log.debug(f'time for migrad execution is {time.time() - t:.0f} sec') - t = time.time() - fit.hesse(ncall=__class__._Ncall) - log.debug(f'time for hesse execution is {time.time() - t:.0f} sec') - valid = fit.valid - except RuntimeError as e : - log.warning(e,exc_info = True) - log.info("change method : re-try with simplex") - fit.reset() - fit.throw_nan = True - if prescan : - log.info("let's do a fisrt brut force scan") - fit.scan() - try : - log.info('simplex execution') - fit.simplex(ncall=__class__._Ncall) - fit.hesse() - valid = fit.valid - except Exception as e : - log.error(e,exc_info = True) - log.warning(f"skip pixel_id : {pixels_id})") - valid = False - - except Exception as e : - log.error(e,exc_info = True) - raise e - - return fit,valid - - def run(self,pixel : int = None,multiproc = False, **kwargs): - def task_simple(i,**kwargs) : - log.info(f"i = {i}") - log.debug(f"{kwargs}") - - if self.charge.mask[i].all() : - log.info(f'do not run fit on pixel {i} (pixel_id = {self.__pixels_id[i]}), it seems to be a broken pixel from charge computation') - else : - log.info(f"running SPE fit for pixel {i} (pixel_id = {self.__pixels_id[i]})") - self._run_obs(i,**kwargs) - - - def task_multiple(funct,parameters : Parameters,pixels_id : list,charge : np.ndarray, histo : np.ndarray,pix) : - _funct = {i : funct(i) for i in pix} - _parameters = copy.deepcopy(parameters) - _pixels_id = copy.deepcopy(pixels_id) - _charge = copy.deepcopy(charge) - _histo = copy.deepcopy(histo) - _class = copy.deepcopy(self.__class__) - def task(i,kwargs) : - log.info(f"i = {i}") - log.debug(f"{kwargs}") - - if charge.mask.all() : - log.info(f'do not run fit on pixel {i} (pixel_id = {pixels_id[i]}), it seems to be a broken pixel from charge computation') - output = {"is_valid" : False, "pixel" : pixels_id[i]} - for parameter in _parameters.parameters : - output[parameter.name] = parameter.value - output[f"{parameter.name}_error"] = parameter.error - else : - log.info(f"running SPE fit for pixel {i} (pixel_id = {pixels_id[i]})") - try : - output = _class._run_obs_static(i,_funct[i], copy.deepcopy(_parameters), _pixels_id[i], _charge[i], _histo[i], **kwargs) - except Exception as e : - log.error(e,exc_info=True) - output = {"is_valid" : False, "pixel" : pixels_id[i]} - for parameter in _parameters.parameters : - output[parameter.name] = parameter.value - output[f"{parameter.name}_error"] = parameter.error - return output - return task - - def task_bis(i,funct,parameters : Parameters,pixels_id : int,charge : np.ndarray, histo : np.ndarray,_class : str,kwargs) : - log.info(f"i = {i}") - log.debug(f"{kwargs}") - if charge.mask.all() : - log.info(f'do not run fit on pixel {i} (pixel_id = {pixels_id}), it seems to be a broken pixel from charge computation') - output = {"is_valid" : False, "pixel" : pixels_id} - for parameter in parameters.parameters : - output[parameter.name] = parameter.value - output[f"{parameter.name}_error"] = parameter.error - else : - log.info(f"running SPE fit for pixel {i} (pixel_id = {pixels_id})") - try : - output = _class._run_obs_static(i,funct, parameters, pixels_id, charge, histo, **kwargs) - except Exception as e : - log.error(e,exc_info=True) - output = {"is_valid" : False, "pixel" : pixels_id} - for parameter in parameters.parameters : - output[parameter.name] = parameter.value - output[f"{parameter.name}_error"] = parameter.error - return output - - - - if pixel is None : - if multiproc : - nproc = min(kwargs.get("nproc",self._Nproc_Multiprocess),self.npixels) - i=0 - log.info(f"pixels : {self.npixels}") - with Pool(nproc) as pool: - chunksize = kwargs.get("chunksize",max(1,self.npixels//(nproc*10))) - log.info(f"pooling with nproc {nproc}, chunksize {chunksize}") - - handlerlevel = [] - for handler in log.handlers : - handlerlevel.append(handler.level) - handler.setLevel(logging.FATAL) - loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict] - loglevel = [logger.getEffectiveLevel() for logger in loggers] - for logger in loggers : logger.setLevel(logging.FATAL) - - result = pool.starmap_async(task_bis, - [(i,self.Chi2_static(i),copy.deepcopy(self.parameters),self.__pixels_id[i], self.__charge[i], self.__histo[i], self.__class__,kwargs) for i in tqdm(range(self.npixels))], - chunksize = chunksize, - error_callback=Multiprocessing.custom_error_callback - ) - - result.wait() - - for i,handler in enumerate(log.handlers) : - handler.setLevel(handlerlevel[i]) - for i,logger in enumerate(loggers) : logger.setLevel(loglevel[i]) - - ###WITH APPLY_ASYNC ### - #result = [pool.apply_async(task_multiple_bis(self.Chi2_static,self.parameters, self.__pixels_id, self.__charge, self.__histo,pixels),args = (i,),kwds = kwargs) for i in tqdm(pixels)] - #output = [] - #for i,pix in tqdm(enumerate(pixels)) : - # log.info(f"watting for result pixel_id {self.__pixels_id[pix]}") - # #result[i].wait() - # output.append(result[i].get()) - #self.make_table_from_output_multi(output) - - try : - res = result.get() - self.make_table_from_output_multi(res) - except Exception as e : - log.error(e,exc_info=True) - log.error(f"results : {res}") - raise e - - else : - for i in tqdm(range(self.npixels)) : - task_simple(i,**kwargs) - else : - if not(isinstance(pixel,np.ndarray)) : - if isinstance(pixel,list) : - pixels = np.asarray(pixel,dtype = np.int16) - else : - pixels = np.asarray([pixel],dtype = np.int16) - - else : - pixels = pixel - - if multiproc : - nproc = min(kwargs.get("nproc",self._Nproc_Multiprocess),len(pixels)) - log.info(f"pixels : {pixels}") - with Pool(nproc) as pool: - chunksize = kwargs.get("chunksize",max(1,len(pixels)//(nproc*10))) - log.info(f"pooling with nproc {nproc}, chunksize {chunksize}") - - handlerlevel = [] - for handler in log.handlers : - handlerlevel.append(handler.level) - handler.setLevel(logging.FATAL) - loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict] - loglevel = [logger.getEffectiveLevel() for logger in loggers] - for logger in loggers : logger.setLevel(logging.FATAL) - - result = pool.starmap_async(task_bis, - [(i,self.Chi2_static(i),copy.deepcopy(self.parameters),self.__pixels_id[i], self.__charge[i], self.__histo[i], self.__class__,kwargs) for i in tqdm(pixels)], - chunksize = chunksize, - error_callback=Multiprocessing.custom_error_callback - ) - - - #result = pool.starmap_async(task_multiple(self.Chi2_static,self.parameters, self.__pixels_id, self.__charge, self.__histo,pixels), - # [(i,kwargs) for i in tqdm(pixels)], - # chunksize = chunksize, - # error_callback=Multiprocessing.custom_error_callback - # ) - result.wait() - - for i,handler in enumerate(log.handlers) : - handler.setLevel(handlerlevel[i]) - for i,logger in enumerate(loggers) : logger.setLevel(loglevel[i]) - - - - ###WITH APPLY_ASYNC ### - #result = [pool.apply_async(task_multiple_bis(self.Chi2_static,self.parameters, self.__pixels_id, self.__charge, self.__histo,pixels),args = (i,),kwds = kwargs) for i in tqdm(pixels)] - #output = [] - #for i,pix in tqdm(enumerate(pixels)) : - # log.info(f"watting for result pixel_id {self.__pixels_id[pix]}") - # #result[i].wait() - # output.append(result[i].get()) - #self.make_table_from_output_multi(output) - try : - res = result.get() - self.make_table_from_output_multi(res) - except Exception as e : - log.error(e,exc_info=True) - log.error(f"results : {res}") - raise e - - else : - for pixel in tqdm(pixels) : - if pixel >= self.npixels : - e = Exception(f"pixel must be < {self.npixels}") - log.error(e,exc_info=True) - raise e - else : - task_simple(pixel,**kwargs) - return 0 - - def save(self,path,**kwargs) : - path = Path(path) - os.makedirs(path,exist_ok = True) - log.info(f'data saved in {path}') - self._output_table.write(f"{path}/output_table.ecsv", format='ascii.ecsv',overwrite = kwargs.get("overwrite",False)) - - #ONLY KEEP STATIC ONE NOW - #def _update_parameters_prefit(self,pixel) : - # self.__pedestal.value = (np.min(self.__charge[pixel]) + np.sum(self.__charge[pixel] * self.__histo[pixel])/(np.sum(self.__histo[pixel])))/2 - # self.__pedestal.min = np.min(self.__charge[pixel]) - # self.__pedestal.max = np.sum(self.__charge[pixel]*self.__histo[pixel])/np.sum(self.__histo[pixel]) - # self._minuitParameters['values']['pedestal'] = self.__pedestal.value - # self._minuitParameters['limit_pedestal'] = (self.__pedestal.min,self.__pedestal.max) - - @classmethod - def _update_parameters_prefit_static(cls,it : int, parameters : Parameters, charge : np.ndarray, histo : np.ndarray,**kwargs) : - coeff,var_matrix = NectarGainSPE._get_pedestal_gaussian_fit(charge,histo,f'{it}_nominal') - pedestal = parameters['pedestal'] - pedestal.value = coeff[1] - pedestal.min = coeff[1] - coeff[2] - pedestal.max = coeff[1] + coeff[2] - log.debug(f"pedestal updated : {pedestal}") - - @abstractclassmethod - def NG_Likelihood_Chi2(cls,**kwargs) : pass - - @property - def charge(self) : return self.__charge - @property - def histo(self) : return self.__histo - - @property - def pedestal(self) : return self.__pedestal - - #run properties - @property - def pixels_id(self) : return self.__pixels_id - @property - def npixels(self) : return self.__charge.shape[0] - - -class NectarGainSPESingleSignal(NectarGainSPESingle): - """class to perform fit of the SPE signal with all free parameters""" - __parameters_file = 'parameters_signal.yaml' - #def __new__(cls) : - # print("NectarGainSPESingleSignal is not instanciable") - # return 0 - def __init__(self,signal : ChargeContainer,parameters_file = None,**kwargs) : - self.__old_lum = None - self.__old_ntotalPE = None - super().__init__(signal,**kwargs) - self.__gain = np.empty((self.charge.shape[0],3)) - #if parameters file is provided - if parameters_file is None : - parameters_file = NectarGainSPESingleSignal.__parameters_file - with open(f"{os.path.dirname(os.path.abspath(__file__))}/{parameters_file}") as parameters : - param = yaml.safe_load(parameters) - self.__pp = Parameter(name = "pp", - value = param["pp"]["value"], - min = param["pp"].get("min",np.nan), - max = param["pp"].get("max",np.nan), - unit = param["pp"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__pp) - - self.__luminosity = Parameter(name = "luminosity", - value = param["luminosity"]["value"], - min = param["luminosity"].get("min",np.nan), - max = param["luminosity"].get("max",np.nan), - unit = param["luminosity"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__luminosity) - - self.__resolution = Parameter(name = "resolution", - value = param["resolution"]["value"], - min = param["resolution"].get("min",np.nan), - max = param["resolution"].get("max",np.nan), - unit = param["resolution"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__resolution) - - self.__mean = Parameter(name = "mean", - value = param["mean"]["value"], - min = param["mean"].get("min",0), - max = param["mean"].get("max",np.nan), - unit = param["mean"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__mean) - - self.__n = Parameter(name = "n", - value = param["n"]["value"], - min = param["n"].get("min",np.nan), - max = param["n"].get("max",np.nan), - unit = param["n"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__n) - - self.__pedestalWidth = Parameter(name = "pedestalWidth", - value = param["pedestalWidth"]["value"], - min = param["pedestalWidth"].get("min",np.nan), - max = param["pedestalWidth"].get("max",np.nan), - unit = param["pedestalWidth"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__pedestalWidth) - - self._make_minuit_parameters() - - self.create_output_table() - - @classmethod - def _run_obs_static(cls,it : int, funct,parameters : Parameters, pixels_id : int, charge : np.ndarray, histo : np.ndarray, prescan = False, **kwargs) : - cls._update_parameters_prefit_static(it,parameters,charge,histo,**kwargs) - fit,valid = cls._run_fit(funct,parameters,pixels_id = pixels_id,prescan = prescan,**kwargs) - - if valid : - log.info(f"fitted value : {fit.values}") - log.info(f"fitted errors : {fit.errors}") - cls._update_parameters_postfit(fit,parameters) - - ndof = histo.data[~histo.mask].shape[0] - fit.nfit - output = cls._make_output_dict_obs(fit,valid,pixels_id,parameters,ndof) - - gain = np.empty(3) - - gain[0] = Gain(parameters['pp'].value, parameters['resolution'].value, parameters['mean'].value, parameters['n'].value) - stat_gain = np.array([Gain(parameters['pp'].value, random.gauss(parameters['resolution'].value, parameters['resolution'].error), random.gauss(parameters['mean'].value, parameters['mean'].error), parameters['n'].value) for i in range(1000)]) - gain[1] = gain[0] - np.quantile(stat_gain,0.16) - gain[2] = np.quantile(stat_gain,0.84) - gain[0] - - log.info(f"Reconstructed gain is {gain[0] - gain[1]:.2f} < {gain[0]:.2f} < {gain[0] + gain[2]:.2f}") - log.info(f"Likelihood value = {fit.fval}") - output['gain'] = gain[0] - output['gain_error'] = np.empty(2) - output['gain_error'][0] = gain[1] - output['gain_error'][1] = gain[2] - - - - if kwargs.get('figpath',0) != 0 : - fig,ax = plt.subplots(1,1,figsize=(8, 8)) - ax.errorbar(charge,histo,np.sqrt(histo),zorder=0,fmt=".",label = "data") - ax.plot(charge, - np.trapz(histo,charge)*MPE2(charge,parameters['pp'].value, parameters['resolution'].value, parameters['mean'].value, parameters['n'].value, parameters['pedestal'].value, parameters['pedestalWidth'].value, parameters['luminosity'].value), - zorder=1, - linewidth=2, - label = f"SPE model fit \n gain : {gain[0] - gain[1]:.2f} < {gain[0]:.2f} < {gain[0] + gain[2]:.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval):.3f},\n likelihood = {fit.fval:.2f}") - ax.set_xlabel("Charge (ADC)", size=15) - ax.set_ylabel("Events", size=15) - ax.set_title(f"SPE fit pixel {it} with pixel_id : {pixels_id}") - ax.set_xlim([parameters['pedestal'].value - 6 * parameters['pedestalWidth'].value, None]) - ax.legend(fontsize=18) - os.makedirs(kwargs.get('figpath'),exist_ok = True) - fig.savefig(f"{kwargs.get('figpath')}/fit_SPE_pixel{pixels_id}.pdf") - fig.clf() - plt.close(fig) - del fig,ax - else : - log.warning(f"fit pixel {it} with pixel_id = {pixels_id} is not valid") - output = cls._make_output_dict_obs(fit,valid,pixels_id,parameters) - - return output - - def _run_obs(self,pixel,prescan = False,**kwargs) : - self._update_parameters_prefit_static(pixel,self._parameters,self.charge[pixel],self.histo[pixel],**kwargs) - fit,valid = self._run_fit(self.Chi2(pixel),self._parameters,pixels_id = self.pixels_id[pixel],prescan = prescan,**kwargs) - - if valid : - log.info(f"fitted value : {fit.values}") - log.info(f"fitted errors : {fit.errors}") - self._update_parameters_postfit(fit,self._parameters) - self.__gain[pixel,0] = Gain(self.__pp.value,self.__resolution.value,self.__mean.value,self.__n.value) - stat_gain = np.array([Gain(self.__pp.value,random.gauss(self.__resolution.value, self.__resolution.error),random.gauss(self.__mean.value, self.__mean.error),self.__n.value) for i in range(1000)]) - self.__gain[pixel,1] = self.__gain[pixel,0] - np.quantile(stat_gain,0.16) - self.__gain[pixel,2] = np.quantile(stat_gain,0.84) - self.__gain[pixel,0] - - ndof = self.histo[pixel].data[~self.histo[pixel].mask].shape[0] - fit.nfit - self.fill_table(pixel,valid,ndof,fit.fval) - log.info(f"Reconstructed gain is {self.__gain[pixel,0] - self.__gain[pixel,1]:.2f} < {self.__gain[pixel,0]:.2f} < {self.__gain[pixel,0] + self.__gain[pixel,2]:.2f}") - log.info(f"Likelihood value = {fit.fval}") - self._output_table['gain'][pixel] = self.__gain[pixel,0] - self._output_table['gain_error'][pixel][0] = self.__gain[pixel,1] - self._output_table['gain_error'][pixel][1] = self.__gain[pixel,2] - - - - if kwargs.get('figpath',0) != 0 : - fig,ax = plt.subplots(1,1,figsize=(8, 8)) - ax.errorbar(self.charge[pixel],self.histo[pixel],np.sqrt(self.histo[pixel]),zorder=0,fmt=".",label = "data") - ax.plot(self.charge[pixel], - np.trapz(self.histo[pixel],self.charge[pixel])*MPE2(self.charge[pixel],self.__pp.value,self.__resolution.value,self.__mean.value,self.__n.value,self.pedestal.value,self.__pedestalWidth.value,self.__luminosity.value), - zorder=1, - linewidth=2, - label = f"SPE model fit \n gain : {self.__gain[pixel,0] - self.__gain[pixel,1]:.2f} < {self.__gain[pixel,0]:.2f} < {self.__gain[pixel,0] + self.__gain[pixel,2]:.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval):.3f},\n likelihood = {fit.fval:.2f}") - ax.set_xlabel("Charge (ADC)", size=15) - ax.set_ylabel("Events", size=15) - ax.set_title(f"SPE fit pixel : {pixel} (pixel id : {self.pixels_id[pixel]})") - ax.set_xlim([self.pedestal.value - 6 * self.pedestalWidth.value, None]) - ax.legend(fontsize=18) - os.makedirs(kwargs.get('figpath'),exist_ok = True) - fig.savefig(f"{kwargs.get('figpath')}/fit_SPE_pixel{self.pixels_id[pixel]}.pdf") - fig.clf() - plt.close(fig) - del fig,ax - else : - log.warning(f"fit pixel_id = {self.pixels_id[pixel]} is not valid") - self.fill_table(pixel,valid,0,0) - - @classmethod - def NG_Likelihood_Chi2(cls,pp,res,mu2,n,muped,sigped,lum,charge,histo,**kwargs): - pdf = MPE2(charge,pp,res,mu2,n,muped,sigped,lum,**kwargs) - #log.debug(f"pdf : {np.sum(pdf)}") - Ntot = np.sum(histo) - #log.debug(f'Ntot : {Ntot}') - mask = histo > 0 - Lik = np.sum(((pdf*Ntot-histo)[mask])**2/histo[mask]) #2 times faster - return Lik - - def Chi2(self,pixel : int): - def _Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity) : - #assert not(np.isnan(pp) or np.isnan(resolution) or np.isnan(mean) or np.isnan(n) or np.isnan(pedestal) or np.isnan(pedestalWidth) or np.isnan(luminosity)) - if self.__old_lum != luminosity : - for i in range(1000): - if (gammainc(i+1,luminosity) < 1e-5): - self.__old_ntotalPE = i - break - self.__old_lum = luminosity - kwargs = {"ntotalPE" : self.__old_ntotalPE} - - return self.NG_Likelihood_Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity,self.charge[pixel],self.histo[pixel],**kwargs) - return _Chi2 - - def Chi2_static(self,pixel : int) : - charge = copy.deepcopy(self.charge[pixel].data[~self.charge[pixel].mask]) - histo = copy.deepcopy(self.histo[pixel].data[~self.histo[pixel].mask]) - def _Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity) : - #assert not(np.isnan(pp) or np.isnan(resolution) or np.isnan(mean) or np.isnan(n) or np.isnan(pedestal) or np.isnan(pedestalWidth) or np.isnan(luminosity)) - ntotalPE = 0 - for i in range(1000): - if (gammainc(i+1,luminosity) < 1e-5): - ntotalPE = i - break - kwargs = {"ntotalPE" : ntotalPE} - return __class__.NG_Likelihood_Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity,charge,histo,**kwargs) - return _Chi2 - - @classmethod - def _update_parameters_prefit_static(cls,it : int, parameters : Parameters, charge : np.ndarray, histo : np.ndarray,**kwargs) : - super(__class__, cls)._update_parameters_prefit_static(it,parameters,charge,histo,**kwargs) - pedestal = parameters['pedestal'] - pedestalWidth = parameters["pedestalWidth"] - pedestalWidth.value = pedestal.max - pedestal.value - pedestalWidth.max = 3 * pedestalWidth.value - log.debug(f"pedestalWidth updated : {pedestalWidth}") - try : - coeff,var_matrix = NectarGainSPE._get_mean_gaussian_fit(charge,histo,f'{it}_nominal') - if (coeff[1] - pedestal.value < 0) or ((coeff[1] - coeff[2]) - pedestal.max < 0) : raise Exception("mean gaussian fit not good") - mean = parameters['mean'] - mean.value = coeff[1] - pedestal.value - mean.min = (coeff[1] - coeff[2]) - pedestal.max - mean.max = (coeff[1] + coeff[2]) - pedestal.min - log.debug(f"mean updated : {mean}") - except Exception as e : - log.warning(e,exc_info=True) - log.warning("mean parameters limits and starting value not changed") - - #fit parameters - @property - def pp(self) : return self.__pp - @property - def luminosity(self) : return self.__luminosity - @property - def mean(self) : return self.__mean - @property - def n(self) : return self.__n - @property - def resolution(self) : return self.__resolution - @property - def pedestalWidth(self) : return self.__pedestalWidth - @property - def gain(self) : return self.__gain - - - #intern parameters - @property - def _old_lum(self) : return self.__old_lum - @_old_lum.setter - def _old_lum(self,value) : self.__old_lum = value - @property - def _old_ntotalPE(self) : return self.__old_ntotalPE - @_old_ntotalPE.setter - def _old_ntotalPE(self,value) : self.__old_ntotalPE = value - - - -class NectarGainSPESingleSignalStd(NectarGainSPESingleSignal): - """class to perform fit of the SPE signal with n and pp fixed""" - __parameters_file = 'parameters_signalStd.yaml' - - #to heavy - #class Chi2() : - # def __init__(self,pixel : int,NectarGainSPESingleSignalStd) : - # self._pixel = pixel - # self.NectarGainSPESingleSignalStd = NectarGainSPESingleSignalStd - # def __call__(self,resolution,mean,pedestal,pedestalWidth,luminosity): - # return super(NectarGainSPESingleSignalStd,self).NG_Likelihood_Chi2(self.NectarGainSPESingleSignalStd.pp.value,resolution,mean,self.NectarGainSPESingleSignalStd.n.value,pedestal,pedestalWidth,luminosity,self.NectarGainSPESingleSignalStd.charge[self._pixel],self.NectarGainSPESingleSignalStd.histo[self._pixel]) - - - - def __init__(self,signal : ChargeContainer,parameters_file = None,**kwargs): - if parameters_file is None : - parameters_file = __class__.__parameters_file - super().__init__(signal,parameters_file,**kwargs) - - self.__fix_parameters() - - self._make_minuit_parameters() - - def __fix_parameters(self) : - """this method should be used to fix n and pp if this hypothesis is valid - """ - log.info("updating parameters by fixing pp and n") - self.pp.frozen = True - self.n.frozen = True - - def Chi2(self,pixel : int): - def _Chi2(resolution,mean,pedestal,pedestalWidth,luminosity) : - if self._old_lum != luminosity : - for i in range(1000): - if (gammainc(i+1,luminosity) < 1e-5): - self._old_ntotalPE = i - break - self._old_lum = luminosity - kwargs = {"ntotalPE" : self._old_ntotalPE} - - return self.NG_Likelihood_Chi2(self.pp.value,resolution,mean,self.n.value,pedestal,pedestalWidth,luminosity,self.charge[pixel],self.histo[pixel],**kwargs) - return _Chi2 - - def Chi2_static(self, pixel : int) : - pp = copy.deepcopy(self.pp) - n = copy.deepcopy(self.n) - charge = copy.deepcopy(self.charge[pixel].data[~self.charge[pixel].mask]) - histo = copy.deepcopy(self.histo[pixel].data[~self.histo[pixel].mask]) - def _Chi2(resolution,mean,pedestal,pedestalWidth,luminosity) : - ntotalPE = 0 - for i in range(1000): - if (gammainc(i+1,luminosity) < 1e-5): - ntotalPE = i - break - kwargs = {"ntotalPE" : ntotalPE} - return __class__.NG_Likelihood_Chi2(pp.value,resolution,mean,n.value,pedestal,pedestalWidth,luminosity,charge,histo,**kwargs) - return _Chi2 - - -class NectarGainSPESingleSignalfromHHVFit(NectarGainSPESingleSignal): - """class to perform fit of the SPE signal at nominal voltage from fitted data obtained with 1400V run - Thus, n, pp and res are fixed""" - __parameters_file = 'parameters_signal_fromHHVFit.yaml' - - def __init__(self,signal : ChargeContainer, nectarGainSPEresult, same_luminosity : bool = True, parameters_file = None,**kwargs): - if parameters_file is None : - parameters_file = __class__.__parameters_file - super().__init__(signal,parameters_file,**kwargs) - - self.__fix_parameters(same_luminosity) - - self.__same_luminosity = same_luminosity - - self.__nectarGainSPEresult = QTable.read(nectarGainSPEresult,format = "ascii.ecsv") - - self._make_minuit_parameters() - - def __fix_parameters(self, same_luminosity : bool) : - """this method should be used to fix n, pp and res - """ - log.info("updating parameters by fixing pp, n and res") - self.pp.frozen = True - self.n.frozen = True - self.resolution.frozen = True - if same_luminosity : - self.luminosity.frozen = True - - - def Chi2(self,pixel : int): - if self.__same_luminosity : - def _Chi2(mean,pedestal,pedestalWidth) : - if self._old_lum != self.__nectarGainSPEresult[self.__pixel_index(pixel)]['luminosity'].value : - for i in range(1000): - if (gammainc(i+1,self.__nectarGainSPEresult[self.__pixel_index(pixel)]['luminosity'].value) < 1e-5): - self._old_ntotalPE = i - break - self._old_lum = self.__nectarGainSPEresult[self.__pixel_index(pixel)]['luminosity'].value - kwargs = {"ntotalPE" : self._old_ntotalPE} - - return self.NG_Likelihood_Chi2(self.__nectarGainSPEresult[self.__pixel_index(pixel)]['pp'].value,self.__nectarGainSPEresult[self.__pixel_index(pixel)]['resolution'].value,mean,self.__nectarGainSPEresult[self.__pixel_index(pixel)]['n'].value,pedestal,pedestalWidth,self.__nectarGainSPEresult[self.__pixel_index(pixel)]['luminosity'],self.charge[pixel],self.histo[pixel],**kwargs) - return _Chi2 - else : - def _Chi2(mean,pedestal,pedestalWidth,luminosity) : - if self._old_lum != luminosity : - for i in range(1000): - if (gammainc(i+1,luminosity) < 1e-5): - self._old_ntotalPE = i - break - self._old_lum = luminosity - kwargs = {"ntotalPE" : self._old_ntotalPE} - return self.NG_Likelihood_Chi2(self.__nectarGainSPEresult[pixel]['pp'].value,self.__nectarGainSPEresult[pixel]['resolution'].value,mean,self.__nectarGainSPEresult[pixel]['n'].value,pedestal,pedestalWidth,luminosity,self.charge[pixel],self.histo[pixel],**kwargs) - return _Chi2 - - def Chi2_static(self,pixel : int): - pp_value = copy.deepcopy(self.__nectarGainSPEresult[pixel]['pp'].value) - resolution_value = copy.deepcopy(self.__nectarGainSPEresult[pixel]['resolution'].value) - n_value = copy.deepcopy(self.__nectarGainSPEresult[pixel]['n'].value) - charge = copy.deepcopy(self.charge[pixel].data[~self.charge[pixel].mask]) - histo = copy.deepcopy(self.histo[pixel].data[~self.histo[pixel].mask]) - if self.__same_luminosity : - luminosity_value = copy.deepcopy(self.__nectarGainSPEresult[pixel]['luminosity'].value) - def _Chi2(mean,pedestal,pedestalWidth) : - ntotalPE = 0 - for i in range(1000): - if (gammainc(i+1,luminosity_value) < 1e-5): - ntotalPE = i - break - kwargs = {"ntotalPE" : ntotalPE} - return self.NG_Likelihood_Chi2(pp_value,resolution_value,mean,n_value,pedestal,pedestalWidth,luminosity_value,charge,histo,**kwargs) - else : - def _Chi2(mean,pedestal,pedestalWidth,luminosity) : - ntotalPE = 0 - for i in range(1000): - if (gammainc(i+1,luminosity) < 1e-5): - ntotalPE = i - break - kwargs = {"ntotalPE" : ntotalPE} - return self.NG_Likelihood_Chi2(pp_value,resolution_value,mean,n_value,pedestal,pedestalWidth,luminosity,charge,histo,**kwargs) - return _Chi2 - - @classmethod - def _update_parameters_prefit_static(cls,it : int, parameters : Parameters, charge : np.ndarray, histo : np.ndarray,**kwargs) : - super(__class__,cls)._update_parameters_prefit_static(it,parameters, charge, histo,**kwargs) - - nectarGainSPEresult = kwargs.get('nectarGainSPEresult') - pixel_id = kwargs.get('pixel_id') - - resolution = parameters["resolution"] - resolution.value = nectarGainSPEresult[nectarGainSPEresult['pixel'] == pixel_id]['resolution'].value - resolution.error = nectarGainSPEresult[nectarGainSPEresult['pixel'] == pixel_id]['resolution_error'].value - - pp = parameters["pp"] - pp.value = nectarGainSPEresult[nectarGainSPEresult['pixel'] == pixel_id]['pp'].value - pp.error = nectarGainSPEresult[nectarGainSPEresult['pixel'] == pixel_id]['pp_error'].value - - n = parameters["n"] - n.value = nectarGainSPEresult[nectarGainSPEresult['pixel'] == pixel_id]['n'].value - n.error = nectarGainSPEresult[nectarGainSPEresult['pixel'] == pixel_id]['n_error'].value - - if kwargs.get('same_luminosity', False): - luminosity = parameters["luminosity"] - luminosity.value = nectarGainSPEresult[nectarGainSPEresult['pixel'] == pixel_id]['luminosity'].value - luminosity.error = nectarGainSPEresult[nectarGainSPEresult['pixel'] == pixel_id]['luminosity_error'].value - - def __pixel_index(self,pixel) : - return np.argmax(self._nectarGainSPEresult['pixel'] == self.pixels_id[pixel]) - - - def run(self,pixel : int = None,multiproc = False, **kwargs): - kwargs['nectarGainSPEresult'] = copy.deepcopy(self.__nectarGainSPEresult) - if multiproc : - kwargs['same_luminosity'] = self.__same_luminosity - super().run(pixel,multiproc,**kwargs) - - def _run_obs(self,pixel,prescan = False,**kwargs) : - if self.__nectarGainSPEresult[pixel]['is_valid'] : - kwargs['pixel_id'] = self.pixels_id[pixel] - super()._run_obs(pixel,prescan,**kwargs) - else : - log.warning(f"fit pixel {pixel} with pixel_id = {self.pixels_id[pixel]} is not valid") - self.fill_table(pixel,False,0,0) - - @classmethod - def _run_obs_static(cls,it : int, funct,parameters: Parameters, pixels_id : int, charge : np.ndarray, histo : np.ndarray, prescan = False, **kwargs) -> dict : - if 'nectarGainSPEresult' in kwargs.keys() and kwargs['nectarGainSPEresult'][kwargs["nectarGainSPEresult"]["pixel"] == pixels_id]['is_valid'].value : - kwargs['pixel_id'] = pixels_id - #__class__._update_parameters_prefit_static(it,parameters, charge, histo,**kwargs) - output = super(__class__,cls)._run_obs_static(it, funct,parameters, pixels_id, charge, histo, prescan = prescan, **kwargs) - - else : - log.warning(f"fit pixel {it} with pixel_id = {pixels_id} is not valid") - output = {"is_valid" : False, "pixel" : pixels_id} - for parameter in parameters.parameters : - output[parameter.name] = parameter.value - output[f"{parameter.name}_error"] = parameter.error - return output - - @property - def _nectarGainSPEresult(self) : return self.__nectarGainSPEresult - - -class NectarGainSPESinglePed(NectarGainSPESingle): - """class to perform fit of the pedestal""" - - __parameters_file = 'parameters_ped.yaml' - #def __new__(cls) : - # print("NectarGainSPESingleSignal is not instanciable") - # return 0 - def __init__(self,signal : ChargeContainer,parameters_file = None,**kwargs) : - super().__init__(signal,**kwargs) - self.__pedestalFitted = np.empty((self.npixels,2)) - #if parameters file is provided - if parameters_file is None : - parameters_file = __class__.__parameters_file - with open(f"{os.path.dirname(os.path.abspath(__file__))}/{parameters_file}") as parameters : - param = yaml.safe_load(parameters) - self.__pedestalWidth = Parameter(name = "pedestalWidth", - value = param["pedestalWidth"]["value"], - min = param["pedestalWidth"].get("min",np.nan), - max = param["pedestalWidth"].get("max",np.nan), - unit = param["pedestalWidth"].get("unit",u.dimensionless_unscaled)) - self._parameters.append(self.__pedestalWidth) - - self.create_output_table() - - self._output_table.remove_column('gain') - self._output_table.remove_column('gain_error') - - - self._make_minuit_parameters() - - def _run_obs(self,pixel,prescan = False,**kwargs) : - self._update_parameters_prefit_static(pixel, self._parameters,self.charge[pixel],self.histo[pixel]) - fit,valid = self._run_fit(self.Chi2(pixel),self._parameters,pixels_id = self.pixels_id[pixel],prescan = prescan,**kwargs) - - if valid : - log.info(f"fitted value : {fit.values}") - log.info(f"fitted errors : {fit.errors}") - self._update_parameters_postfit(fit,self._parameters) - self.__pedestalFitted[pixel,0] = self.pedestal.value - self.__pedestalFitted[pixel,1] = self.pedestal.error - - log.info(f"pedestal is {self.__pedestalFitted[pixel,0]:.2f} +/- {self.__pedestalFitted[pixel,1]:.2f}") - log.info(f"Likelihood value = {fit.fval}") - - ndof = self.histo[pixel].data[~self.histo[pixel].mask].shape[0] - fit.nfit - self.fill_table(pixel,valid,ndof,fit.fval) - - if kwargs.get('figpath',0) != 0 : - fig,ax = plt.subplots(1,1,figsize=(8, 6)) - ax.errorbar(self.charge[pixel],self.histo[pixel],np.sqrt(self.histo[pixel]),zorder=0,fmt=".",label = "data") - ax.plot(self.charge[pixel], - np.trapz(self.histo[pixel],self.charge[pixel])*gaussian(self.charge[pixel],self.pedestal.value,self.__pedestalWidth.value), - zorder=1, - linewidth=2, - label = f"MPE model fit \n Pedestal = {round(self.__pedestalFitted[pixel,0]):.2f} +/- {round(self.__pedestalFitted[pixel,1],2):.2f} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval):.3f},\n likelihood = {fit.fval:.2f}") - ax.set_xlabel("Charge (ADC)", size=15) - ax.set_ylabel("Events", size=15) - ax.set_title(f"Pedestal fit pixel : {pixel} (pixel id : {self.pixels_id[pixel]})") - ax.legend(fontsize=15) - os.makedirs(kwargs.get('figpath'),exist_ok = True) - fig.savefig(f"{kwargs.get('figpath')}/fit_Ped_pixel{pixel}.pdf") - fig.clf() - plt.close(fig) - del fig,ax - else : - log.warning(f"fit pixel_id = {self.pixels_id[pixel]} is not valid") - self.fill_table(pixel,valid,0,0) - - @classmethod - def _run_obs_static(cls,it : int, funct,parameters : Parameters, pixels_id : int, charge : np.ndarray, histo : np.ndarray, prescan = False, **kwargs) : - cls._update_parameters_prefit_static(it,parameters,charge,histo) - fit,valid = __class__._run_fit(funct,parameters,pixels_id = pixels_id,prescan = prescan,**kwargs) - - if valid : - log.info(f"fitted value : {fit.values}") - log.info(f"fitted errors : {fit.errors}") - __class__._update_parameters_postfit(fit,parameters) - - ndof = histo.data[~histo.mask].shape[0] - fit.nfit - output = __class__._make_output_dict_obs(fit,valid,pixels_id,parameters,ndof) - - pedestalFitted = np.empty(2) - pedestalFitted[0] = parameters["pedestal"].value - pedestalFitted[1] = parameters["pedestal"].error - - log.info(f"pedestal is {pedestalFitted[0]:.2f} +/- {pedestalFitted[1]:.2}") - log.info(f"Likelihood value = {fit.fval}") - - - output['pedestalFitted'] = pedestalFitted[0] - output['pedestalFitted_error'] = pedestalFitted[1] - - if kwargs.get('figpath',0) != 0 : - fig,ax = plt.subplots(1,1,figsize=(8, 6)) - ax.errorbar(charge, histo, np.sqrt(histo),zorder=0,fmt=".",label = "data") - ax.plot(charge, - np.trapz(histo, charge)*gaussian(charge, parameters["pedestal"].value,parameters["pedestalWidth"].value), - zorder=1, - linewidth=2, - label = f"MPE model fit \n Pedestal = {round(parameters['pedestal'].value):.2f} +/- {round(parameters['pedestal'].error,2):.2e} ADC/pe, pvalue = {Statistics.chi2_pvalue(ndof,fit.fval):.3f},\n likelihood = {fit.fval:.2f}") - ax.set_xlabel("Charge (ADC)", size=15) - ax.set_ylabel("Events", size=15) - ax.set_title(f"Pedestal fit pixel_id = {pixels_id})") - ax.legend(fontsize=15) - os.makedirs(kwargs.get('figpath'),exist_ok = True) - fig.savefig(f"{kwargs.get('figpath')}/fit_Ped_pixel{pixels_id}.pdf") - fig.clf() - plt.close(fig) - del fig,ax - else : - log.warning(f"fit pixel_id = {pixels_id} is not valid") - output = __class__._make_output_dict_obs(fit,valid,pixels_id,parameters) - - return output - - @classmethod - def NG_Likelihood_Chi2(cls,muped,sigped,charge,histo,**kwargs): - Lik = 0 - Ntot = np.sum(histo) - mask = histo > 0 - Lik = np.sum((((gaussian(charge,muped,sigped)*Ntot - histo)[mask])**2)/histo[mask]) - return Lik - - def Chi2(self,pixel : int): - def _Chi2(pedestal,pedestalWidth) : - return self.NG_Likelihood_Chi2(pedestal,pedestalWidth,self.charge[pixel],self.histo[pixel]) - return _Chi2 - - def Chi2_static(self,pixel : int): - charge = copy.deepcopy(self.charge[pixel].data[~self.charge[pixel].mask]) - histo = copy.deepcopy(self.histo[pixel].data[~self.histo[pixel].mask]) - def _Chi2(pedestal,pedestalWidth) : - return self.NG_Likelihood_Chi2(pedestal,pedestalWidth,charge,histo) - return _Chi2 - - - @property - def pedestalWidth(self) : return self.__pedestalWidth \ No newline at end of file diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/__init__.py b/src/nectarchain/calibration/NectarGain/SPEfit/__init__.py deleted file mode 100644 index 178bb295..00000000 --- a/src/nectarchain/calibration/NectarGain/SPEfit/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .NectarGainSPE_singlerun import * -from .NectarGainSPE_combined import * diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/parameters.py b/src/nectarchain/calibration/NectarGain/SPEfit/parameters.py deleted file mode 100644 index b01e48f8..00000000 --- a/src/nectarchain/calibration/NectarGain/SPEfit/parameters.py +++ /dev/null @@ -1,104 +0,0 @@ -import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') -log = logging.getLogger(__name__) - -import copy -import numpy as np - -import astropy.units as u - -__all__ = ["Parameter","Parameters"] - -class Parameter() : - def __init__(self, name, value, min = np.nan, max = np.nan, error = np.nan, unit = u.dimensionless_unscaled, frozen : bool = False): - self.__name = name - self.__value = value - self.__error = error - self.__min = min - self.__max = max - self.__unit = unit - self.__frozen = frozen - - @classmethod - def from_instance(cls,parameter): - return cls(parameter.name,parameter.value,parameter.min,parameter.max,parameter.unit,parameter.frozen) - - def __str__(self): - return f"name : {self.__name}, value : {self.__value}, error : {self.__error}, unit : {self.__unit}, min : {self.__min}, max : {self.__max},frozen : {self.__frozen}" - - - @property - def name(self) : return self.__name - @name.setter - def name(self,value) : self.__name = value - - @property - def value(self) : return self.__value - @value.setter - def value(self,value) : self.__value = value - - @property - def min(self): return self.__min - @min.setter - def min(self,value) : self.__min = value - - @property - def max(self): return self.__max - @max.setter - def max(self,value) : self.__max = value - - @property - def unit(self) : return self.__unit - @unit.setter - def unit(self,value) : self.__unit = value - - @property - def error(self) : return self.__error - @error.setter - def error(self,value) : self.__error = value - - @property - def frozen(self) : return self.__frozen - @frozen.setter - def frozen(self,value : bool) : self.__frozen = value - -class Parameters() : - def __init__(self,parameters_liste : list = []) -> None: - self.__parameters = copy.deepcopy(parameters_liste) - - - def append(self,parameter : Parameter) : - self.__parameters.append(parameter) - - def __getitem__(self,key) : - for parameter in self.__parameters : - if parameter.name == key : - return parameter - return [] - - def __str__(self): - string="" - for parameter in self.__parameters : - string += str(parameter)+"\n" - return string - - - @property - def parameters(self) : return self.__parameters - - @property - def size(self) : return len(self.__parameters) - - @property - def parnames(self) : return [parameter.name for parameter in self.__parameters] - - @property - def parvalues(self) : return [parameter.value for parameter in self.__parameters] - - @property - def unfrozen(self) : - parameters = Parameters() - for parameter in self.__parameters : - if not(parameter.frozen) : - parameters.append(parameter) - return parameters diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_ped.yaml b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_ped.yaml deleted file mode 100644 index e1fab9db..00000000 --- a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_ped.yaml +++ /dev/null @@ -1,7 +0,0 @@ -{ - pedestalWidth : { - value : 50, - min : 1, - max : 150 - } -} \ No newline at end of file diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal.yaml b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal.yaml deleted file mode 100644 index 89ece153..00000000 --- a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{ - pedestalWidth : { - value : 50, - min : 1, - max : 150 - }, - luminosity : { - value : 1, - min : 0.01, - max : 5.0 - }, - pp : { - value : 0.45, - min : 0.2, - max : 0.8 - }, - resolution : { - value : 0.5, - min : 0.3, - max : 0.7 - }, - n : { - value : 0.697, - min : 0.5, - max : 0.9 - }, - mean : { - value : 500, - min : 200, - max : 600 - } -} \ No newline at end of file diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signalStd.yaml b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signalStd.yaml deleted file mode 100644 index 12746d7f..00000000 --- a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signalStd.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{ - pedestalWidth : { - value : 50, - min : 1, - max : 150 - }, - luminosity : { - value : 1, - min : 0.01, - max : 5.0 - }, - pp : { - value : 0.45, - min : .NAN, - max : .NAN - }, - resolution : { - value : 0.5, - min : 0.3, - max : 0.7 - }, - n : { - value : 0.697, - min : .NAN, - max : .NAN - }, - mean : { - value : 500, - min : 200, - max : 600 - } -} \ No newline at end of file diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_combined.yaml b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_combined.yaml deleted file mode 100644 index a04fc05a..00000000 --- a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_combined.yaml +++ /dev/null @@ -1,42 +0,0 @@ -{ - pedestalWidth : { - value : 50, - min : 1, - max : 150 - }, - luminosity : { - value : 1, - min : 0.01, - max : 5.0 - }, - luminosityHHV : { - value : 1, - min : 0.01, - max : 5.0 - }, - pp : { - value : 0.45, - min : .NAN, - max : .NAN - }, - resolution : { - value : 0.5, - min : 0.3, - max : 0.7 - }, - n : { - value : 0.697, - min : .NAN, - max : .NAN - }, - mean : { - value : 50, - min : 10, - max : 200 - }, - meanHHV : { - value : 500, - min : 200, - max : 600 - } -} \ No newline at end of file diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_fromHHVFit.yaml b/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_fromHHVFit.yaml deleted file mode 100644 index 62515304..00000000 --- a/src/nectarchain/calibration/NectarGain/SPEfit/parameters_signal_fromHHVFit.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{ - pedestalWidth : { - value : 50, - min : 1, - max : 150 - }, - luminosity : { - value : 1, - min : 0.01, - max : 5.0 - }, - pp : { - value : 0.45, - min : .NAN, - max : .NAN - }, - resolution : { - value : 0.5, - min : .NAN, - max : .NAN - }, - n : { - value : 0.697, - min : .NAN, - max : .NAN - }, - mean : { - value : 50, - min : 10, - max : 200 - } -} \ No newline at end of file diff --git a/src/nectarchain/calibration/NectarGain/SPEfit/utils.py b/src/nectarchain/calibration/NectarGain/SPEfit/utils.py deleted file mode 100644 index 44686680..00000000 --- a/src/nectarchain/calibration/NectarGain/SPEfit/utils.py +++ /dev/null @@ -1,298 +0,0 @@ -import logging -import math - -import numpy as np -from iminuit import Minuit -from scipy import interpolate, signal -from scipy.special import gammainc -from scipy.stats import norm,chi2 - -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') -log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers - -__all__ = ['UtilsMinuit','Multiprocessing'] -from .parameters import Parameters - -class Multiprocessing() : - @staticmethod - def custom_error_callback(error): - log.error(f'Got an error: {error}') - log.error(error,exc_info=True) - -class Statistics() : - @staticmethod - def chi2_pvalue(ndof : int, likelihood : float) : - return 1 - chi2(df = ndof).cdf(likelihood) - -class UtilsMinuit() : - @staticmethod - def make_minuit_par_kwargs(parameters : Parameters): - """Create *Parameter Keyword Arguments* for the `Minuit` constructor. - - updated for Minuit >2.0 - """ - names = parameters.parnames - kwargs = {"names": names,"values" : {}} - - for parameter in parameters.parameters: - kwargs["values"][parameter.name] = parameter.value - min_ = None if np.isnan(parameter.min) else parameter.min - max_ = None if np.isnan(parameter.max) else parameter.max - error = 0.1 if np.isnan(parameter.error) else parameter.error - kwargs[f"limit_{parameter.name}"] = (min_, max_) - kwargs[f"error_{parameter.name}"] = error - if parameter.frozen : - kwargs[f"fix_{parameter.name}"] = True - return kwargs - - @staticmethod - def set_minuit_parameters_limits_and_errors(m : Minuit,parameters : dict) : - """function to set minuit parameter limits and errors with Minuit >2.0 - - Args: - m (Minuit): a Minuit instance - parameters (dict): dict containing parameters names, limits errors and values - """ - for name in parameters["names"] : - m.limits[name] = parameters[f"limit_{name}"] - m.errors[name] = parameters[f"error_{name}"] - if parameters.get(f"fix_{name}",False) : - m.fixed[name] = True - - -# Usefull fucntions for the fit -def gaussian(x, mu, sig): - #return (1./(sig*np.sqrt(2*math.pi)))*np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.))) - return norm.pdf(x,loc = mu,scale = sig) - -def weight_gaussian(x,N, mu, sig) : - return N * gaussian(x, mu, sig) - -def doubleGauss(x,sig1,mu2,sig2,p): - return p *2 *gaussian(x, 0, sig1) + (1-p) * gaussian(x, mu2, sig2) - -def PMax(r): - """p_{max} in equation 6 in Caroff et al. (2019) - - Args: - r (float): SPE resolution - - Returns: - float : p_{max} - """ - if r > np.sqrt((np.pi -2 )/ 2) : - pmax = np.pi/(2 * (r**2 + 1)) - else : - pmax = np.pi*r**2/(np.pi*r**2 + np.pi - 2*r**2 - 2) - return pmax - -def ax(p,res): - """a in equation 4 in Caroff et al. (2019) - - Args: - p (float): proportion of the low charge component (2 gaussians model) - res (float): SPE resolution - - Returns: - float : a - """ - return ((2/np.pi)*p**2-p/(res**2+1)) - -def bx(p,mu2): - """b in equation 4 in Caroff et al. (2019) - - Args: - p (float): proportion of the low charge component (2 gaussians model) - mu2 (float): position of the high charge Gaussian - - Returns: - float : b - """ - return (np.sqrt(2/np.pi)*2*p*(1-p)*mu2) - -def cx(sig2,mu2,res,p): - """c in equation 4 in Caroff et al. (2019) - Note : There is a typo in the article 1-p**2 -> (1-p)**2 - - Args: - sig2 (float): width of the high charge Gaussian - mu2 (float): position of the high charge Gaussian - res (float): SPE resolution - p (float): proportion of the low charge component (2 gaussians model) - - Returns: - float : c - """ - return (1-p)**2*mu2**2 - (1-p)*(sig2**2+mu2**2)/(res**2+1) - -def delta(p,res,sig2,mu2): - """well known delta in 2nd order polynom - - Args: - p (_type_): _description_ - res (_type_): _description_ - sig2 (_type_): _description_ - mu2 (_type_): _description_ - - Returns: - float : b**2 - 4*a*c - """ - return bx(p,mu2)*bx(p,mu2) - 4*ax(p,res)*cx(sig2,mu2,res,p) - -def ParamU(p,r): - """d in equation 6 in Caroff et al. (2019) - - Args: - p (float): proportion of the low charge component (2 gaussians model) - r (float): SPE resolution - - Returns: - float : d - """ - return ((8*(1-p)**2*p**2)/np.pi - 4*(2*p**2/np.pi - p/(r**2+1))*((1-p)**2-(1-p)/(r**2+1))) - -def ParamS(p,r): - """e in equation 6 in Caroff et al. (2019) - - Args: - p (float): proportion of the low charge component (2 gaussians model) - r (float): SPE resolution - - Returns: - float : e - """ - e = (4*(2*p**2/np.pi - p/(r**2+1))*(1-p))/(r**2+1) - return e - -def SigMin(p,res,mu2): - """sigma_{high,min} in equation 6 in Caroff et al. (2019) - - Args: - p (float): proportion of the low charge component (2 gaussians model) - res (float): SPE resolution - mu2 (float): position of the high charge Gaussian - - Returns: - float : sigma_{high,min} - """ - return mu2*np.sqrt((-ParamU(p,res)+(bx(p,mu2)**2/mu2**2))/(ParamS(p,res))) - -def SigMax(p,res,mu2): - """sigma_{high,min} in equation 6 in Caroff et al. (2019) - - Args: - p (float): proportion of the low charge component (2 gaussians model) - res (float): SPE resolution - mu2 (float): position of the high charge Gaussian - - Returns: - float : sigma_{high,min} - """ - temp = (-ParamU(p,res))/(ParamS(p,res)) - if temp < 0 : - err = ValueError("-d/e must be < 0") - log.error(err,exc_info=True) - raise err - else : - return mu2*np.sqrt(temp) - -def sigma1(p,res,sig2,mu2): - """sigma_{low} in equation 5 in Caroff et al. (2019) - - Args: - sig2 (float): width of the high charge Gaussian - mu2 (float): position of the high charge Gaussian - res (float): SPE resolution - p (float): proportion of the low charge component (2 gaussians model) - - Returns: - float : sigma_{low} - """ - return (-bx(p,mu2)+np.sqrt(delta(p,res,sig2,mu2)))/(2*ax(p,res)) - -def sigma2(n,p,res,mu2): - """sigma_{high} in equation 7 in Caroff et al. (2019) - - Args: - n (float): parameter n in equation - mu2 (float): position of the high charge Gaussian - res (float): SPE resolution - p (float): proportion of the low charge component (2 gaussians model) - - Returns: - float : sigma_{high} - """ - if ((-ParamU(p,res)+(bx(p,mu2)**2/mu2**2))/(ParamS(p,res)) > 0): - return SigMin(p,res,mu2)+n*(SigMax(p,res,mu2)-SigMin(p,res,mu2)) - else: - return n*SigMax(p,res,mu2) - - # The real final model callign all the above for luminosity (lum) + PED, wil return probability of number of Spe -def MPE2(x,pp,res,mu2,n,muped,sigped,lum,**kwargs): - log.debug(f"pp = {pp}, res = {res}, mu2 = {mu2}, n = {n}, muped = {muped}, sigped = {sigped}, lum = {lum}") - f = 0 - ntotalPE = kwargs.get("ntotalPE",0) - if ntotalPE == 0 : - #about 1sec - for i in range(1000): - if (gammainc(i+1,lum) < 1e-5): - ntotalPE = i - break - #print(ntotalPE) - #about 8 sec, 1 sec by nPEPDF call - #for i in range(ntotalPE): - # f = f + ((lum**i)/math.factorial(i)) * np.exp(-lum) * nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) - - f = np.sum([(lum**i)/math.factorial(i) * np.exp(-lum) * nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) for i in range(ntotalPE)],axis = 0) # 10 % faster - return f - -# Fnal model shape/function (for one SPE) -def doubleGaussConstrained(x,pp,res,mu2,n): - p = pp*PMax(res) - sig2 = sigma2(n,p,res,mu2) - sig1 = sigma1(p,res,sig2,mu2) - return doubleGauss(x,sig1,mu2,sig2,p) - -# Get the gain from the parameters model -def Gain(pp,res,mu2,n): - """analytic gain computatuon - - Args: - mu2 (float): position of the high charge Gaussian - res (float): SPE resolution - pp (float): p' in equation 7 in Caroff et al. (2019) - n (float): n in equation 7 in Caroff et al. (2019) - - Returns: - float : gain - """ - p = pp*PMax(res) - sig2 = sigma2(n,p,res,mu2) - return (1-p)*mu2 + 2*p*sigma1(p,res,sig2,mu2)/np.sqrt(2*np.pi) - -def nPEPDF(x,pp,res,mu2,n,muped,sigped,nph,size_charge): - allrange = np.linspace(-1 * size_charge,size_charge,size_charge*2) - spe = [] - #about 2 sec this is the main pb - #for i in range(len(allrange)): - # if (allrange[i]>=0): - # spe.append(doubleGaussConstrained(allrange[i],pp,res,mu2,n)) - # else: - # spe.append(0) - - spe = doubleGaussConstrained(allrange,pp,res,mu2,n) * (allrange>=0 * np.ones(allrange.shape)) #100 times faster - - # ~ plt.plot(allrange,spe) - #npe = semi_gaussian(allrange, muped, sigped) - npe = gaussian(allrange, 0, sigped) - # ~ plt.plot(allrange,npe) - # ~ plt.show() - for i in range(nph): - #npe = np.convolve(npe,spe,"same") - npe = signal.fftconvolve(npe,spe,"same") - # ~ plt.plot(allrange,npe) - # ~ plt.show() - fff = interpolate.UnivariateSpline(allrange,npe,ext=1,k=3,s=0) - norm = np.trapz(fff(allrange),allrange) - return fff(x-muped)/norm \ No newline at end of file diff --git a/src/nectarchain/calibration/NectarGain/__init__.py b/src/nectarchain/calibration/NectarGain/__init__.py deleted file mode 100644 index 4dc3ce8c..00000000 --- a/src/nectarchain/calibration/NectarGain/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .SPEfit import * -from .PhotoStat import * diff --git a/src/nectarchain/calibration/NectarGain/utils/__init__.py b/src/nectarchain/calibration/NectarGain/utils/__init__.py deleted file mode 100644 index 3efdae04..00000000 --- a/src/nectarchain/calibration/NectarGain/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .error import * \ No newline at end of file diff --git a/src/nectarchain/calibration/NectarGain/utils/error.py b/src/nectarchain/calibration/NectarGain/utils/error.py deleted file mode 100644 index 16800a68..00000000 --- a/src/nectarchain/calibration/NectarGain/utils/error.py +++ /dev/null @@ -1,11 +0,0 @@ -import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') -log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers - -class DifferentPixelsID(Exception) : - def __init__(self,message) : - self.__message = message - - @property - def message(self) : return self.__message From 992e26497db40ead0a47d76b7fb9e9f06295fa22 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Tue, 12 Sep 2023 01:06:25 +0200 Subject: [PATCH 32/62] makers : core module enhancement (docstrings, unit test) --- src/nectarchain/calibration/makers/core.py | 130 ++++++++++++++++----- src/nectarchain/calibration/makers/test.py | 58 +++++++++ 2 files changed, 161 insertions(+), 27 deletions(-) create mode 100644 src/nectarchain/calibration/makers/test.py diff --git a/src/nectarchain/calibration/makers/core.py b/src/nectarchain/calibration/makers/core.py index 0213f723..088440e1 100644 --- a/src/nectarchain/calibration/makers/core.py +++ b/src/nectarchain/calibration/makers/core.py @@ -1,66 +1,142 @@ -import sys import logging logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') log = logging.getLogger(__name__) log.handlers = logging.getLogger('__main__').handlers import os from pathlib import Path +import numpy as np -from abc import ABC, ABCMeta, abstractmethod +from abc import ABC, abstractmethod from astropy.table import QTable,Column import astropy.units as u from copy import copy from datetime import date +from collections.abc import Iterable __all__ = [""] +class CalibrationMaker(ABC): + """ + Mother class for all calibration makers that can be defined to compute calibration coefficients from data. -class CalibrationMaker(ABC) : - """mother class for all the calibration makers that can be defined to compute calibration coeficients from data + Attributes: + _reduced_name (str): A string representing the name of the calibration. + PIXELS_ID_COLUMN (str): A string representing the name of the column in the result table that stores the pixels id. + NP_PIXELS (str): A string representing the key in the metadata that stores the number of pixels. + _pixels_id (ndarray): A private property that stores the pixels id. + _results (QTable): A private property that stores the result table. """ + _reduced_name = "Calibration" + PIXELS_ID_COLUMN = "pixels_id" + NP_PIXELS = "npixels" + + def __new__(cls, *args, **kwargs): + """ + Constructor. + + Returns: + CalibrationMaker: An instance of the CalibrationMaker class. + """ + return super(CalibrationMaker, cls).__new__(cls) -#constructors - def __new__(cls,*args,**kwargs) : - return super(CalibrationMaker,cls).__new__(cls) + def __init__(self, pixels_id, *args, **kwargs) -> None: + """ + Initialize the CalibrationMaker object. - def __init__(self,pixels_id,*args,**kwargs) -> None: + Args: + pixels_id (iterable, np.ndarray): The list of pixels id. + """ super().__init__() - self.__pixels_id = pixels_id + if not(isinstance(pixels_id, Iterable)): + raise TypeError("pixels_id must be iterable") + self.__pixels_id = np.array(pixels_id) self.__results = QTable() - self.__results.add_column(Column(self.__pixels_id,"pixels_id",unit = u.dimensionless_unscaled)) - self.__results.meta['npixels'] = self.npixels + self.__results.add_column(Column(self.__pixels_id, __class__.PIXELS_ID_COLUMN, unit=u.dimensionless_unscaled)) + self.__results.meta[__class__.NP_PIXELS] = self.npixels self.__results.meta['comments'] = f'Produced with NectarChain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' -#methods @abstractmethod - def make(self,*args,**kwargs) : + def make(self, *args, **kwargs): + """ + Abstract method that needs to be implemented by subclasses. + Used to compute the calibration coefficients. + """ pass -#I/O method - def save(self,path,**kwargs) : + def save(self, path, **kwargs): + """ + Saves the results to a file in the specified path. + + Args: + path (str): The path to save the results. + **kwargs: Additional keyword arguments. + + Keyword Args: + overwrite (bool): Whether to overwrite an existing file. Defaults to False. + """ path = Path(path) - os.makedirs(path,exist_ok = True) + path.mkdir(parents=True, exist_ok=True) log.info(f'data saved in {path}') - self._results.write(f"{path}/results_{self._reduced_name}.ecsv", format='ascii.ecsv',overwrite = kwargs.get("overwrite",False)) + self._results.write(f"{path}/results_{self._reduced_name}.ecsv", format='ascii.ecsv', overwrite=kwargs.get("overwrite", False)) -#getters and setters @property - def _pixels_id(self) : return self.__pixels_id + def _pixels_id(self): + """ + Get the pixels id. + + Returns: + ndarray: The pixels id. + """ + return self.__pixels_id + @_pixels_id.setter - def _pixels_id(self,value) : self.__pixels_id = value - @property - def pixels_id(self) : return copy(self.__pixels_id) + def _pixels_id(self, value): + """ + Set the pixels id. + + Args: + value (ndarray): The pixels id. + """ + self.__pixels_id = value @property - def npixels(self) : return len(self.__pixels_id) - + def pixels_id(self): + """ + Get a copy of the pixels id. + + Returns: + ndarray: A copy of the pixels id. + """ + return copy(self.__pixels_id) + @property - def _results(self) : return self.__results + def npixels(self): + """ + Get the number of pixels. + + Returns: + int: The number of pixels. + """ + return len(self.__pixels_id) + @property - def results(self) : return copy(self.__results) - + def _results(self): + """ + Get the result table. + Returns: + QTable: The result table. + """ + return self.__results + @property + def results(self): + """ + Get a copy of the result table. + Returns: + QTable: A copy of the result table. + """ + return copy(self.__results) \ No newline at end of file diff --git a/src/nectarchain/calibration/makers/test.py b/src/nectarchain/calibration/makers/test.py new file mode 100644 index 00000000..801d09e0 --- /dev/null +++ b/src/nectarchain/calibration/makers/test.py @@ -0,0 +1,58 @@ +# Generated by CodiumAI +from nectarchain.calibration.makers.core import CalibrationMaker +from ctapipe_io_nectarcam import np +from datetime import date + + +import pytest + +class TestCalibrationMaker: + + # Tests that the constructor initializes the object with the correct attributes and metadata when valid input is provided + def test_constructor_with_valid_input(self): + pixels_id = [1, 2, 3] + calibration_maker = CalibrationMaker(pixels_id) + + assert calibration_maker._pixels_id == np.array(pixels_id) + assert calibration_maker._results[calibration_maker.PIXELS_ID_COLUMN] == np.array(pixels_id) + assert calibration_maker._results.meta[calibration_maker.NP_PIXELS] == len(pixels_id) + assert isinstance(calibration_maker._results.meta['comments'],str) + + + # Tests that the constructor raises an error when a non-iterable pixels_id is provided + def test_constructor_with_non_iterable_pixels_id(self): + pixels_id = 123 + with pytest.raises(TypeError): + CalibrationMaker(pixels_id) + + # Tests that saving the results to a non-existent path raises an error + def test_save_to_nonexistent_path(self): + pixels_id = [1, 2, 3] + calibration_maker = CalibrationMaker(pixels_id) + + with pytest.raises(FileNotFoundError): + calibration_maker.save("nonexistent/path") + + # Tests that saving the results to an existing file with overwrite=False raises an error + def test_save_to_existing_file_with_overwrite_false(self, tmp_path): + pixels_id = [1, 2, 3] + calibration_maker = CalibrationMaker(pixels_id) + + # Create a temporary file + file_path = tmp_path / "results_Calibration.ecsv" + file_path.touch() + + with pytest.raises(FileExistsError): + calibration_maker.save(file_path, overwrite=False) + + # Tests that changing the pixels_id attribute updates the results table with the expected values + def test_change_pixels_id_attribute(self): + pixels_id = [1, 2, 3] + calibration_maker = CalibrationMaker(pixels_id) + + new_pixels_id = [4, 5, 6] + calibration_maker._pixels_id = np.array(new_pixels_id) + + assert calibration_maker._pixels_id == np.array(new_pixels_id) + assert calibration_maker._results[calibration_maker.PIXELS_ID_COLUMN] == np.array(new_pixels_id) + assert calibration_maker._results.meta[calibration_maker.NP_PIXELS] == len(new_pixels_id) \ No newline at end of file From 211ce40171a41c489e8938d98b183e7bd0e707ec Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Tue, 12 Sep 2023 22:04:41 +0200 Subject: [PATCH 33/62] cleaning --- src/nectarchain/calibration/makers/test.py | 58 ---------------------- 1 file changed, 58 deletions(-) delete mode 100644 src/nectarchain/calibration/makers/test.py diff --git a/src/nectarchain/calibration/makers/test.py b/src/nectarchain/calibration/makers/test.py deleted file mode 100644 index 801d09e0..00000000 --- a/src/nectarchain/calibration/makers/test.py +++ /dev/null @@ -1,58 +0,0 @@ -# Generated by CodiumAI -from nectarchain.calibration.makers.core import CalibrationMaker -from ctapipe_io_nectarcam import np -from datetime import date - - -import pytest - -class TestCalibrationMaker: - - # Tests that the constructor initializes the object with the correct attributes and metadata when valid input is provided - def test_constructor_with_valid_input(self): - pixels_id = [1, 2, 3] - calibration_maker = CalibrationMaker(pixels_id) - - assert calibration_maker._pixels_id == np.array(pixels_id) - assert calibration_maker._results[calibration_maker.PIXELS_ID_COLUMN] == np.array(pixels_id) - assert calibration_maker._results.meta[calibration_maker.NP_PIXELS] == len(pixels_id) - assert isinstance(calibration_maker._results.meta['comments'],str) - - - # Tests that the constructor raises an error when a non-iterable pixels_id is provided - def test_constructor_with_non_iterable_pixels_id(self): - pixels_id = 123 - with pytest.raises(TypeError): - CalibrationMaker(pixels_id) - - # Tests that saving the results to a non-existent path raises an error - def test_save_to_nonexistent_path(self): - pixels_id = [1, 2, 3] - calibration_maker = CalibrationMaker(pixels_id) - - with pytest.raises(FileNotFoundError): - calibration_maker.save("nonexistent/path") - - # Tests that saving the results to an existing file with overwrite=False raises an error - def test_save_to_existing_file_with_overwrite_false(self, tmp_path): - pixels_id = [1, 2, 3] - calibration_maker = CalibrationMaker(pixels_id) - - # Create a temporary file - file_path = tmp_path / "results_Calibration.ecsv" - file_path.touch() - - with pytest.raises(FileExistsError): - calibration_maker.save(file_path, overwrite=False) - - # Tests that changing the pixels_id attribute updates the results table with the expected values - def test_change_pixels_id_attribute(self): - pixels_id = [1, 2, 3] - calibration_maker = CalibrationMaker(pixels_id) - - new_pixels_id = [4, 5, 6] - calibration_maker._pixels_id = np.array(new_pixels_id) - - assert calibration_maker._pixels_id == np.array(new_pixels_id) - assert calibration_maker._results[calibration_maker.PIXELS_ID_COLUMN] == np.array(new_pixels_id) - assert calibration_maker._results.meta[calibration_maker.NP_PIXELS] == len(new_pixels_id) \ No newline at end of file From 8de8cffa02c4dd77dacde53b343b95e096460193 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Tue, 12 Sep 2023 22:06:12 +0200 Subject: [PATCH 34/62] moving container and makers in top of calibration directory --- src/nectarchain/{calibration => }/container/__init__.py | 0 src/nectarchain/{calibration => }/container/charge.py | 0 src/nectarchain/{calibration => }/container/charge_extractor.py | 0 src/nectarchain/{calibration => }/container/utils.py | 0 src/nectarchain/{calibration => }/container/waveforms.py | 0 src/nectarchain/{calibration => }/makers/__init__.py | 0 src/nectarchain/{calibration => }/makers/core.py | 0 src/nectarchain/{calibration => }/makers/flatfieldMakers.py | 0 .../{calibration => }/makers/gain/FlatFieldSPEMakers.py | 0 .../{calibration => }/makers/gain/PhotoStatisticMakers.py | 0 .../{calibration => }/makers/gain/WhiteTargetSPEMakers.py | 0 src/nectarchain/{calibration => }/makers/gain/__init__.py | 0 src/nectarchain/{calibration => }/makers/gain/gainMakers.py | 0 src/nectarchain/{calibration => }/makers/gain/parameters.py | 0 .../{calibration => }/makers/gain/parameters_signal.yaml | 0 .../{calibration => }/makers/gain/parameters_signalStd.yaml | 0 .../{calibration => }/makers/gain/parameters_signal_combined.yaml | 0 .../makers/gain/parameters_signal_fromHHVFit.yaml | 0 src/nectarchain/{calibration => }/makers/gain/utils/__init__.py | 0 src/nectarchain/{calibration => }/makers/gain/utils/error.py | 0 src/nectarchain/{calibration => }/makers/gain/utils/utils.py | 0 src/nectarchain/{calibration => }/makers/pedestalMakers.py | 0 22 files changed, 0 insertions(+), 0 deletions(-) rename src/nectarchain/{calibration => }/container/__init__.py (100%) rename src/nectarchain/{calibration => }/container/charge.py (100%) rename src/nectarchain/{calibration => }/container/charge_extractor.py (100%) rename src/nectarchain/{calibration => }/container/utils.py (100%) rename src/nectarchain/{calibration => }/container/waveforms.py (100%) rename src/nectarchain/{calibration => }/makers/__init__.py (100%) rename src/nectarchain/{calibration => }/makers/core.py (100%) rename src/nectarchain/{calibration => }/makers/flatfieldMakers.py (100%) rename src/nectarchain/{calibration => }/makers/gain/FlatFieldSPEMakers.py (100%) rename src/nectarchain/{calibration => }/makers/gain/PhotoStatisticMakers.py (100%) rename src/nectarchain/{calibration => }/makers/gain/WhiteTargetSPEMakers.py (100%) rename src/nectarchain/{calibration => }/makers/gain/__init__.py (100%) rename src/nectarchain/{calibration => }/makers/gain/gainMakers.py (100%) rename src/nectarchain/{calibration => }/makers/gain/parameters.py (100%) rename src/nectarchain/{calibration => }/makers/gain/parameters_signal.yaml (100%) rename src/nectarchain/{calibration => }/makers/gain/parameters_signalStd.yaml (100%) rename src/nectarchain/{calibration => }/makers/gain/parameters_signal_combined.yaml (100%) rename src/nectarchain/{calibration => }/makers/gain/parameters_signal_fromHHVFit.yaml (100%) rename src/nectarchain/{calibration => }/makers/gain/utils/__init__.py (100%) rename src/nectarchain/{calibration => }/makers/gain/utils/error.py (100%) rename src/nectarchain/{calibration => }/makers/gain/utils/utils.py (100%) rename src/nectarchain/{calibration => }/makers/pedestalMakers.py (100%) diff --git a/src/nectarchain/calibration/container/__init__.py b/src/nectarchain/container/__init__.py similarity index 100% rename from src/nectarchain/calibration/container/__init__.py rename to src/nectarchain/container/__init__.py diff --git a/src/nectarchain/calibration/container/charge.py b/src/nectarchain/container/charge.py similarity index 100% rename from src/nectarchain/calibration/container/charge.py rename to src/nectarchain/container/charge.py diff --git a/src/nectarchain/calibration/container/charge_extractor.py b/src/nectarchain/container/charge_extractor.py similarity index 100% rename from src/nectarchain/calibration/container/charge_extractor.py rename to src/nectarchain/container/charge_extractor.py diff --git a/src/nectarchain/calibration/container/utils.py b/src/nectarchain/container/utils.py similarity index 100% rename from src/nectarchain/calibration/container/utils.py rename to src/nectarchain/container/utils.py diff --git a/src/nectarchain/calibration/container/waveforms.py b/src/nectarchain/container/waveforms.py similarity index 100% rename from src/nectarchain/calibration/container/waveforms.py rename to src/nectarchain/container/waveforms.py diff --git a/src/nectarchain/calibration/makers/__init__.py b/src/nectarchain/makers/__init__.py similarity index 100% rename from src/nectarchain/calibration/makers/__init__.py rename to src/nectarchain/makers/__init__.py diff --git a/src/nectarchain/calibration/makers/core.py b/src/nectarchain/makers/core.py similarity index 100% rename from src/nectarchain/calibration/makers/core.py rename to src/nectarchain/makers/core.py diff --git a/src/nectarchain/calibration/makers/flatfieldMakers.py b/src/nectarchain/makers/flatfieldMakers.py similarity index 100% rename from src/nectarchain/calibration/makers/flatfieldMakers.py rename to src/nectarchain/makers/flatfieldMakers.py diff --git a/src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py b/src/nectarchain/makers/gain/FlatFieldSPEMakers.py similarity index 100% rename from src/nectarchain/calibration/makers/gain/FlatFieldSPEMakers.py rename to src/nectarchain/makers/gain/FlatFieldSPEMakers.py diff --git a/src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py b/src/nectarchain/makers/gain/PhotoStatisticMakers.py similarity index 100% rename from src/nectarchain/calibration/makers/gain/PhotoStatisticMakers.py rename to src/nectarchain/makers/gain/PhotoStatisticMakers.py diff --git a/src/nectarchain/calibration/makers/gain/WhiteTargetSPEMakers.py b/src/nectarchain/makers/gain/WhiteTargetSPEMakers.py similarity index 100% rename from src/nectarchain/calibration/makers/gain/WhiteTargetSPEMakers.py rename to src/nectarchain/makers/gain/WhiteTargetSPEMakers.py diff --git a/src/nectarchain/calibration/makers/gain/__init__.py b/src/nectarchain/makers/gain/__init__.py similarity index 100% rename from src/nectarchain/calibration/makers/gain/__init__.py rename to src/nectarchain/makers/gain/__init__.py diff --git a/src/nectarchain/calibration/makers/gain/gainMakers.py b/src/nectarchain/makers/gain/gainMakers.py similarity index 100% rename from src/nectarchain/calibration/makers/gain/gainMakers.py rename to src/nectarchain/makers/gain/gainMakers.py diff --git a/src/nectarchain/calibration/makers/gain/parameters.py b/src/nectarchain/makers/gain/parameters.py similarity index 100% rename from src/nectarchain/calibration/makers/gain/parameters.py rename to src/nectarchain/makers/gain/parameters.py diff --git a/src/nectarchain/calibration/makers/gain/parameters_signal.yaml b/src/nectarchain/makers/gain/parameters_signal.yaml similarity index 100% rename from src/nectarchain/calibration/makers/gain/parameters_signal.yaml rename to src/nectarchain/makers/gain/parameters_signal.yaml diff --git a/src/nectarchain/calibration/makers/gain/parameters_signalStd.yaml b/src/nectarchain/makers/gain/parameters_signalStd.yaml similarity index 100% rename from src/nectarchain/calibration/makers/gain/parameters_signalStd.yaml rename to src/nectarchain/makers/gain/parameters_signalStd.yaml diff --git a/src/nectarchain/calibration/makers/gain/parameters_signal_combined.yaml b/src/nectarchain/makers/gain/parameters_signal_combined.yaml similarity index 100% rename from src/nectarchain/calibration/makers/gain/parameters_signal_combined.yaml rename to src/nectarchain/makers/gain/parameters_signal_combined.yaml diff --git a/src/nectarchain/calibration/makers/gain/parameters_signal_fromHHVFit.yaml b/src/nectarchain/makers/gain/parameters_signal_fromHHVFit.yaml similarity index 100% rename from src/nectarchain/calibration/makers/gain/parameters_signal_fromHHVFit.yaml rename to src/nectarchain/makers/gain/parameters_signal_fromHHVFit.yaml diff --git a/src/nectarchain/calibration/makers/gain/utils/__init__.py b/src/nectarchain/makers/gain/utils/__init__.py similarity index 100% rename from src/nectarchain/calibration/makers/gain/utils/__init__.py rename to src/nectarchain/makers/gain/utils/__init__.py diff --git a/src/nectarchain/calibration/makers/gain/utils/error.py b/src/nectarchain/makers/gain/utils/error.py similarity index 100% rename from src/nectarchain/calibration/makers/gain/utils/error.py rename to src/nectarchain/makers/gain/utils/error.py diff --git a/src/nectarchain/calibration/makers/gain/utils/utils.py b/src/nectarchain/makers/gain/utils/utils.py similarity index 100% rename from src/nectarchain/calibration/makers/gain/utils/utils.py rename to src/nectarchain/makers/gain/utils/utils.py diff --git a/src/nectarchain/calibration/makers/pedestalMakers.py b/src/nectarchain/makers/pedestalMakers.py similarity index 100% rename from src/nectarchain/calibration/makers/pedestalMakers.py rename to src/nectarchain/makers/pedestalMakers.py From 808841ab4e81c46fb869ab6e80d32ef1c2f9f9be Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Tue, 12 Sep 2023 23:24:56 +0200 Subject: [PATCH 35/62] folder restructuring --- src/nectarchain/{ => data}/container/__init__.py | 0 src/nectarchain/{ => data}/container/charge.py | 0 src/nectarchain/{ => data}/container/waveforms.py | 0 src/nectarchain/makers/{ => calibration}/__init__.py | 0 src/nectarchain/makers/{ => calibration}/core.py | 0 src/nectarchain/makers/{ => calibration}/flatfieldMakers.py | 0 .../makers/{ => calibration}/gain/FlatFieldSPEMakers.py | 0 .../makers/{ => calibration}/gain/PhotoStatisticMakers.py | 0 .../makers/{ => calibration}/gain/WhiteTargetSPEMakers.py | 0 src/nectarchain/makers/{ => calibration}/gain/__init__.py | 0 src/nectarchain/makers/{ => calibration}/gain/gainMakers.py | 0 src/nectarchain/makers/{ => calibration}/gain/parameters.py | 0 .../makers/{ => calibration}/gain/parameters_signal.yaml | 0 .../makers/{ => calibration}/gain/parameters_signalStd.yaml | 0 .../makers/{ => calibration}/gain/parameters_signal_combined.yaml | 0 .../{ => calibration}/gain/parameters_signal_fromHHVFit.yaml | 0 src/nectarchain/makers/{ => calibration}/gain/utils/__init__.py | 0 src/nectarchain/makers/{ => calibration}/gain/utils/error.py | 0 src/nectarchain/makers/{ => calibration}/gain/utils/utils.py | 0 src/nectarchain/makers/{ => calibration}/pedestalMakers.py | 0 .../{container => makers/extractor}/charge_extractor.py | 0 src/nectarchain/{container => makers/extractor}/utils.py | 0 22 files changed, 0 insertions(+), 0 deletions(-) rename src/nectarchain/{ => data}/container/__init__.py (100%) rename src/nectarchain/{ => data}/container/charge.py (100%) rename src/nectarchain/{ => data}/container/waveforms.py (100%) rename src/nectarchain/makers/{ => calibration}/__init__.py (100%) rename src/nectarchain/makers/{ => calibration}/core.py (100%) rename src/nectarchain/makers/{ => calibration}/flatfieldMakers.py (100%) rename src/nectarchain/makers/{ => calibration}/gain/FlatFieldSPEMakers.py (100%) rename src/nectarchain/makers/{ => calibration}/gain/PhotoStatisticMakers.py (100%) rename src/nectarchain/makers/{ => calibration}/gain/WhiteTargetSPEMakers.py (100%) rename src/nectarchain/makers/{ => calibration}/gain/__init__.py (100%) rename src/nectarchain/makers/{ => calibration}/gain/gainMakers.py (100%) rename src/nectarchain/makers/{ => calibration}/gain/parameters.py (100%) rename src/nectarchain/makers/{ => calibration}/gain/parameters_signal.yaml (100%) rename src/nectarchain/makers/{ => calibration}/gain/parameters_signalStd.yaml (100%) rename src/nectarchain/makers/{ => calibration}/gain/parameters_signal_combined.yaml (100%) rename src/nectarchain/makers/{ => calibration}/gain/parameters_signal_fromHHVFit.yaml (100%) rename src/nectarchain/makers/{ => calibration}/gain/utils/__init__.py (100%) rename src/nectarchain/makers/{ => calibration}/gain/utils/error.py (100%) rename src/nectarchain/makers/{ => calibration}/gain/utils/utils.py (100%) rename src/nectarchain/makers/{ => calibration}/pedestalMakers.py (100%) rename src/nectarchain/{container => makers/extractor}/charge_extractor.py (100%) rename src/nectarchain/{container => makers/extractor}/utils.py (100%) diff --git a/src/nectarchain/container/__init__.py b/src/nectarchain/data/container/__init__.py similarity index 100% rename from src/nectarchain/container/__init__.py rename to src/nectarchain/data/container/__init__.py diff --git a/src/nectarchain/container/charge.py b/src/nectarchain/data/container/charge.py similarity index 100% rename from src/nectarchain/container/charge.py rename to src/nectarchain/data/container/charge.py diff --git a/src/nectarchain/container/waveforms.py b/src/nectarchain/data/container/waveforms.py similarity index 100% rename from src/nectarchain/container/waveforms.py rename to src/nectarchain/data/container/waveforms.py diff --git a/src/nectarchain/makers/__init__.py b/src/nectarchain/makers/calibration/__init__.py similarity index 100% rename from src/nectarchain/makers/__init__.py rename to src/nectarchain/makers/calibration/__init__.py diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/calibration/core.py similarity index 100% rename from src/nectarchain/makers/core.py rename to src/nectarchain/makers/calibration/core.py diff --git a/src/nectarchain/makers/flatfieldMakers.py b/src/nectarchain/makers/calibration/flatfieldMakers.py similarity index 100% rename from src/nectarchain/makers/flatfieldMakers.py rename to src/nectarchain/makers/calibration/flatfieldMakers.py diff --git a/src/nectarchain/makers/gain/FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py similarity index 100% rename from src/nectarchain/makers/gain/FlatFieldSPEMakers.py rename to src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py diff --git a/src/nectarchain/makers/gain/PhotoStatisticMakers.py b/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py similarity index 100% rename from src/nectarchain/makers/gain/PhotoStatisticMakers.py rename to src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py diff --git a/src/nectarchain/makers/gain/WhiteTargetSPEMakers.py b/src/nectarchain/makers/calibration/gain/WhiteTargetSPEMakers.py similarity index 100% rename from src/nectarchain/makers/gain/WhiteTargetSPEMakers.py rename to src/nectarchain/makers/calibration/gain/WhiteTargetSPEMakers.py diff --git a/src/nectarchain/makers/gain/__init__.py b/src/nectarchain/makers/calibration/gain/__init__.py similarity index 100% rename from src/nectarchain/makers/gain/__init__.py rename to src/nectarchain/makers/calibration/gain/__init__.py diff --git a/src/nectarchain/makers/gain/gainMakers.py b/src/nectarchain/makers/calibration/gain/gainMakers.py similarity index 100% rename from src/nectarchain/makers/gain/gainMakers.py rename to src/nectarchain/makers/calibration/gain/gainMakers.py diff --git a/src/nectarchain/makers/gain/parameters.py b/src/nectarchain/makers/calibration/gain/parameters.py similarity index 100% rename from src/nectarchain/makers/gain/parameters.py rename to src/nectarchain/makers/calibration/gain/parameters.py diff --git a/src/nectarchain/makers/gain/parameters_signal.yaml b/src/nectarchain/makers/calibration/gain/parameters_signal.yaml similarity index 100% rename from src/nectarchain/makers/gain/parameters_signal.yaml rename to src/nectarchain/makers/calibration/gain/parameters_signal.yaml diff --git a/src/nectarchain/makers/gain/parameters_signalStd.yaml b/src/nectarchain/makers/calibration/gain/parameters_signalStd.yaml similarity index 100% rename from src/nectarchain/makers/gain/parameters_signalStd.yaml rename to src/nectarchain/makers/calibration/gain/parameters_signalStd.yaml diff --git a/src/nectarchain/makers/gain/parameters_signal_combined.yaml b/src/nectarchain/makers/calibration/gain/parameters_signal_combined.yaml similarity index 100% rename from src/nectarchain/makers/gain/parameters_signal_combined.yaml rename to src/nectarchain/makers/calibration/gain/parameters_signal_combined.yaml diff --git a/src/nectarchain/makers/gain/parameters_signal_fromHHVFit.yaml b/src/nectarchain/makers/calibration/gain/parameters_signal_fromHHVFit.yaml similarity index 100% rename from src/nectarchain/makers/gain/parameters_signal_fromHHVFit.yaml rename to src/nectarchain/makers/calibration/gain/parameters_signal_fromHHVFit.yaml diff --git a/src/nectarchain/makers/gain/utils/__init__.py b/src/nectarchain/makers/calibration/gain/utils/__init__.py similarity index 100% rename from src/nectarchain/makers/gain/utils/__init__.py rename to src/nectarchain/makers/calibration/gain/utils/__init__.py diff --git a/src/nectarchain/makers/gain/utils/error.py b/src/nectarchain/makers/calibration/gain/utils/error.py similarity index 100% rename from src/nectarchain/makers/gain/utils/error.py rename to src/nectarchain/makers/calibration/gain/utils/error.py diff --git a/src/nectarchain/makers/gain/utils/utils.py b/src/nectarchain/makers/calibration/gain/utils/utils.py similarity index 100% rename from src/nectarchain/makers/gain/utils/utils.py rename to src/nectarchain/makers/calibration/gain/utils/utils.py diff --git a/src/nectarchain/makers/pedestalMakers.py b/src/nectarchain/makers/calibration/pedestalMakers.py similarity index 100% rename from src/nectarchain/makers/pedestalMakers.py rename to src/nectarchain/makers/calibration/pedestalMakers.py diff --git a/src/nectarchain/container/charge_extractor.py b/src/nectarchain/makers/extractor/charge_extractor.py similarity index 100% rename from src/nectarchain/container/charge_extractor.py rename to src/nectarchain/makers/extractor/charge_extractor.py diff --git a/src/nectarchain/container/utils.py b/src/nectarchain/makers/extractor/utils.py similarity index 100% rename from src/nectarchain/container/utils.py rename to src/nectarchain/makers/extractor/utils.py From bc65fe56b28c25e77dcb343cf948c1a43b50992a Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Tue, 12 Sep 2023 23:57:58 +0200 Subject: [PATCH 36/62] cleaning after the restructuration --- src/nectarchain/{calibration => }/README.md | 2 +- src/nectarchain/calibration/__init__.py | 2 - src/nectarchain/data/__init__.py | 1 + src/nectarchain/data/container/charge.py | 3 +- src/nectarchain/data/management.py | 146 ++++++++++++++ src/nectarchain/dqm/start_calib.py | 2 +- src/nectarchain/makers/__init__.py | 1 + src/nectarchain/makers/calibration/core.py | 14 +- .../calibration/gain/FlatFieldSPEMakers.py | 2 +- .../calibration/gain/PhotoStatisticMakers.py | 2 +- src/nectarchain/makers/core.py | 19 ++ src/nectarchain/makers/extractor/__init__.py | 1 + src/nectarchain/makers/extractor/utils.py | 181 +----------------- .../ggrolleron/gain_PhotoStat_computation.py | 2 +- .../gain_SPEfit_combined_computation.py | 4 +- .../ggrolleron/gain_SPEfit_computation.py | 4 +- .../ggrolleron/load_wfs_compute_charge.py | 2 +- .../user_scripts/ggrolleron/test.py | 4 +- .../submitGainFitDIRAC/submitGainFitter.py | 2 +- 19 files changed, 187 insertions(+), 207 deletions(-) rename src/nectarchain/{calibration => }/README.md (84%) delete mode 100644 src/nectarchain/calibration/__init__.py create mode 100644 src/nectarchain/data/__init__.py create mode 100644 src/nectarchain/data/management.py create mode 100644 src/nectarchain/makers/__init__.py create mode 100644 src/nectarchain/makers/core.py create mode 100644 src/nectarchain/makers/extractor/__init__.py diff --git a/src/nectarchain/calibration/README.md b/src/nectarchain/README.md similarity index 84% rename from src/nectarchain/calibration/README.md rename to src/nectarchain/README.md index b48f235e..5e1bfce9 100644 --- a/src/nectarchain/calibration/README.md +++ b/src/nectarchain/README.md @@ -1,7 +1,7 @@ Environment variable needed: export NECTARCAMDATA="path to local NectarCam data, it can contain fits.fz run files, WaveformsContainer or ChargeContainer FITS files" -Environment variables which can be defined: +Environment variables which can be defined but are optional: export NECTARCHAIN_TEST="path to test for nectarchain" export NECTARCHAIN_LOG="path to log for nectarchain" export NECTARCHAIN_FIGURES="path to log figures for nectarchain" \ No newline at end of file diff --git a/src/nectarchain/calibration/__init__.py b/src/nectarchain/calibration/__init__.py deleted file mode 100644 index 7869ff92..00000000 --- a/src/nectarchain/calibration/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -"""nectarchain command line tools. -""" diff --git a/src/nectarchain/data/__init__.py b/src/nectarchain/data/__init__.py new file mode 100644 index 00000000..92ba4ff4 --- /dev/null +++ b/src/nectarchain/data/__init__.py @@ -0,0 +1 @@ +from .management import * \ No newline at end of file diff --git a/src/nectarchain/data/container/charge.py b/src/nectarchain/data/container/charge.py index 9fe724e9..fc7d03d7 100644 --- a/src/nectarchain/data/container/charge.py +++ b/src/nectarchain/data/container/charge.py @@ -34,7 +34,7 @@ from numba import guvectorize, float64, int64, bool_ from .waveforms import WaveformsContainer,WaveformsContainers -from .utils import CtaPipeExtractor +from ...makers.extractor.utils import CtapipeExtractor @@ -56,7 +56,6 @@ list_nectarchain_charge_extractor = ['gradient_extractor'] - @guvectorize( [ (int64[:], float64[:], bool_, bool_[:], int64[:]), diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py new file mode 100644 index 00000000..90b5973b --- /dev/null +++ b/src/nectarchain/data/management.py @@ -0,0 +1,146 @@ +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +import os +import glob +import mechanize +import requests +import browser_cookie3 + +from DIRAC.Interfaces.API.Dirac import Dirac +from pathlib import Path +from typing import List,Tuple + +__all__ = ['DataManagement','ChainGenerator'] + +class DataManagement() : + @staticmethod + def findrun(run_number : int,search_on_GRID = True) -> Tuple[Path,List[Path]]: + """method to find in NECTARCAMDATA the list of *.fits.fz files associated to run_number + + Args: + run_number (int): the run number + + Returns: + (PosixPath,list): the path list of *fits.fz files + """ + basepath=os.environ['NECTARCAMDATA'] + list = glob.glob(basepath+'**/*'+str(run_number)+'*.fits.fz',recursive=True) + list_path = [Path(chemin) for chemin in list] + if len(list_path) == 0 : + e = FileNotFoundError(f"run {run_number} is not present in {basepath}") + if search_on_GRID : + log.warning(e,exc_info=True) + log.info('will search files on GRID and fetch them') + lfns = DataManagement.get_GRID_location(run_number) + DataManagement.getRunFromDIRAC(lfns) + list = glob.glob(basepath+'**/*'+str(run_number)+'*.fits.fz',recursive=True) + list_path = [Path(chemin) for chemin in list] + else : + log.error(e,exc_info=True) + raise e + + + + name = list_path[0].name.split(".") + name[2] = "*" + name = Path(str(list_path[0].parent))/(f"{name[0]}.{name[1]}.{name[2]}.{name[3]}.{name[4]}") + log.info(f"Found {len(list_path)} files matching {name}") + + #to sort list path + _sorted = sorted([[file,int(file.suffixes[1][1:])] for file in list_path]) + list_path = [_sorted[i][0] for i in range(len(_sorted))] + + return name,list_path + + @staticmethod + def getRunFromDIRAC(lfns : list): + """method do get run files from GRID-EGI from input lfns + + Args: + lfns (list): list of lfns path + """ + + dirac = Dirac() + for lfn in lfns : + if not(os.path.exists(f'{os.environ["NECTARCAMDATA"]}/{os.path.basename(lfn)}')): + dirac.getFile(lfn=lfn,destDir=os.environ["NECTARCAMDATA"],printOutput=True) + + + @staticmethod + def get_GRID_location(run_number : int,output_lfns = True, username = None,password = None) : + """method to get run location on GRID from Elog (work in progress!) + + Args: + run_number (int): run number + output_lfns (bool, optional): if True, return lfns path of fits.gz files, else return parent directory of run location. Defaults to True. + username (_type_, optional): username for Elog login. Defaults to None. + password (_type_, optional): password for Elog login. Defaults to None. + + Returns: + _type_: _description_ + """ + + url = "http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?cmd=Find" + + url_run = f"http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?mode=full&reverse=0&reverse=1&npp=20&subtext=%23{run_number}" + + if not(username is None or password is None) : + log.debug('log to Elog with username and password') + #log to Elog + br = mechanize.Browser() + br.open(url) + form = br.select_form('form1') + for i in range(4) : + log.debug(br.form.find_control(nr=i).name) + br.form['uname'] = username + br.form['upassword'] = password + br.method = "POST" + req = br.submit() + #html_page = req.get_data() + cookies = br._ua_handlers['_cookies'].cookiejar + #get data + req = requests.get(f'http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?jcmd=&mode=Raw&attach=1&printable=1&reverse=0&reverse=1&npp=20&ma=&da=&ya=&ha=&na=&ca=&last=&mb=&db=&yb=&hb=&nb=&cb=&Author=&Setup=&Category=&Keyword=&Subject=%23{run_number}&ModuleCount=&subtext=',cookies = cookies) + + else : + #try to acces data by getting cookies from firefox and Chrome + log.debug('try to get data with cookies from Firefox abnd Chrome') + cookies = browser_cookie3.load() + req = requests.get(f'http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?jcmd=&mode=Raw&attach=1&printable=1&reverse=0&reverse=1&npp=20&ma=&da=&ya=&ha=&na=&ca=&last=&mb=&db=&yb=&hb=&nb=&cb=&Author=&Setup=&Category=&Keyword=&Subject=%23{run_number}&ModuleCount=&subtext=',cookies = cookies) + + #if "ELOG Login" in req.text : + + lines = req.text.split('\r\n') + + url_data = None + for i,line in enumerate(lines) : + if '

' in line : + url_data = line.split("

")[0].split('FC:')[1] + log.debug(f"url_data found {url_data}") + break + + if i == len(lines)-1 : + e=Exception("lfns not found on GRID") + log.error(e,exc_info=True) + log.debug(lines) + raise e + + if output_lfns : + lfns = [] + try : + #Dirac + dirac = Dirac() + loc = f"/vo.cta.in2p3.fr/nectarcam/{url_data.split('/')[-2]}/{url_data.split('/')[-1]}" + log.debug(f"searching in Dirac filecatalog at {loc}") + res = dirac.listCatalogDirectory(loc, printOutput=True) + + for key in res['Value']['Successful'][loc]['Files'].keys(): + if str(run_number) in key and "fits.fz" in key : + lfns.append(key) + except Exception as e : + log.error(e,exc_info = True) + return lfns + else : + return url_data diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index d1972eb6..b8802e16 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -53,7 +53,7 @@ if args.runnb is not None: # Grab runs automatically from DIRAC is the -r option is provided - from nectarchain.calibration.container import utils + from nectarchain.data.container import utils dm = utils.DataManagement() _, filelist = dm.findrun(args.runnb) args.input_files = [s.name for s in filelist] diff --git a/src/nectarchain/makers/__init__.py b/src/nectarchain/makers/__init__.py new file mode 100644 index 00000000..a8ce586c --- /dev/null +++ b/src/nectarchain/makers/__init__.py @@ -0,0 +1 @@ +from .core import * \ No newline at end of file diff --git a/src/nectarchain/makers/calibration/core.py b/src/nectarchain/makers/calibration/core.py index 088440e1..3281150c 100644 --- a/src/nectarchain/makers/calibration/core.py +++ b/src/nectarchain/makers/calibration/core.py @@ -6,17 +6,17 @@ from pathlib import Path import numpy as np -from abc import ABC, abstractmethod - from astropy.table import QTable,Column import astropy.units as u from copy import copy from datetime import date from collections.abc import Iterable +from ..core import GeneralMaker + __all__ = [""] -class CalibrationMaker(ABC): +class CalibrationMaker(GeneralMaker): """ Mother class for all calibration makers that can be defined to compute calibration coefficients from data. @@ -57,14 +57,6 @@ def __init__(self, pixels_id, *args, **kwargs) -> None: self.__results.meta[__class__.NP_PIXELS] = self.npixels self.__results.meta['comments'] = f'Produced with NectarChain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' - @abstractmethod - def make(self, *args, **kwargs): - """ - Abstract method that needs to be implemented by subclasses. - Used to compute the calibration coefficients. - """ - pass - def save(self, path, **kwargs): """ Saves the results to a file in the specified path. diff --git a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py index b84bedb5..eadbb3c7 100644 --- a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py @@ -33,7 +33,7 @@ from .gainMakers import GainMaker -from ...container import ChargeContainer +from ....data.container import ChargeContainer from .parameters import Parameter, Parameters diff --git a/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py b/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py index 2c29b78c..53dbe0a0 100644 --- a/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py +++ b/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py @@ -18,7 +18,7 @@ from ctapipe_io_nectarcam import constants -from ...container import ChargeContainer +from ....data.container import ChargeContainer from .gainMakers import GainMaker diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py new file mode 100644 index 00000000..ba583269 --- /dev/null +++ b/src/nectarchain/makers/core.py @@ -0,0 +1,19 @@ +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +from abc import ABC, abstractmethod + +__all__ = ["GeneralMaker"] + +class GeneralMaker(ABC): + """Mother class for all the makers, the role of makers is to do computation on the data. + """ + @abstractmethod + def make(self, *args, **kwargs): + """ + Abstract method that needs to be implemented by subclasses. + This method is the main one, which computes and does the work. + """ + pass \ No newline at end of file diff --git a/src/nectarchain/makers/extractor/__init__.py b/src/nectarchain/makers/extractor/__init__.py new file mode 100644 index 00000000..b8e298f9 --- /dev/null +++ b/src/nectarchain/makers/extractor/__init__.py @@ -0,0 +1 @@ +from .charge_extractor import * \ No newline at end of file diff --git a/src/nectarchain/makers/extractor/utils.py b/src/nectarchain/makers/extractor/utils.py index 519d1e04..3ba701ef 100644 --- a/src/nectarchain/makers/extractor/utils.py +++ b/src/nectarchain/makers/extractor/utils.py @@ -1,187 +1,10 @@ -from email.generator import Generator -import os -import glob -import re -import mechanize -import requests -import browser_cookie3 - -from DIRAC.Interfaces.API.Dirac import Dirac -from pathlib import Path -from typing import List,Tuple - -from ctapipe.containers import DL1CameraContainer - - import logging logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') log = logging.getLogger(__name__) log.handlers = logging.getLogger('__main__').handlers -__all__ = ['DataManagement','ChainGenerator'] - -class DataManagement() : - @staticmethod - def findrun(run_number : int,search_on_GRID = True) -> Tuple[Path,List[Path]]: - """method to find in NECTARCAMDATA the list of *.fits.fz files associated to run_number - - Args: - run_number (int): the run number - - Returns: - (PosixPath,list): the path list of *fits.fz files - """ - basepath=os.environ['NECTARCAMDATA'] - list = glob.glob(basepath+'**/*'+str(run_number)+'*.fits.fz',recursive=True) - list_path = [Path(chemin) for chemin in list] - if len(list_path) == 0 : - e = FileNotFoundError(f"run {run_number} is not present in {basepath}") - if search_on_GRID : - log.warning(e,exc_info=True) - log.info('will search files on GRID and fetch them') - lfns = DataManagement.get_GRID_location(run_number) - DataManagement.getRunFromDIRAC(lfns) - list = glob.glob(basepath+'**/*'+str(run_number)+'*.fits.fz',recursive=True) - list_path = [Path(chemin) for chemin in list] - else : - log.error(e,exc_info=True) - raise e - - - - name = list_path[0].name.split(".") - name[2] = "*" - name = Path(str(list_path[0].parent))/(f"{name[0]}.{name[1]}.{name[2]}.{name[3]}.{name[4]}") - log.info(f"Found {len(list_path)} files matching {name}") - - #to sort list path - _sorted = sorted([[file,int(file.suffixes[1][1:])] for file in list_path]) - list_path = [_sorted[i][0] for i in range(len(_sorted))] - - return name,list_path - - @staticmethod - def getRunFromDIRAC(lfns : list): - """method do get run files from GRID-EGI from input lfns - - Args: - lfns (list): list of lfns path - """ - - dirac = Dirac() - for lfn in lfns : - if not(os.path.exists(f'{os.environ["NECTARCAMDATA"]}/{os.path.basename(lfn)}')): - dirac.getFile(lfn=lfn,destDir=os.environ["NECTARCAMDATA"],printOutput=True) - - - @staticmethod - def get_GRID_location(run_number : int,output_lfns = True, username = None,password = None) : - """method to get run location on GRID from Elog (work in progress!) - - Args: - run_number (int): run number - output_lfns (bool, optional): if True, return lfns path of fits.gz files, else return parent directory of run location. Defaults to True. - username (_type_, optional): username for Elog login. Defaults to None. - password (_type_, optional): password for Elog login. Defaults to None. - - Returns: - _type_: _description_ - """ - - url = "http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?cmd=Find" - - url_run = f"http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?mode=full&reverse=0&reverse=1&npp=20&subtext=%23{run_number}" - - if not(username is None or password is None) : - log.debug('log to Elog with username and password') - #log to Elog - br = mechanize.Browser() - br.open(url) - form = br.select_form('form1') - for i in range(4) : - log.debug(br.form.find_control(nr=i).name) - br.form['uname'] = username - br.form['upassword'] = password - br.method = "POST" - req = br.submit() - #html_page = req.get_data() - cookies = br._ua_handlers['_cookies'].cookiejar - #get data - req = requests.get(f'http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?jcmd=&mode=Raw&attach=1&printable=1&reverse=0&reverse=1&npp=20&ma=&da=&ya=&ha=&na=&ca=&last=&mb=&db=&yb=&hb=&nb=&cb=&Author=&Setup=&Category=&Keyword=&Subject=%23{run_number}&ModuleCount=&subtext=',cookies = cookies) - - else : - #try to acces data by getting cookies from firefox and Chrome - log.debug('try to get data with cookies from Firefox abnd Chrome') - cookies = browser_cookie3.load() - req = requests.get(f'http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?jcmd=&mode=Raw&attach=1&printable=1&reverse=0&reverse=1&npp=20&ma=&da=&ya=&ha=&na=&ca=&last=&mb=&db=&yb=&hb=&nb=&cb=&Author=&Setup=&Category=&Keyword=&Subject=%23{run_number}&ModuleCount=&subtext=',cookies = cookies) - - #if "ELOG Login" in req.text : - - lines = req.text.split('\r\n') - - url_data = None - for i,line in enumerate(lines) : - if '

' in line : - url_data = line.split("

")[0].split('FC:')[1] - log.debug(f"url_data found {url_data}") - break - - if i == len(lines)-1 : - e=Exception("lfns not found on GRID") - log.error(e,exc_info=True) - log.debug(lines) - raise e - - if output_lfns : - lfns = [] - try : - #Dirac - dirac = Dirac() - loc = f"/vo.cta.in2p3.fr/nectarcam/{url_data.split('/')[-2]}/{url_data.split('/')[-1]}" - log.debug(f"searching in Dirac filecatalog at {loc}") - res = dirac.listCatalogDirectory(loc, printOutput=True) - - for key in res['Value']['Successful'][loc]['Files'].keys(): - if str(run_number) in key and "fits.fz" in key : - lfns.append(key) - except Exception as e : - log.error(e,exc_info = True) - return lfns - else : - return url_data - - -class ChainGenerator(): - @staticmethod - def chain(a : Generator ,b : Generator) : - """generic metghod to chain 2 generators - - Args: - a (Generator): generator to chain - b (Generator): generator to chain - - Yields: - Generator: a chain of a and b - """ - yield from a - yield from b - - - @staticmethod - def chainEventSource(list : list,max_events : int = None) : #useless with ctapipe_io_nectarcam.NectarCAMEventSource - """recursive method to chain a list of ctapipe.io.EventSource (which may be associated to the list of *.fits.fz file of one run) - - Args: - list (EventSource): a list of EventSource - - Returns: - Generator: a generator which chains EventSource - """ - if len(list) == 2 : - return ChainGenerator.chain(list[0],list[1]) - else : - return ChainGenerator.chain(list[0],ChainGenerator.chainEventSource(list[1:])) +from ctapipe.containers import DL1CameraContainer -class CtaPipeExtractor(): +class CtapipeExtractor(): def get_image_peak_time(cameraContainer : DL1CameraContainer) : return cameraContainer.image, cameraContainer.peak_time diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py index 1e5ed511..8ffff837 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py @@ -14,7 +14,7 @@ from astropy.table import QTable -from nectarchain.calibration.makers.gain.PhotoStatisticMakers import PhotoStatisticMaker +from nectarchain.makers.calibration.gain.PhotoStatisticMakers import PhotoStatisticMaker parser = argparse.ArgumentParser( diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py index 51326243..25d8386f 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py @@ -12,8 +12,8 @@ import argparse #import seaborn as sns -from nectarchain.calibration.container import ChargeContainer -from nectarchain.calibration.makers.gain.FlatFieldSPEMakers import FlatFieldSingleNominalSPEMaker +from nectarchain.data.container import ChargeContainer +from nectarchain.makers.calibration.gain.FlatFieldSPEMakers import FlatFieldSingleNominalSPEMaker parser = argparse.ArgumentParser( prog = 'gain_SPEfit_combined_computation.py', diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py index 3eb4f6cb..02604694 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py @@ -13,8 +13,8 @@ import argparse #import seaborn as sns -from nectarchain.calibration.container import ChargeContainer -from nectarchain.calibration.makers.gain.FlatFieldSPEMakers import FlatFieldSingleHHVSPEMaker,FlatFieldSingleHHVStdSPEMaker +from nectarchain.data.container import ChargeContainer +from nectarchain.makers.calibration.gain.FlatFieldSPEMakers import FlatFieldSingleHHVSPEMaker,FlatFieldSingleHHVStdSPEMaker parser = argparse.ArgumentParser( prog = 'gain_SPEfit_computation.py', diff --git a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py index 10112ce5..7a2ae177 100644 --- a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py +++ b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py @@ -10,7 +10,7 @@ logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level=logging.DEBUG,filename = f"{os.environ.get('NECTARCHAIN_LOG')}/{Path(__file__).stem}_{os.getpid()}.log") log = logging.getLogger(__name__) -from nectarchain.calibration.container import WaveformsContainer,WaveformsContainers,ChargeContainer,ChargeContainers +from nectarchain.data.container import WaveformsContainer,WaveformsContainers,ChargeContainer,ChargeContainers parser = argparse.ArgumentParser( prog = 'load_wfs_compute_charge', diff --git a/src/nectarchain/user_scripts/ggrolleron/test.py b/src/nectarchain/user_scripts/ggrolleron/test.py index 8e2d2b27..637ae3d3 100644 --- a/src/nectarchain/user_scripts/ggrolleron/test.py +++ b/src/nectarchain/user_scripts/ggrolleron/test.py @@ -20,8 +20,8 @@ -from nectarchain.calibration.container import ChargeContainer,WaveformsContainer,ChargeContainers,WaveformsContainers -from nectarchain.calibration.container.utils import DataManagement +from nectarchain.data.container import ChargeContainer,WaveformsContainer,ChargeContainers,WaveformsContainers +from nectarchain.data.container.utils import DataManagement def test_check_wfs() : diff --git a/src/nectarchain/user_scripts/jlenain/submitGainFitDIRAC/submitGainFitter.py b/src/nectarchain/user_scripts/jlenain/submitGainFitDIRAC/submitGainFitter.py index 3002a647..a65113f1 100755 --- a/src/nectarchain/user_scripts/jlenain/submitGainFitDIRAC/submitGainFitter.py +++ b/src/nectarchain/user_scripts/jlenain/submitGainFitDIRAC/submitGainFitter.py @@ -10,7 +10,7 @@ from astropy import time # nectarchain imports -from nectarchain.calibration.container.utils import DataManagement as dm +from nectarchain.data.container.utils import DataManagement as dm # DIRAC imports from DIRAC.Interfaces.API.Dirac import Dirac From aacd51b78f0d356f5f74c15cd84118648264db22 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 13 Sep 2023 02:41:06 +0200 Subject: [PATCH 37/62] unit test for chargecontainer --- src/nectarchain/data/container/charge.py | 191 ++++++++---- .../data/container/tests/test_charge.py | 283 ++++++++++++++++++ .../data/container/tests/test_waveforms.py | 0 src/nectarchain/data/container/waveforms.py | 4 +- src/nectarchain/data/management.py | 2 +- .../makers/extractor/charge_extractor.py | 5 +- 6 files changed, 422 insertions(+), 63 deletions(-) create mode 100644 src/nectarchain/data/container/tests/test_charge.py create mode 100644 src/nectarchain/data/container/tests/test_waveforms.py diff --git a/src/nectarchain/data/container/charge.py b/src/nectarchain/data/container/charge.py index fc7d03d7..6f0608d2 100644 --- a/src/nectarchain/data/container/charge.py +++ b/src/nectarchain/data/container/charge.py @@ -1,45 +1,28 @@ +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + from argparse import ArgumentError import numpy as np import numpy.ma as ma -from matplotlib import pyplot as plt -import copy from pathlib import Path import glob import time -import sys import os -import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') -log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers -from ctapipe.visualization import CameraDisplay -from ctapipe.coordinates import CameraFrame,EngineeringCameraFrame from ctapipe.instrument import CameraGeometry -from ctapipe.image.extractor import (FullWaveformSum, - FixedWindowSum, - GlobalPeakWindowSum, - LocalPeakWindowSum, - SlidingWindowMaxSum, - NeighborPeakWindowSum, - BaselineSubtractedNeighborPeakWindowSum, - TwoPassWindowSum) - -from ctapipe_io_nectarcam import NectarCAMEventSource,constants -from ctapipe.io import EventSource, EventSeeker - -from astropy.table import Table + +from ctapipe_io_nectarcam import constants + from astropy.io import fits from numba import guvectorize, float64, int64, bool_ from .waveforms import WaveformsContainer,WaveformsContainers -from ...makers.extractor.utils import CtapipeExtractor -#from .charge_extractor import * - __all__ = ['ChargeContainer','ChargeContainers'] list_ctapipe_charge_extractor = ["FullWaveformSum", @@ -107,12 +90,85 @@ def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): #print("this pixel is broken, skipped") pass + class ChargeContainer() : - """class used to compute charge from waveforms container""" + """ + class used to compute charge from waveforms container + + Attributes: + TEL_ID (int): Telescope ID + CAMERA (CameraGeometry): Camera geometry + + Methods: + __init__(self, charge_hg, charge_lg, peak_hg, peak_lg, run_number, pixels_id, nevents, npixels, method="FullWaveformSum"): Initializes a ChargeContainer instance + from_waveforms(cls, waveformContainer, method="FullWaveformSum", **kwargs): Creates a ChargeContainer instance from a WaveformsContainer + write(self, path, **kwargs): Writes the charge data to a FITS file + from_file(path, run_number, **kwargs): Loads charge data from a FITS file + compute_charge(waveformContainer, channel, method="FullWaveformSum", **kwargs): Computes charge from waveforms + histo_hg(self, n_bins=1000, autoscale=True): Computes the histogram of the high-gain (HG) channel charge values + histo_lg(self, n_bins=1000, autoscale=True): Computes the histogram of the low-gain (LG) channel charge values + select_charge_hg(self, pixel_id): Extracts the high-gain (HG) charge values for the specified pixel IDs + select_charge_lg(self, pixel_id): Extracts the low-gain (LG) charge values for the specified pixel IDs + sort(self, method='event_id'): Sorts the charge data based on the specified method + run_number(self): Returns the run number + pixels_id(self): Returns the pixel IDs + npixels(self): Returns the number of pixels + nevents(self): Returns the number of events + method(self): Returns the charge computation method + multiplicity(self): Returns the multiplicity of events + trig_pattern(self): Returns the trigger pattern + + Example Usage: + # Create a ChargeContainer instance from a WaveformsContainer + chargeContainer = ChargeContainer.from_waveforms(waveformContainer) + + # Write the charge data to a FITS file + chargeContainer.write(path) + + # Load charge data from a FITS file + chargeContainer = ChargeContainer.from_file(path, run_number) + + # Compute histograms of the charge values + hist_hg = chargeContainer.histo_hg() + hist_lg = chargeContainer.histo_lg() + + # Select charge values for specific pixels + pixel_ids = [1, 2, 3] + charge_hg = chargeContainer.select_charge_hg(pixel_ids) + charge_lg = chargeContainer.select_charge_lg(pixel_ids) + + # Sort the charge data based on event ID + chargeContainer.sort() + + # Access properties of the ChargeContainer + run_number = chargeContainer.run_number + pixels_id = chargeContainer.pixels_id + npixels = chargeContainer.npixels + nevents = chargeContainer.nevents + method = chargeContainer.method + """ TEL_ID = 0 CAMERA = CameraGeometry.from_name("NectarCam-003") - def __init__(self,charge_hg,charge_lg,peak_hg,peak_lg,run_number,pixels_id,nevents,npixels, method = "FullWaveformSum") : + def __init__(self, charge_hg, charge_lg, peak_hg, peak_lg, run_number, pixels_id, nevents, npixels, method="FullWaveformSum"): + """ + Initializes a ChargeContainer instance with the provided arguments and sets some additional attributes to default values. + + Args: + charge_hg (array): The high-gain charge values. + charge_lg (array): The low-gain charge values. + peak_hg (array): The high-gain peak time values. + peak_lg (array): The low-gain peak time values. + run_number (int): The run number. + pixels_id (array): The pixel IDs. + nevents (int): The number of events. + npixels (int): The number of pixels. + method (str, optional): The charge computation method. Defaults to "FullWaveformSum". + + Returns: + None + + """ self.charge_hg = charge_hg self.charge_lg = charge_lg self.peak_hg = peak_hg @@ -123,42 +179,37 @@ def __init__(self,charge_hg,charge_lg,peak_hg,peak_lg,run_number,pixels_id,neven self._nevents = nevents self._npixels = npixels - - self.ucts_timestamp = np.zeros((self.nevents),dtype = np.uint64) - self.ucts_busy_counter = np.zeros((self.nevents),dtype = np.uint16) - self.ucts_event_counter = np.zeros((self.nevents),dtype = np.uint16) - self.event_type = np.zeros((self.nevents),dtype = np.uint8) - self.event_id = np.zeros((self.nevents),dtype = np.uint16) - self.trig_pattern_all = np.zeros((self.nevents,self.CAMERA.n_pixels,4),dtype = bool) + self.ucts_timestamp = np.zeros((self.nevents), dtype=np.uint64) + self.ucts_busy_counter = np.zeros((self.nevents), dtype=np.uint16) + self.ucts_event_counter = np.zeros((self.nevents), dtype=np.uint16) + self.event_type = np.zeros((self.nevents), dtype=np.uint8) + self.event_id = np.zeros((self.nevents), dtype=np.uint16) + self.trig_pattern_all = np.zeros((self.nevents, self.CAMERA.n_pixels, 4), dtype=bool) @classmethod - def from_waveforms(cls,waveformContainer : WaveformsContainer,method : str = "FullWaveformSum",**kwargs) : - """ create a new ChargeContainer from a WaveformsContainer + def from_waveforms(cls, waveformContainer: WaveformsContainer, method: str = "FullWaveformSum", **kwargs) -> 'ChargeContainer': + """Create a new ChargeContainer instance from a WaveformsContainer. Args: - waveformContainer (WaveformsContainer): the waveforms - method (str, optional): Ctapipe ImageExtractor method. Defaults to "FullWaveformSum". - + waveformContainer (WaveformsContainer): The waveforms container object from which to compute the charge values. + method (str, optional): The method to use for charge computation. Defaults to "FullWaveformSum". Returns: - chargeContainer : the ChargeContainer instance + ChargeContainer: The created ChargeContainer instance with computed charge values and other attributes. """ log.info(f"computing hg charge with {method} method") - charge_hg,peak_hg = ChargeContainer.compute_charge(waveformContainer,constants.HIGH_GAIN,method,**kwargs) - charge_hg = np.array(charge_hg,dtype = np.uint16) + charge_hg, peak_hg = ChargeContainer.compute_charge(waveformContainer, constants.HIGH_GAIN, method, **kwargs) + charge_hg = np.array(charge_hg, dtype=np.uint16) log.info(f"computing lg charge with {method} method") - charge_lg,peak_lg = ChargeContainer.compute_charge(waveformContainer,constants.LOW_GAIN,method,**kwargs) - charge_lg = np.array(charge_lg,dtype = np.uint16) - - chargeContainer = cls(charge_hg,charge_lg,peak_hg,peak_lg,waveformContainer.run_number,waveformContainer.pixels_id,waveformContainer.nevents,waveformContainer.npixels ,method) - + charge_lg, peak_lg = ChargeContainer.compute_charge(waveformContainer, constants.LOW_GAIN, method, **kwargs) + charge_lg = np.array(charge_lg, dtype=np.uint16) + chargeContainer = cls(charge_hg, charge_lg, peak_hg, peak_lg, waveformContainer.run_number, waveformContainer.pixels_id, waveformContainer.nevents, waveformContainer.npixels, method) chargeContainer.ucts_timestamp = waveformContainer.ucts_timestamp chargeContainer.ucts_busy_counter = waveformContainer.ucts_busy_counter chargeContainer.ucts_event_counter = waveformContainer.ucts_event_counter chargeContainer.event_type = waveformContainer.event_type chargeContainer.event_id = waveformContainer.event_id chargeContainer.trig_pattern_all = waveformContainer.trig_pattern_all - - return chargeContainer + return chargeContainer def write(self,path : Path,**kwargs) : @@ -286,6 +337,9 @@ def compute_charge(waveformContainer : WaveformsContainer,channel : int,method : Returns: output of the extractor called on waveforms """ + + #import is here for fix issue with pytest (TypeError : inference is not possible with python <3.9 (Numba conflict bc there is no inference...)) + from ...makers.extractor.utils import CtapipeExtractor if not(method in list_ctapipe_charge_extractor or method in list_nectarchain_charge_extractor) : raise ArgumentError(f"method must be in {list_ctapipe_charge_extractor}") @@ -302,10 +356,10 @@ def compute_charge(waveformContainer : WaveformsContainer,channel : int,method : ImageExtractor = eval(method)(waveformContainer.subarray,**extractor_kwargs) if channel == constants.HIGH_GAIN: - out = np.array([CtaPipeExtractor.get_image_peak_time(ImageExtractor(waveformContainer.wfs_hg[i],waveformContainer.TEL_ID,channel,waveformContainer.broken_pixels_hg)) for i in range(len(waveformContainer.wfs_hg))]).transpose(1,0,2) + out = np.array([CtapipeExtractor.get_image_peak_time(ImageExtractor(waveformContainer.wfs_hg[i],waveformContainer.TEL_ID,channel,waveformContainer.broken_pixels_hg)) for i in range(len(waveformContainer.wfs_hg))]).transpose(1,0,2) return out[0],out[1] elif channel == constants.LOW_GAIN: - out = np.array([CtaPipeExtractor.get_image_peak_time(ImageExtractor(waveformContainer.wfs_lg[i],waveformContainer.TEL_ID,channel,waveformContainer.broken_pixels_lg)) for i in range(len(waveformContainer.wfs_lg))]).transpose(1,0,2) + out = np.array([CtapipeExtractor.get_image_peak_time(ImageExtractor(waveformContainer.wfs_lg[i],waveformContainer.TEL_ID,channel,waveformContainer.broken_pixels_lg)) for i in range(len(waveformContainer.wfs_lg))]).transpose(1,0,2) return out[0],out[1] else : raise ArgumentError(f"channel must be {constants.LOW_GAIN} or {constants.HIGH_GAIN}") @@ -443,7 +497,6 @@ def nevents(self) : return self._nevents @property def method(self) : return self._method - #physical properties @property def multiplicity(self) : return np.uint16(np.count_nonzero(self.trig_pattern,axis = 1)) @@ -452,12 +505,40 @@ def multiplicity(self) : return np.uint16(np.count_nonzero(self.trig_pattern,ax def trig_pattern(self) : return self.trig_pattern_all.any(axis = 1) - class ChargeContainers() : + """ + The `ChargeContainers` class is used to store and manipulate a collection of `ChargeContainer` objects. It provides methods for creating, writing, and merging `ChargeContainer` instances. + + Example Usage: + # Create a `ChargeContainers` instance from a `WaveformsContainers` object + waveform_containers = WaveformsContainers() + charge_containers = ChargeContainers.from_waveforms(waveform_containers) + + # Write the `ChargeContainers` to disk + charge_containers.write("path/to/save") + + # Load `ChargeContainers` from a FITS file + charge_containers = ChargeContainers.from_file("path/to/file", run_number=123) + + # Merge the `ChargeContainers` into a single `ChargeContainer` + merged_charge_container = charge_containers.merge() + + Methods: + - `from_waveforms(waveformContainers: WaveformsContainers, **kwargs)`: Creates a `ChargeContainers` instance from a `WaveformsContainers` object. + - `write(path: str, **kwargs)`: Writes each `ChargeContainer` in `ChargeContainers` to disk. + - `from_file(path: Path, run_number: int, **kwargs)`: Loads `ChargeContainers` from FITS files previously written with `write()` method. + - `append(chargeContainer: ChargeContainer)`: Stacks a `ChargeContainer` into the `ChargeContainers` instance. + - `merge() -> ChargeContainer`: Merges the `ChargeContainers` into a single `ChargeContainer`. + + Fields: + - `chargeContainers`: A list to store `ChargeContainer` objects. + - `__nChargeContainer`: The number of `ChargeContainer` objects stored in `chargeContainers`. + - `nChargeContainer`: A property that returns the number of `ChargeContainer` objects in `chargeContainers`. + - `nevents`: A property that returns the total number of events in all `ChargeContainer` objects in `chargeContainers`. + """ def __init__(self, *args, **kwargs) : self.chargeContainers = [] self.__nChargeContainer = 0 - @classmethod def from_waveforms(cls,waveformContainers : WaveformsContainers,**kwargs) : """create ChargeContainers from waveformContainers @@ -503,7 +584,7 @@ def from_file(path : Path,run_number : int,**kwargs) : else : cls = ChargeContainers.__new__(ChargeContainers) cls.chargeContainers = [] - cls.__nchargeContainers = len(files) + cls.__nChargeContainer = len(files) for file in files : cls.chargeContainers.append(ChargeContainer.from_file(path,run_number,explicit_filename = file)) return cls diff --git a/src/nectarchain/data/container/tests/test_charge.py b/src/nectarchain/data/container/tests/test_charge.py new file mode 100644 index 00000000..db52191a --- /dev/null +++ b/src/nectarchain/data/container/tests/test_charge.py @@ -0,0 +1,283 @@ +from nectarchain.data.container.charge import ChargeContainers, ChargeContainer +from nectarchain.data.container.waveforms import WaveformsContainers +import glob +import numpy as np +import pytest + +def create_fake_chargeContainer() : + pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) + nevents = 40 + npixels = 10 + charge_hg = np.random.randn(nevents,npixels) + charge_lg = np.random.randn(nevents,npixels) + peak_hg = np.random.randn(nevents,npixels) + peak_lg = np.random.randn(nevents,npixels) + run_number = 1234 + return ChargeContainer( + charge_hg = charge_hg , + charge_lg = charge_lg, + peak_hg = peak_hg, + peak_lg = peak_lg, + run_number = run_number, + pixels_id = pixels_id, + nevents = nevents, + npixels = npixels + ) + +class TestChargeContainer: + + # Tests that a ChargeContainer object can be created with valid input parameters. + def test_create_charge_container(self): + pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) + nevents = 40 + npixels = 10 + charge_hg = np.random.randn(nevents,npixels) + charge_lg = np.random.randn(nevents,npixels) + peak_hg = np.random.randn(nevents,npixels) + peak_lg = np.random.randn(nevents,npixels) + run_number = 1234 + method = 'FullWaveformSum' + charge_container = ChargeContainer( + charge_hg = charge_hg , + charge_lg = charge_lg, + peak_hg = peak_hg, + peak_lg = peak_lg, + run_number = run_number, + pixels_id = pixels_id, + nevents = nevents, + npixels = npixels, + method = method + ) + + assert np.allclose(charge_container.charge_hg,charge_hg) + assert np.allclose(charge_container.charge_lg,charge_lg) + assert np.allclose(charge_container.peak_hg,peak_hg) + assert np.allclose(charge_container.peak_lg,peak_lg) + assert charge_container.run_number == run_number + assert charge_container.pixels_id.tolist() == pixels_id.tolist() + assert charge_container.nevents == nevents + assert charge_container.npixels == npixels + assert charge_container.method == method + + # Tests that the from_waveforms method can be called with a valid waveformContainer and method parameter. + #def test_from_waveforms_valid_input(self): + # waveform_container = WaveformsContainer(...) + # method = 'FullWaveformSum' + # + # charge_container = ChargeContainer.from_waveforms(waveform_container, method) + # + # assert isinstance(charge_container, ChargeContainer) + + # Tests that the ChargeContainer object can be written to a file and the file is created. + def test_write_charge_container(self, tmp_path = "/tmp"): + charge_container = create_fake_chargeContainer() + + charge_container.write(tmp_path) + + assert len(glob.glob(f"{tmp_path}/charge_run1234.fits")) == 1 + + # Tests that a ChargeContainer object can be loaded from a file and the object is correctly initialized. + def test_load_charge_container(self, tmp_path = "/tmp"): + pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) + nevents = 40 + npixels = 10 + charge_hg = np.random.randn(nevents,npixels) + charge_lg = np.random.randn(nevents,npixels) + peak_hg = np.random.randn(nevents,npixels) + peak_lg = np.random.randn(nevents,npixels) + run_number = np.random.randn(1)[0] + method = 'FullWaveformSum' + charge_container = ChargeContainer( + charge_hg = charge_hg , + charge_lg = charge_lg, + peak_hg = peak_hg, + peak_lg = peak_lg, + run_number = run_number, + pixels_id = pixels_id, + nevents = nevents, + npixels = npixels, + method = method + ) + + charge_container.write(tmp_path) + + loaded_charge_container = ChargeContainer.from_file(tmp_path, run_number) + + assert isinstance(loaded_charge_container, ChargeContainer) + assert np.allclose(loaded_charge_container.charge_hg,charge_hg) + assert np.allclose(loaded_charge_container.charge_lg,charge_lg) + assert np.allclose(loaded_charge_container.peak_hg,peak_hg) + assert np.allclose(loaded_charge_container.peak_lg,peak_lg) + assert loaded_charge_container.run_number == run_number + assert loaded_charge_container.pixels_id.tolist() == pixels_id.tolist() + assert loaded_charge_container.nevents == nevents + assert loaded_charge_container.npixels == npixels + assert loaded_charge_container.method == method + + # Tests that the ChargeContainer object can be sorted by event_id and the object is sorted accordingly. + def test_sort_charge_container(self): + charge_container = create_fake_chargeContainer() + + charge_container.sort() + + assert charge_container.event_id.tolist() == sorted(charge_container.event_id.tolist()) + + # Tests that the run_number, pixels_id, npixels, nevents, method, multiplicity, and trig_pattern properties of the ChargeContainer object can be accessed and the values are correct. + def test_access_properties(self): + pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) + nevents = 40 + npixels = 10 + charge_hg = np.random.randn(nevents,npixels) + charge_lg = np.random.randn(nevents,npixels) + peak_hg = np.random.randn(nevents,npixels) + peak_lg = np.random.randn(nevents,npixels) + run_number = 1234 + method = 'FullWaveformSum' + charge_container = ChargeContainer( + charge_hg = charge_hg , + charge_lg = charge_lg, + peak_hg = peak_hg, + peak_lg = peak_lg, + run_number = run_number, + pixels_id = pixels_id, + nevents = nevents, + npixels = npixels, + method = method + ) + + assert charge_container.run_number == run_number + assert charge_container.pixels_id.tolist() == pixels_id.tolist() + assert charge_container.npixels == npixels + assert charge_container.nevents == nevents + assert charge_container.method == method + assert charge_container.multiplicity.shape == (nevents,) + assert charge_container.trig_pattern.shape == (nevents,4) + + +class TestChargeContainers: + + # Tests that an instance of ChargeContainers can be created with default arguments + def test_create_instance_with_default_arguments(self): + charge_containers = ChargeContainers() + assert len(charge_containers.chargeContainers) == 0 + assert charge_containers.nChargeContainer == 0 + + # Tests that an instance of ChargeContainers can be created from a WaveformsContainers instance + #def test_create_instance_from_waveforms(self): + # waveform_containers = WaveformsContainers(run_number = 1234) + # charge_containers = ChargeContainers.from_waveforms(waveform_containers) + # assert len(charge_containers.chargeContainers) == waveform_containers.nWaveformsContainer + # assert charge_containers.nChargeContainer == waveform_containers.nWaveformsContainer + + # Tests that ChargeContainers can be written to disk + def test_write_to_disk(self, tmpdir): + charge_containers = ChargeContainers() + + charge_container = create_fake_chargeContainer() + + charge_containers.append(charge_container) + path = str(tmpdir) + charge_containers.write(path) + assert len(glob.glob(f"{path}/charge_run*1234*.fits")) == 1 + + # Tests that ChargeContainers can be loaded from disk + def test_load_from_disk(self, tmpdir): + charge_containers = ChargeContainers() + + pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) + nevents = 40 + npixels = 10 + charge_hg = np.random.randn(nevents,npixels) + charge_lg = np.random.randn(nevents,npixels) + peak_hg = np.random.randn(nevents,npixels) + peak_lg = np.random.randn(nevents,npixels) + run_number = np.random.randn(1)[0] + + charge_container = ChargeContainer( + charge_hg = charge_hg , + charge_lg = charge_lg, + peak_hg = peak_hg, + peak_lg = peak_lg, + run_number = run_number, + pixels_id = pixels_id, + nevents = nevents, + npixels = npixels + ) + charge_containers.append(charge_container) + path = str(tmpdir) + charge_containers.write(path) + loaded_charge_containers = ChargeContainers.from_file(path, run_number) + assert len(loaded_charge_containers.chargeContainers) == 1 + assert loaded_charge_containers.nChargeContainer == 1 + + # Tests that a ChargeContainer can be appended to a ChargeContainers instance + def test_append_charge_container(self): + charge_containers = ChargeContainers() + pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) + nevents = 40 + npixels = 10 + charge_hg = np.random.randn(nevents,npixels) + charge_lg = np.random.randn(nevents,npixels) + peak_hg = np.random.randn(nevents,npixels) + peak_lg = np.random.randn(nevents,npixels) + run_number = 1234 + + charge_container = ChargeContainer( + charge_hg = charge_hg , + charge_lg = charge_lg, + peak_hg = peak_hg, + peak_lg = peak_lg, + run_number = run_number, + pixels_id = pixels_id, + nevents = nevents, + npixels = npixels + ) + charge_containers.append(charge_container) + assert len(charge_containers.chargeContainers) == 1 + assert charge_containers.nChargeContainer == 1 + + # Tests that a ChargeContainers instance can be merged into a single ChargeContainer + def test_merge_into_single_charge_container(self): + charge_containers = ChargeContainers() + pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) + nevents = 40 + npixels = 10 + charge_hg = np.random.randn(nevents,npixels) + charge_lg = np.random.randn(nevents,npixels) + peak_hg = np.random.randn(nevents,npixels) + peak_lg = np.random.randn(nevents,npixels) + run_number = 1234 + + charge_container1 = ChargeContainer( + charge_hg = charge_hg , + charge_lg = charge_lg, + peak_hg = peak_hg, + peak_lg = peak_lg, + run_number = run_number, + pixels_id = pixels_id, + nevents = nevents, + npixels = npixels + ) + pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) + nevents = 40 + npixels = 10 + charge_hg = np.random.randn(nevents,npixels) + charge_lg = np.random.randn(nevents,npixels) + peak_hg = np.random.randn(nevents,npixels) + peak_lg = np.random.randn(nevents,npixels) + run_number = 1234 + + charge_container2 = ChargeContainer( + charge_hg = charge_hg , + charge_lg = charge_lg, + peak_hg = peak_hg, + peak_lg = peak_lg, + run_number = run_number, + pixels_id = pixels_id, + nevents = nevents, + npixels = npixels + ) + charge_containers.append(charge_container1) + charge_containers.append(charge_container2) + merged_charge_container = charge_containers.merge() + assert merged_charge_container.nevents == charge_container1.nevents + charge_container2.nevents \ No newline at end of file diff --git a/src/nectarchain/data/container/tests/test_waveforms.py b/src/nectarchain/data/container/tests/test_waveforms.py new file mode 100644 index 00000000..e69de29b diff --git a/src/nectarchain/data/container/waveforms.py b/src/nectarchain/data/container/waveforms.py index c077fa5a..af357241 100644 --- a/src/nectarchain/data/container/waveforms.py +++ b/src/nectarchain/data/container/waveforms.py @@ -17,10 +17,8 @@ from ctapipe.instrument import CameraGeometry,SubarrayDescription,TelescopeDescription from ctapipe_io_nectarcam import NectarCAMEventSource -from ctapipe.containers import EventType -from ctapipe.io import EventSource, EventSeeker -from .utils import DataManagement,ChainGenerator +from ..management import DataManagement import sys import logging diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py index 90b5973b..ed548466 100644 --- a/src/nectarchain/data/management.py +++ b/src/nectarchain/data/management.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import List,Tuple -__all__ = ['DataManagement','ChainGenerator'] +__all__ = ['DataManagement'] class DataManagement() : @staticmethod diff --git a/src/nectarchain/makers/extractor/charge_extractor.py b/src/nectarchain/makers/extractor/charge_extractor.py index fc851fb1..3f75c101 100644 --- a/src/nectarchain/makers/extractor/charge_extractor.py +++ b/src/nectarchain/makers/extractor/charge_extractor.py @@ -40,17 +40,14 @@ def __call__(self, waveforms, telid, selected_gain_channel,substract_ped = False return peak_time, charge_integral - - @guvectorize( [ - (np.uint16[:], np.uint8, np.uint8, np.uint16[:]), + '(np.uint16[:], np.uint8, np.uint8, np.uint16[:])', ], "(n,p,s),(),()->(n,p)", nopython=True, cache=True, ) - def extract_charge(y,height_peak,fixed_window) : x = np.linspace(0, len(y), len(y)) xi = np.linspace(0, len(y), 251) From 72da7ad32b56b877516bcdf7e9cc6eb6c4a9fac5 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 13 Sep 2023 03:03:16 +0200 Subject: [PATCH 38/62] test extractor utlis --- .../makers/extractor/tests/test_utils.py | 20 +++++++++++++++++++ src/nectarchain/makers/extractor/utils.py | 15 +++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 src/nectarchain/makers/extractor/tests/test_utils.py diff --git a/src/nectarchain/makers/extractor/tests/test_utils.py b/src/nectarchain/makers/extractor/tests/test_utils.py new file mode 100644 index 00000000..ee4835f0 --- /dev/null +++ b/src/nectarchain/makers/extractor/tests/test_utils.py @@ -0,0 +1,20 @@ +import pytest + +class TestCtapipeExtractor: + + @pytest.mark.skip('numba conflict') + # Tests that the function returns the image and peak_time values from a valid DL1CameraContainer object. + def test_get_image_peak_time_valid_object(self): + from ctapipe.containers import DL1CameraContainer + from nectarchain.makers.extractor.utils import CtapipeExtractor + # Create a valid DL1CameraContainer object + container = DL1CameraContainer() + container.image = [1, 2, 3, 4, 5] + container.peak_time = [10, 4, 5, 6, 9] + + # Call the function under test + result_image, result_peak_time = CtapipeExtractor.get_image_peak_time(container) + + # Check the result + assert result_image == [1, 2, 3, 4, 5] + assert result_peak_time == [10, 4, 5, 6, 9] \ No newline at end of file diff --git a/src/nectarchain/makers/extractor/utils.py b/src/nectarchain/makers/extractor/utils.py index 3ba701ef..de096258 100644 --- a/src/nectarchain/makers/extractor/utils.py +++ b/src/nectarchain/makers/extractor/utils.py @@ -6,5 +6,18 @@ from ctapipe.containers import DL1CameraContainer class CtapipeExtractor(): - def get_image_peak_time(cameraContainer : DL1CameraContainer) : + """ + A class to extract the image and peak time from a DL1CameraContainer object. + """ + + def get_image_peak_time(cameraContainer): + """ + Extracts the image and peak time from a DL1CameraContainer object. + + Parameters: + cameraContainer (DL1CameraContainer): The DL1CameraContainer object to extract the image and peak time from. + + Returns: + tuple: A tuple containing the image and peak time values from the container. + """ return cameraContainer.image, cameraContainer.peak_time From d89345c7e055ccd3c3dacc6b13bcd2c502528c86 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 13 Sep 2023 03:59:02 +0200 Subject: [PATCH 39/62] unit test CalibrationMaker and GainMakers --- src/nectarchain/makers/calibration/core.py | 2 + .../makers/calibration/gain/gainMakers.py | 104 ++++++++++++++---- .../calibration/gain/tests/test_gainMakers.py | 58 ++++++++++ .../makers/calibration/tests/test_core.py | 58 ++++++++++ 4 files changed, 202 insertions(+), 20 deletions(-) create mode 100644 src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py create mode 100644 src/nectarchain/makers/calibration/tests/test_core.py diff --git a/src/nectarchain/makers/calibration/core.py b/src/nectarchain/makers/calibration/core.py index 3281150c..d6b515da 100644 --- a/src/nectarchain/makers/calibration/core.py +++ b/src/nectarchain/makers/calibration/core.py @@ -24,6 +24,8 @@ class CalibrationMaker(GeneralMaker): _reduced_name (str): A string representing the name of the calibration. PIXELS_ID_COLUMN (str): A string representing the name of the column in the result table that stores the pixels id. NP_PIXELS (str): A string representing the key in the metadata that stores the number of pixels. + + Members: _pixels_id (ndarray): A private property that stores the pixels id. _results (QTable): A private property that stores the result table. """ diff --git a/src/nectarchain/makers/calibration/gain/gainMakers.py b/src/nectarchain/makers/calibration/gain/gainMakers.py index a9adac8c..b48e0de4 100644 --- a/src/nectarchain/makers/calibration/gain/gainMakers.py +++ b/src/nectarchain/makers/calibration/gain/gainMakers.py @@ -12,34 +12,98 @@ __all__ = ["GainMaker"] -class GainMaker(CalibrationMaker) : - """mother class for of the gain calibration +class GainMaker(CalibrationMaker): """ + A class for gain calibration calculations on data. -#constructors - def __init__(self,*args,**kwargs) -> None: - super().__init__(*args,**kwargs) - self.__high_gain = np.empty((self.npixels),dtype = np.float64) - self.__low_gain = np.empty((self.npixels),dtype = np.float64) - self._results.add_column(Column(data = self.__high_gain,name = "high_gain",unit = u.dimensionless_unscaled)) - self._results.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"high_gain_error",unit = u.dimensionless_unscaled)) - self._results.add_column(Column(data = self.__low_gain,name = "low_gain",unit = u.dimensionless_unscaled)) - self._results.add_column(Column(np.empty((self.npixels,2),dtype = np.float64),"low_gain_error",unit = u.dimensionless_unscaled)) - - self._results.add_column(Column(np.zeros((self.npixels),dtype = bool),"is_valid",unit = u.dimensionless_unscaled)) + Inherits from the `CalibrationMaker` class and adds functionality specific to gain calibration. + Members: + __high_gain (ndarray): Private field to store the high gain values. + __low_gain (ndarray): Private field to store the low gain values. + + Methods: + __init__(self, *args, **kwargs): Initializes the `GainMaker` object and sets up the result table with columns for high gain, high gain error, low gain, low gain error, and validity flag. + _high_gain.setter: Sets the high gain values. + high_gain(self): Returns a copy of the high gain values. + _low_gain.setter: Sets the low gain values. + low_gain(self): Returns a copy of the low gain values. + """ + + def __init__(self, *args, **kwargs): + """ + Initializes the `GainMaker` object and sets up the result table with columns for high gain, high gain error, low gain, low gain error, and validity flag. + + Args: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ + super().__init__(*args, **kwargs) + self.__high_gain = np.empty((self.npixels), dtype=np.float64) + self.__low_gain = np.empty((self.npixels), dtype=np.float64) + self._results.add_column(Column(data=self.__high_gain, name="high_gain", unit=u.dimensionless_unscaled)) + self._results.add_column(Column(np.empty((self.npixels, 2), dtype=np.float64), "high_gain_error", unit=u.dimensionless_unscaled)) + self._results.add_column(Column(data=self.__low_gain, name="low_gain", unit=u.dimensionless_unscaled)) + self._results.add_column(Column(np.empty((self.npixels, 2), dtype=np.float64), "low_gain_error", unit=u.dimensionless_unscaled)) + self._results.add_column(Column(np.zeros((self.npixels), dtype=bool), "is_valid", unit=u.dimensionless_unscaled)) -#getters and setters @property - def _high_gain(self) : return self.__high_gain + def _high_gain(self): + """ + Getter for the high gain values. + + Returns: + ndarray: A copy of the high gain values. + """ + return self.__high_gain + @_high_gain.setter - def _high_gain(self,value) : self.__high_gain = value + def _high_gain(self, value): + """ + Setter for the high gain values. + + Args: + value (ndarray): The high gain values. + """ + self.__high_gain = value + @property - def high_gain(self) : return copy(self.__high_gain) + def high_gain(self): + """ + Getter for the high gain values. + + Returns: + ndarray: A copy of the high gain values. + """ + return copy(self.__high_gain) + @property - def _low_gain(self) : return self.__low_gain + def _low_gain(self): + """ + Getter for the low gain values. + + Returns: + ndarray: A copy of the low gain values. + """ + return self.__low_gain + @_low_gain.setter - def _low_gain(self,value) : self.__low_gain = value + def _low_gain(self, value): + """ + Setter for the low gain values. + + Args: + value (ndarray): The low gain values. + """ + self.__low_gain = value + @property - def low_gain(self) : return copy(self.__low_gain) + def low_gain(self): + """ + Getter for the low gain values. + + Returns: + ndarray: A copy of the low gain values. + """ + return copy(self.__low_gain) \ No newline at end of file diff --git a/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py b/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py new file mode 100644 index 00000000..79cbb4a3 --- /dev/null +++ b/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py @@ -0,0 +1,58 @@ + +# Generated by CodiumAI +from nectarchain.makers.calibration.gain.gainMakers import GainMaker +import numpy as np +from pathlib import Path +from astropy.table import QTable + +import pytest + +class GainMakerforTest(GainMaker) : + _reduced_name = "test" + def make() : + pass + +class TestGainMaker: + + # Tests that an instance of GainMaker can be created with a list of pixel ids as input. + def test_create_instance_with_pixel_ids(self): + pixel_ids = [1, 2, 3, 4, 5] + gain_maker = GainMakerforTest(pixel_ids) + assert isinstance(gain_maker, GainMakerforTest) + assert np.array_equal(gain_maker.pixels_id, np.array(pixel_ids)) + + # Tests that high gain values can be set and retrieved for all pixels. + def test_set_and_get_high_gain_values(self): + pixel_ids = [1, 2, 3, 4, 5] + gain_maker = GainMakerforTest(pixel_ids) + high_gain_values = np.array([0.5, 0.6, 0.7, 0.8, 0.9]) + gain_maker._high_gain = high_gain_values + assert np.array_equal(gain_maker.high_gain, high_gain_values) + + # Tests that low gain values can be set and retrieved for all pixels. + def test_set_and_get_low_gain_values(self): + pixel_ids = [1, 2, 3, 4, 5] + gain_maker = GainMakerforTest(pixel_ids) + low_gain_values = np.array([0.1, 0.2, 0.3, 0.4, 0.5]) + gain_maker._low_gain = low_gain_values + assert np.array_equal(gain_maker.low_gain, low_gain_values) + + # Tests that the results can be saved to a file. + def test_save_results_to_file(self, tmp_path = Path('/tmp')): + pixel_ids = [1, 2, 3, 4, 5] + gain_maker = GainMakerforTest(pixel_ids) + high_gain_values = np.array([0.5, 0.6, 0.7, 0.8, 0.9]) + gain_maker._high_gain = high_gain_values + low_gain_values = np.array([0.1, 0.2, 0.3, 0.4, 0.5]) + gain_maker._low_gain = low_gain_values + gain_maker.save(tmp_path) + assert (tmp_path / "results_test.ecsv").exists() + + # Tests that a copy of the result table can be obtained. + def test_get_copy_of_result_table(self): + pixel_ids = [1, 2, 3, 4, 5] + gain_maker = GainMakerforTest(pixel_ids) + result_table_copy = gain_maker.results + assert isinstance(result_table_copy, QTable) + assert np.array_equal(result_table_copy[GainMakerforTest.PIXELS_ID_COLUMN], np.array(pixel_ids)) + diff --git a/src/nectarchain/makers/calibration/tests/test_core.py b/src/nectarchain/makers/calibration/tests/test_core.py new file mode 100644 index 00000000..2d1ef078 --- /dev/null +++ b/src/nectarchain/makers/calibration/tests/test_core.py @@ -0,0 +1,58 @@ +from nectarchain.makers.calibration.core import CalibrationMaker +import numpy as np +from pathlib import Path + +import pytest + +class CalibrationMakerforTest(CalibrationMaker) : + _reduced_name = "test" + def make() : + pass + + +class TestflatfieldMaker: + + # Tests that the constructor initializes the object with the correct attributes and metadata when valid input is provided + def test_constructor_with_valid_input(self): + pixels_id = [1, 2, 3] + calibration_maker = CalibrationMaker(pixels_id) + + assert np.equal(calibration_maker._pixels_id,pixels_id).all() + assert np.equal(calibration_maker._results[calibration_maker.PIXELS_ID_COLUMN],np.array(pixels_id)).all() + assert calibration_maker._results.meta[calibration_maker.NP_PIXELS] == len(pixels_id) + assert isinstance(calibration_maker._results.meta['comments'],str) + + + # Tests that the constructor raises an error when a non-iterable pixels_id is provided + def test_constructor_with_non_iterable_pixels_id(self): + pixels_id = 123 + with pytest.raises(TypeError): + CalibrationMaker(pixels_id) + + # Tests that saving the results to an existing file with overwrite=False raises an error + def test_save_to_existing_file_with_overwrite_false(self, tmp_path = Path('/tmp')): + pixels_id = [1, 2, 3] + calibration_maker = CalibrationMaker(pixels_id) + + # Create a temporary file + file_path = tmp_path / "results_Calibration.ecsv" + file_path.touch() + + with pytest.raises(FileExistsError): + calibration_maker.save(file_path, overwrite=False) + + # Tests that changing the pixels_id attribute updates the results table with the expected values + def test_change_pixels_id_attribute(self): + pixels_id = [1, 2, 3] + calibration_maker = CalibrationMaker(pixels_id) + + new_pixels_id = [4, 5, 6] + calibration_maker._pixels_id = np.array(new_pixels_id) + + assert np.equal(calibration_maker._pixels_id,new_pixels_id).all() + + + # Tests that an instance of CalibrationMaker cannot be created with an empty list of pixel ids as input. + def test_create_instance_with_empty_pixel_ids(self): + with pytest.raises(TypeError): + gain_maker = CalibrationMaker([]) From dbc56efba527bb15309abeb57b17400c6f5caddb Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 13 Sep 2023 06:23:13 +0200 Subject: [PATCH 40/62] unit test flatfieldSPEMakers --- .../data/container/tests/test_charge.py | 1 - .../calibration/gain/FlatFieldSPEMakers.py | 69 ++++++++- .../gain/tests/test_FlatFieldSPEMakers.py | 132 ++++++++++++++++++ .../calibration/gain/tests/test_gainMakers.py | 2 - 4 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py diff --git a/src/nectarchain/data/container/tests/test_charge.py b/src/nectarchain/data/container/tests/test_charge.py index db52191a..64e2cc04 100644 --- a/src/nectarchain/data/container/tests/test_charge.py +++ b/src/nectarchain/data/container/tests/test_charge.py @@ -2,7 +2,6 @@ from nectarchain.data.container.waveforms import WaveformsContainers import glob import numpy as np -import pytest def create_fake_chargeContainer() : pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) diff --git a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py index eadbb3c7..326cbca2 100644 --- a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py @@ -46,6 +46,44 @@ class FlatFieldSPEMaker(GainMaker) : + + """ + The `FlatFieldSPEMaker` class is used for flat field single photoelectron (SPE) calibration calculations on data. It inherits from the `GainMaker` class and adds functionality specific to flat field SPE calibration. + + Example Usage: + # Create an instance of the FlatFieldSPEMaker class + flat_field_maker = FlatFieldSPEMaker() + + # Read parameters from a YAML file + flat_field_maker.read_param_from_yaml("parameters.yaml") + + # Update parameters based on data + flat_field_maker._update_parameters(parameters, charge, counts) + + # Update the result table based on the parameters + flat_field_maker._update_table_from_parameters() + + Main functionalities: + - Inherits from the `GainMaker` class and adds functionality specific to flat field SPE calibration. + - Reads parameters from a YAML file and updates the internal parameters of the class. + - Updates the parameters based on data, such as charge and counts. + - Updates a result table based on the parameters. + + Methods: + - `read_param_from_yaml(parameters_file, only_update)`: Reads parameters from a YAML file and updates the internal parameters of the class. If `only_update` is True, only the parameters that exist in the YAML file will be updated. + - `_update_parameters(parameters, charge, counts, **kwargs)`: Updates the parameters based on data, such as charge and counts. It performs a Gaussian fit on the data to determine the pedestal and mean values, and updates the corresponding parameters accordingly. + - `_get_mean_gaussian_fit(charge, counts, extension, **kwargs)`: Performs a Gaussian fit on the data to determine the pedestal and mean values. It returns the fit coefficients. + - `_update_table_from_parameters()`: Updates a result table based on the parameters. It adds columns to the table for each parameter and its corresponding error. + + Attributes: + - `_Windows_lenght`: A class attribute that represents the length of the windows used for smoothing the data. + - `_Order`: A class attribute that represents the order of the polynomial used for smoothing the data. + + Members: + - `npixels`: A property that returns the number of pixels. + - `parameters`: A property that returns a deep copy of the internal parameters of the class. + - `_parameters`: A property that returns the internal parameters of the class. + """ _Windows_lenght = 40 _Order = 2 @@ -179,8 +217,32 @@ def _update_table_from_parameters(self) -> None: + class FlatFieldSingleHHVSPEMaker(FlatFieldSPEMaker) : - """class to perform fit of the SPE signal with all free parameters""" + """ + class to perform fit of the SPE signal with all free parameters + + Attributes: + - __parameters_file: Private field that stores the name of the parameters file. + - __fit_array: Private field that stores the fit instances. + - _reduced_name: Protected field that stores the name of the reduced data. + - __nproc_default: Private field that stores the default number of processes for multiprocessing. + - __chunksize_default: Private field that stores the default chunk size for multiprocessing. + + Members: + - __charge: Private field that stores the charge data. + - __counts: Private field that stores the counts data. + - __pedestal: Private field that stores the pedestal parameter. + - _parameters: Protected field that stores the parameters. + - _results: Protected field that stores the fit results. + + Methods: + - __init__(self, charge, counts, *args, **kwargs): Constructor method that initializes the instance with the charge and counts data. + - create_from_chargeContainer(cls, signal, **kwargs): Class method that creates an instance from a charge container. + - create_from_run_number(cls, run_number, **kwargs): Class method that creates an instance from a run number. + - make(self, pixels_id=None, multiproc=True, display=True, **kwargs): Method that performs the fit on the specified pixels and returns the fit results. + - display(self, pixels_id, **kwargs): Method that plots the fit for the specified pixels. + """ __parameters_file = 'parameters_signal.yaml' __fit_array = None @@ -188,6 +250,7 @@ class FlatFieldSingleHHVSPEMaker(FlatFieldSPEMaker) : __nproc_default = 8 __chunksize_default = 1 + #constructors def __init__(self,charge,counts,*args,**kwargs) -> None: super().__init__(*args,**kwargs) @@ -214,6 +277,10 @@ def __init__(self,charge,counts,*args,**kwargs) -> None: def create_from_chargeContainer(cls, signal : ChargeContainer,**kwargs) : histo = signal.histo_hg(autoscale = True) return cls(charge = histo[1],counts = histo[0],pixels_id = signal.pixels_id,**kwargs) + + @classmethod + def create_from_run_number(cls, run_number : int, **kwargs) : + raise NotImplementedError() #getters and setters @property diff --git a/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py new file mode 100644 index 00000000..dc3822f6 --- /dev/null +++ b/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py @@ -0,0 +1,132 @@ +from nectarchain.makers.calibration.gain.FlatFieldSPEMakers import FlatFieldSPEMaker +from nectarchain.makers.calibration.gain.parameters import Parameter,Parameters +from nectarchain.makers.calibration.gain import FlatFieldSingleHHVSPEMaker +import astropy.units as u +from nectarchain.data.container import ChargeContainer +import numpy as np + +import pytest + +class FlatFieldSPEMakerforTest(FlatFieldSPEMaker) : + def make() : + pass + +def create_fake_chargeContainer() : + pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) + nevents = 40 + npixels = 10 + charge_hg = np.random.randint(nevents,npixels) + charge_lg = np.random.randint(nevents,npixels) + peak_hg = np.random.randint(nevents,npixels) + peak_lg = np.random.randint(nevents,npixels) + run_number = 1234 + return ChargeContainer( + charge_hg = charge_hg , + charge_lg = charge_lg, + peak_hg = peak_hg, + peak_lg = peak_lg, + run_number = run_number, + pixels_id = pixels_id, + nevents = nevents, + npixels = npixels + ) + +class TestFlatFieldSPEMaker: + + # Tests that the object can be initialized without errors + def test_initialize_object(self): + pixels_id = [2,3,5] + flat_field_spe_maker = FlatFieldSPEMakerforTest(pixels_id) + assert isinstance(flat_field_spe_maker, FlatFieldSPEMakerforTest) + + # Tests that parameters can be read from a YAML file + def test_read_parameters_from_yaml(self): + pixels_id = [2,3,5] + flat_field_spe_maker = FlatFieldSPEMakerforTest(pixels_id) + flat_field_spe_maker.read_param_from_yaml("parameters_signal.yaml") + assert flat_field_spe_maker.parameters.size == 6 + assert isinstance(flat_field_spe_maker.parameters,Parameters) + + # Tests that parameters can be updated from a YAML file + def test_update_parameters_from_yaml(self): + pixels_id = [2,3,5] + flat_field_spe_maker = FlatFieldSPEMakerforTest(pixels_id) + flat_field_spe_maker.read_param_from_yaml("parameters_signal.yaml") + flat_field_spe_maker.read_param_from_yaml("parameters_signalStd.yaml",only_update = True) + assert flat_field_spe_maker.parameters.parameters[-2].value == 0.697 + + # Tests that parameters can be updated from a fit + def test_update_parameters_from_fit(self): + pixels_id = [2] + flat_field_spe_maker = FlatFieldSPEMakerforTest(pixels_id) + flat_field_spe_maker.read_param_from_yaml("parameters_signal.yaml") + updated_parameters = flat_field_spe_maker._update_parameters(flat_field_spe_maker.parameters, charge=[1, 2, 3,4,5,6,7,8,9,10], counts=[1,3,9,5,3,5,6,3,2,1]) + + # Tests that the table can be updated from parameters + def test_update_table_from_parameters(self): + pixels_id = [2,3,5] + flat_field_spe_maker = FlatFieldSPEMakerforTest(pixels_id) + flat_field_spe_maker._parameters.append(Parameter(name="param1", value=1, unit=u.dimensionless_unscaled)) + flat_field_spe_maker._parameters.append(Parameter(name="param2", value=2, unit=u.dimensionless_unscaled)) + + flat_field_spe_maker._update_table_from_parameters() + + assert "param1" in flat_field_spe_maker._results.colnames + assert "param1_error" in flat_field_spe_maker._results.colnames + assert "param2" in flat_field_spe_maker._results.colnames + assert "param2_error" in flat_field_spe_maker._results.colnames + + +class TestFlatFieldSingleHHVSPEMaker: + + # Tests that creating an instance of FlatFieldSingleHHVSPEMaker with valid input parameters is successful + def test_create_instance_valid_input(self): + charge = [1, 2, 3] + counts = [10, 20, 30] + pixels_id = [2,3,5] + maker = FlatFieldSingleHHVSPEMaker(charge, counts, pixels_id) + assert isinstance(maker, FlatFieldSingleHHVSPEMaker) + + # Tests that creating an instance of FlatFieldSingleHHVSPEMaker with invalid input parameters raises an error + def test_create_instance_invalid_input(self): + charge = [1, 2, 3] + counts = [10, 20] # Invalid input, counts and charge must have the same length + pixels_id = [2,3,5] + + with pytest.raises(Exception): + FlatFieldSingleHHVSPEMaker(charge, counts,pixels_id) + + # Tests that calling create_from_chargeContainer method with valid input parameters is successful + def test_create_from_chargeContainer_valid_input(self): + chargeContainer = create_fake_chargeContainer() + maker = FlatFieldSingleHHVSPEMaker.create_from_chargeContainer(chargeContainer) + assert isinstance(maker, FlatFieldSingleHHVSPEMaker) + + + def test_fill_results_table_from_dict(self): + pass + def test_NG_Likelihood_Chi2(self):pass + def test_cost(self):pass + def test_make_fit_array_from_parameters(self):pass + def test_run_fit(self) : pass + def test_make(self):pass + def test_plot_single(self) : pass + def test_display(self) : pass + + + + # Tests that calling make method with valid input parameters is successful + def test_make_valid_input(self): + charge = [1, 2, 3] + counts = [10, 20, 30] + maker = FlatFieldSingleHHVSPEMaker(charge, counts) + result = maker.make() + assert isinstance(result, np.ndarray) + + # Tests that calling make method with invalid input parameters raises an error + def test_make_invalid_input(self, mocker): + charge = [1, 2, 3] + counts = [10, 20] # Invalid input, counts and charge must have the same length + maker = FlatFieldSingleHHVSPEMaker(charge, counts) + with pytest.raises(Exception): + maker.make() \ No newline at end of file diff --git a/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py b/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py index 79cbb4a3..013c2601 100644 --- a/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py +++ b/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py @@ -5,8 +5,6 @@ from pathlib import Path from astropy.table import QTable -import pytest - class GainMakerforTest(GainMaker) : _reduced_name = "test" def make() : From 0c428bb79c9b3b1c09c40f6858f47c8348ea9e6f Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 13 Sep 2023 17:00:54 +0200 Subject: [PATCH 41/62] unit test implementation follow up --- src/nectarchain/data/container/charge.py | 2 +- .../data/container/tests/test_charge.py | 9 +++-- .../calibration/gain/FlatFieldSPEMakers.py | 10 ++++- .../gain/tests/test_FlatFieldSPEMakers.py | 38 +++++++++---------- .../calibration/gain/tests/test_gainMakers.py | 2 +- .../makers/calibration/tests/test_core.py | 10 ++--- 6 files changed, 37 insertions(+), 34 deletions(-) diff --git a/src/nectarchain/data/container/charge.py b/src/nectarchain/data/container/charge.py index 6f0608d2..26a60d5b 100644 --- a/src/nectarchain/data/container/charge.py +++ b/src/nectarchain/data/container/charge.py @@ -364,7 +364,7 @@ def compute_charge(waveformContainer : WaveformsContainer,channel : int,method : else : raise ArgumentError(f"channel must be {constants.LOW_GAIN} or {constants.HIGH_GAIN}") - def histo_hg(self,n_bins : int = 1000,autoscale : bool = True) -> np.ndarray: + def histo_hg(self,n_bins : int = 1000,autoscale : bool = True) -> ma.masked_array: """method to compute histogram of HG channel Numba is used to compute histograms in vectorized way diff --git a/src/nectarchain/data/container/tests/test_charge.py b/src/nectarchain/data/container/tests/test_charge.py index 64e2cc04..8018ed09 100644 --- a/src/nectarchain/data/container/tests/test_charge.py +++ b/src/nectarchain/data/container/tests/test_charge.py @@ -7,10 +7,11 @@ def create_fake_chargeContainer() : pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) nevents = 40 npixels = 10 - charge_hg = np.random.randn(nevents,npixels) - charge_lg = np.random.randn(nevents,npixels) - peak_hg = np.random.randn(nevents,npixels) - peak_lg = np.random.randn(nevents,npixels) + rng = np.random.default_rng() + charge_hg = rng.integers(low=0, high=1000, size= (nevents,npixels)) + charge_lg = rng.integers(low=0, high=1000, size= (nevents,npixels)) + peak_hg = rng.integers(low=0, high=60, size= (nevents,npixels)) + peak_lg = rng.integers(low=0, high=60, size= (nevents,npixels)) run_number = 1234 return ChargeContainer( charge_hg = charge_hg , diff --git a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py index 326cbca2..c138a6cc 100644 --- a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py @@ -254,8 +254,14 @@ class to perform fit of the SPE signal with all free parameters #constructors def __init__(self,charge,counts,*args,**kwargs) -> None: super().__init__(*args,**kwargs) - self.__charge = charge - self.__counts = counts + if isinstance(charge,np.ma.masked_array) : + self.__charge = charge + else : + self.__charge = np.ma.asarray(charge) + if isinstance(counts,np.ma.masked_array) : + self.__counts = counts + else : + self.__counts = np.ma.asarray(counts) self.__pedestal = Parameter(name = "pedestal", value = (np.min(self.__charge) + np.sum(self.__charge * self.__counts)/(np.sum(self.__counts)))/2, diff --git a/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py index dc3822f6..376e1f68 100644 --- a/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py @@ -1,6 +1,6 @@ from nectarchain.makers.calibration.gain.FlatFieldSPEMakers import FlatFieldSPEMaker from nectarchain.makers.calibration.gain.parameters import Parameter,Parameters -from nectarchain.makers.calibration.gain import FlatFieldSingleHHVSPEMaker +from nectarchain.makers.calibration.gain import FlatFieldSingleHHVSPEMaker,FlatFieldSingleHHVStdSPEMaker import astropy.units as u from nectarchain.data.container import ChargeContainer import numpy as np @@ -15,10 +15,11 @@ def create_fake_chargeContainer() : pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) nevents = 40 npixels = 10 - charge_hg = np.random.randint(nevents,npixels) - charge_lg = np.random.randint(nevents,npixels) - peak_hg = np.random.randint(nevents,npixels) - peak_lg = np.random.randint(nevents,npixels) + rng = np.random.default_rng() + charge_hg = rng.integers(low=0, high=1000, size= (nevents,npixels)) + charge_lg = rng.integers(low=0, high=1000, size= (nevents,npixels)) + peak_hg = rng.integers(low=0, high=60, size= (nevents,npixels)) + peak_lg = rng.integers(low=0, high=60, size= (nevents,npixels)) run_number = 1234 return ChargeContainer( charge_hg = charge_hg , @@ -97,7 +98,7 @@ def test_create_instance_invalid_input(self): FlatFieldSingleHHVSPEMaker(charge, counts,pixels_id) # Tests that calling create_from_chargeContainer method with valid input parameters is successful - def test_create_from_chargeContainer_valid_input(self): + def test_create_from_ChargeContainer_valid_input(self): chargeContainer = create_fake_chargeContainer() maker = FlatFieldSingleHHVSPEMaker.create_from_chargeContainer(chargeContainer) assert isinstance(maker, FlatFieldSingleHHVSPEMaker) @@ -113,20 +114,15 @@ def test_make(self):pass def test_plot_single(self) : pass def test_display(self) : pass - - - # Tests that calling make method with valid input parameters is successful - def test_make_valid_input(self): +class TestFlatFieldSingleHHVStdSPEMaker: + def test_create_instance(self): charge = [1, 2, 3] - counts = [10, 20, 30] - maker = FlatFieldSingleHHVSPEMaker(charge, counts) - result = maker.make() - assert isinstance(result, np.ndarray) + counts = [10, 20,30] # Invalid input, counts and charge must have the same length + pixels_id = [2,3,5] + instance = FlatFieldSingleHHVStdSPEMaker(charge, counts,pixels_id) + assert isinstance(instance,FlatFieldSingleHHVStdSPEMaker) - # Tests that calling make method with invalid input parameters raises an error - def test_make_invalid_input(self, mocker): - charge = [1, 2, 3] - counts = [10, 20] # Invalid input, counts and charge must have the same length - maker = FlatFieldSingleHHVSPEMaker(charge, counts) - with pytest.raises(Exception): - maker.make() \ No newline at end of file + +class TestFlatFieldSingleNominalSPEMaker: + def test_create_instance(self): + pass diff --git a/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py b/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py index 013c2601..1acbac89 100644 --- a/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py +++ b/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py @@ -36,7 +36,7 @@ def test_set_and_get_low_gain_values(self): assert np.array_equal(gain_maker.low_gain, low_gain_values) # Tests that the results can be saved to a file. - def test_save_results_to_file(self, tmp_path = Path('/tmp')): + def test_save_results_to_file(self, tmp_path = Path(f'/tmp/{np.random.rand()}')): pixel_ids = [1, 2, 3, 4, 5] gain_maker = GainMakerforTest(pixel_ids) high_gain_values = np.array([0.5, 0.6, 0.7, 0.8, 0.9]) diff --git a/src/nectarchain/makers/calibration/tests/test_core.py b/src/nectarchain/makers/calibration/tests/test_core.py index 2d1ef078..a06eccd5 100644 --- a/src/nectarchain/makers/calibration/tests/test_core.py +++ b/src/nectarchain/makers/calibration/tests/test_core.py @@ -15,7 +15,7 @@ class TestflatfieldMaker: # Tests that the constructor initializes the object with the correct attributes and metadata when valid input is provided def test_constructor_with_valid_input(self): pixels_id = [1, 2, 3] - calibration_maker = CalibrationMaker(pixels_id) + calibration_maker = CalibrationMakerforTest(pixels_id) assert np.equal(calibration_maker._pixels_id,pixels_id).all() assert np.equal(calibration_maker._results[calibration_maker.PIXELS_ID_COLUMN],np.array(pixels_id)).all() @@ -27,12 +27,12 @@ def test_constructor_with_valid_input(self): def test_constructor_with_non_iterable_pixels_id(self): pixels_id = 123 with pytest.raises(TypeError): - CalibrationMaker(pixels_id) + CalibrationMakerforTest(pixels_id) # Tests that saving the results to an existing file with overwrite=False raises an error def test_save_to_existing_file_with_overwrite_false(self, tmp_path = Path('/tmp')): pixels_id = [1, 2, 3] - calibration_maker = CalibrationMaker(pixels_id) + calibration_maker = CalibrationMakerforTest(pixels_id) # Create a temporary file file_path = tmp_path / "results_Calibration.ecsv" @@ -44,7 +44,7 @@ def test_save_to_existing_file_with_overwrite_false(self, tmp_path = Path('/tmp' # Tests that changing the pixels_id attribute updates the results table with the expected values def test_change_pixels_id_attribute(self): pixels_id = [1, 2, 3] - calibration_maker = CalibrationMaker(pixels_id) + calibration_maker = CalibrationMakerforTest(pixels_id) new_pixels_id = [4, 5, 6] calibration_maker._pixels_id = np.array(new_pixels_id) @@ -55,4 +55,4 @@ def test_change_pixels_id_attribute(self): # Tests that an instance of CalibrationMaker cannot be created with an empty list of pixel ids as input. def test_create_instance_with_empty_pixel_ids(self): with pytest.raises(TypeError): - gain_maker = CalibrationMaker([]) + gain_maker = CalibrationMakerforTest([]) From 7db7034fbfababa0eacb4ded34c57d3ff0f6b88a Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Wed, 13 Sep 2023 22:30:17 +0200 Subject: [PATCH 42/62] mv wavefomrs in makers --- .../{data/container/waveforms.py => makers/waveformsMakers.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/nectarchain/{data/container/waveforms.py => makers/waveformsMakers.py} (100%) diff --git a/src/nectarchain/data/container/waveforms.py b/src/nectarchain/makers/waveformsMakers.py similarity index 100% rename from src/nectarchain/data/container/waveforms.py rename to src/nectarchain/makers/waveformsMakers.py From 7c3b9077ec345fc2b6b8aee8b704ddfeb284f0fb Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 14 Sep 2023 04:42:10 +0200 Subject: [PATCH 43/62] code into WavefomContainers split into 2 parts : one for the data format, the other for the computation --- src/nectarchain/data/container/__init__.py | 2 +- src/nectarchain/data/container/charge.py | 9 +- .../data/container/waveformsContainer.py | 242 +++++++ src/nectarchain/makers/__init__.py | 3 +- src/nectarchain/makers/calibration/core.py | 4 +- src/nectarchain/makers/core.py | 29 +- .../makers/tests/test_waveformsMakers.py | 61 ++ src/nectarchain/makers/waveformsMakers.py | 625 ++++++------------ 8 files changed, 546 insertions(+), 429 deletions(-) create mode 100644 src/nectarchain/data/container/waveformsContainer.py create mode 100644 src/nectarchain/makers/tests/test_waveformsMakers.py diff --git a/src/nectarchain/data/container/__init__.py b/src/nectarchain/data/container/__init__.py index 8eae44c4..2bb52a30 100644 --- a/src/nectarchain/data/container/__init__.py +++ b/src/nectarchain/data/container/__init__.py @@ -1,2 +1,2 @@ -from .waveforms import * +from .waveformsContainer import * from .charge import * \ No newline at end of file diff --git a/src/nectarchain/data/container/charge.py b/src/nectarchain/data/container/charge.py index 26a60d5b..1ee7604b 100644 --- a/src/nectarchain/data/container/charge.py +++ b/src/nectarchain/data/container/charge.py @@ -19,11 +19,11 @@ from numba import guvectorize, float64, int64, bool_ -from .waveforms import WaveformsContainer,WaveformsContainers +from .waveformsContainer import WaveformsContainer -__all__ = ['ChargeContainer','ChargeContainers'] +__all__ = ['ChargeContainer'] list_ctapipe_charge_extractor = ["FullWaveformSum", "FixedWindowSum", @@ -504,7 +504,7 @@ def multiplicity(self) : return np.uint16(np.count_nonzero(self.trig_pattern,ax @property def trig_pattern(self) : return self.trig_pattern_all.any(axis = 1) - +''' class ChargeContainers() : """ The `ChargeContainers` class is used to store and manipulate a collection of `ChargeContainer` objects. It provides methods for creating, writing, and merging `ChargeContainer` instances. @@ -649,4 +649,5 @@ def merge(self) -> ChargeContainer : cls.sort() - return cls \ No newline at end of file + return cls +''' \ No newline at end of file diff --git a/src/nectarchain/data/container/waveformsContainer.py b/src/nectarchain/data/container/waveformsContainer.py new file mode 100644 index 00000000..9673f013 --- /dev/null +++ b/src/nectarchain/data/container/waveformsContainer.py @@ -0,0 +1,242 @@ +import sys +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +import os +from ctapipe.containers import Container,Field +from ctapipe.io import TableWriter,TableLoader +from ctapipe.instrument.subarray import SubarrayDescription +import numpy as np + +from pathlib import Path + +from enum import Enum + +from tqdm import tqdm + +from astropy.io import fits +from astropy.table import QTable,Column,Table +import astropy.units as u +from abc import ABC + +class WaveformsContainer(Container): + """ + A container that holds information about waveforms from a specific run. + + Attributes: + run_number (int): The run number associated with the waveforms. + nevents (int): The number of events. + npixels (int): The number of pixels. + nsamples (int): The number of samples in the waveforms. + subarray (str): The name of the subarray. + camera (str): The name of the camera. + pixels_id (np.ndarray): An array of pixel IDs. + wfs_hg (np.ndarray): An array of high gain waveforms. + wfs_lg (np.ndarray): An array of low gain waveforms. + broken_pixels_hg (np.ndarray): An array of high gain broken pixels. + broken_pixels_lg (np.ndarray): An array of low gain broken pixels. + ucts_timestamp (np.ndarray): An array of events' UCTS timestamps. + ucts_busy_counter (np.ndarray): An array of UCTS busy counters. + ucts_event_counter (np.ndarray): An array of UCTS event counters. + event_type (np.ndarray): An array of trigger event types. + event_id (np.ndarray): An array of event IDs. + trig_pattern_all (np.ndarray): An array of trigger patterns. + trig_pattern (np.ndarray): An array of reduced trigger patterns. + multiplicity (np.ndarray): An array of events' multiplicities. + """ + + run_number = Field( + type=int, + description="run number associated to the waveforms", + ) + nevents = Field( + type=int, + description="number of events", + ) + npixels = Field( + type=int, + description="number of effective pixels", + ) + nsamples = Field( + type=int, + description="number of samples in the waveforms", + ) + subarray = Field( + type=SubarrayDescription, + description="The subarray description" + ) + camera = Field( + type=str, + description="camera name", + ) + pixels_id = Field( + type=np.ndarray, + description="pixel ids" + ) + wfs_hg = Field( + type=np.ndarray, + description="high gain waveforms" + ) + wfs_lg = Field( + type=np.ndarray, + description="low gain waveforms" + ) + broken_pixels_hg = Field( + type=np.ndarray, + description="high gain broken pixels" + ) + broken_pixels_lg = Field( + type=np.ndarray, + description="low gain broken pixels" + ) + ucts_timestamp = Field( + type=np.ndarray, + description="events ucts timestamp" + ) + ucts_busy_counter = Field( + type=np.ndarray, + description="ucts busy counter" + ) + ucts_event_counter = Field( + type=np.ndarray, + description="ucts event counter" + ) + event_type = Field( + type=np.ndarray, + description="trigger event type" + ) + event_id = Field( + type=np.ndarray, + description="event ids" + ) + trig_pattern_all = Field( + type=np.ndarray, + description="trigger pattern" + ) + trig_pattern = Field( + type=np.ndarray, + description="reduced trigger pattern" + ) + multiplicity = Field( + type=np.ndarray, + description="events multiplicity" + ) + + +class WaveformsContainerIO(ABC) : + + @staticmethod + def write(path : str, containers : WaveformsContainer, **kwargs): + '''method to write in an output FITS file the WaveformsContainer. Two files are created, one FITS representing the data + and one HDF5 file representing the subarray configuration + + Args: + path (str): the directory where you want to save data + ''' + suffix = kwargs.get("suffix","") + if suffix != "" : suffix = f"_{suffix}" + + log.info(f"saving in {path}") + os.makedirs(path,exist_ok = True) + + hdr = fits.Header() + hdr['RUN'] = containers.run_number + hdr['NEVENTS'] = containers.nevents + hdr['NPIXELS'] = containers.npixels + hdr['NSAMPLES'] = containers.nsamples + hdr['SUBARRAY'] = containers.subarray.name + hdr['CAMERA'] = containers.camera + + + containers.subarray.to_hdf(f"{Path(path)}/subarray_run{containers.run_number}{suffix}.hdf5",overwrite=kwargs.get('overwrite',False)) + + + + hdr['COMMENT'] = f"The waveforms containeur for run {containers.run_number} : primary is the pixels id, 2nd HDU : high gain waveforms, 3rd HDU : low gain waveforms, 4th HDU : event properties and 5th HDU trigger paterns." + + primary_hdu = fits.PrimaryHDU(containers.pixels_id,header=hdr) + + wfs_hg_hdu = fits.ImageHDU(containers.wfs_hg,name = "HG Waveforms") + wfs_lg_hdu = fits.ImageHDU(containers.wfs_lg,name = "LG Waveforms") + + col1 = fits.Column(array = containers.broken_pixels_hg, name = "HG broken pixels", format = f'{containers.broken_pixels_hg.shape[1]}L') + col2 = fits.Column(array = containers.broken_pixels_lg, name = "LG broken pixels", format = f'{containers.broken_pixels_lg.shape[1]}L') + coldefs = fits.ColDefs([col1, col2]) + broken_pixels = fits.BinTableHDU.from_columns(coldefs,name = 'trigger patern') + + col1 = fits.Column(array = containers.event_id, name = "event_id", format = '1J') + col2 = fits.Column(array = containers.event_type, name = "event_type", format = '1I') + col3 = fits.Column(array = containers.ucts_timestamp, name = "ucts_timestamp", format = '1K') + col4 = fits.Column(array = containers.ucts_busy_counter, name = "ucts_busy_counter", format = '1J') + col5 = fits.Column(array = containers.ucts_event_counter, name = "ucts_event_counter", format = '1J') + col6 = fits.Column(array = containers.multiplicity, name = "multiplicity", format = '1I') + + coldefs = fits.ColDefs([col1, col2, col3, col4, col5, col6]) + event_properties = fits.BinTableHDU.from_columns(coldefs,name = 'event properties') + + col1 = fits.Column(array = containers.trig_pattern_all, name = "trig_pattern_all", format = f'{4 * containers.trig_pattern_all.shape[1]}L',dim = f'({containers.trig_pattern_all.shape[1]},4)') + col2 = fits.Column(array = containers.trig_pattern, name = "trig_pattern", format = f'{containers.trig_pattern_all.shape[1]}L') + coldefs = fits.ColDefs([col1, col2]) + trigger_patern = fits.BinTableHDU.from_columns(coldefs,name = 'trigger patern') + + hdul = fits.HDUList([primary_hdu, wfs_hg_hdu, wfs_lg_hdu,broken_pixels,event_properties,trigger_patern]) + try : + hdul.writeto(Path(path)/f"waveforms_run{containers.run_number}{suffix}.fits",overwrite=kwargs.get('overwrite',False)) + log.info(f"run saved in {Path(path)}/waveforms_run{containers.run_number}{suffix}.fits") + except OSError as e : + log.warning(e) + except Exception as e : + log.error(e,exc_info = True) + raise e + + @staticmethod + def load(path : str) : + '''load WaveformsContainer from FITS file previously written with WaveformsContainer.write() method + Note : 2 files are loaded, the FITS one representing the waveforms data and a HDF5 file representing the subarray configuration. + This second file has to be next to the FITS file. + + Args: + path (str): path of the FITS file + + Returns: + WaveformsContainer: WaveformsContainer instance + ''' + log.info(f"loading from {path}") + with fits.open(Path(path)) as hdul : + containers = WaveformsContainer() + + containers.run_number = hdul[0].header['RUN'] + containers.nevents = hdul[0].header['NEVENTS'] + containers.npixels = hdul[0].header['NPIXELS'] + containers.nsamples = hdul[0].header['NSAMPLES'] + containers.camera = hdul[0].header['CAMERA'] + + + containers.subarray = SubarrayDescription.from_hdf(Path(path.replace('waveforms_','subarray_').replace('fits','hdf5'))) + + + containers.pixels_id = hdul[0].data + containers.wfs_hg = hdul[1].data + containers.wfs_lg = hdul[2].data + + broken_pixels = hdul[3].data + containers.broken_pixels_hg = broken_pixels["HG broken pixels"] + containers.broken_pixels_lg = broken_pixels["LG broken pixels"] + + + + table_prop = hdul[4].data + containers.event_id = table_prop["event_id"] + containers.event_type = table_prop["event_type"] + containers.ucts_timestamp = table_prop["ucts_timestamp"] + containers.ucts_busy_counter = table_prop["ucts_busy_counter"] + containers.ucts_event_counter = table_prop["ucts_event_counter"] + containers.multiplicity = table_prop["multiplicity"] + + table_trigger = hdul[5].data + containers.trig_pattern_all = table_trigger["trig_pattern_all"] + containers.trig_pattern = table_trigger["trig_pattern"] + + return containers \ No newline at end of file diff --git a/src/nectarchain/makers/__init__.py b/src/nectarchain/makers/__init__.py index a8ce586c..a467a348 100644 --- a/src/nectarchain/makers/__init__.py +++ b/src/nectarchain/makers/__init__.py @@ -1 +1,2 @@ -from .core import * \ No newline at end of file +from .core import * +from .waveformsMakers import * \ No newline at end of file diff --git a/src/nectarchain/makers/calibration/core.py b/src/nectarchain/makers/calibration/core.py index d6b515da..7026d963 100644 --- a/src/nectarchain/makers/calibration/core.py +++ b/src/nectarchain/makers/calibration/core.py @@ -12,11 +12,11 @@ from datetime import date from collections.abc import Iterable -from ..core import GeneralMaker +from ..core import BaseMaker __all__ = [""] -class CalibrationMaker(GeneralMaker): +class CalibrationMaker(BaseMaker): """ Mother class for all calibration makers that can be defined to compute calibration coefficients from data. diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index ba583269..0c91e598 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -5,9 +5,13 @@ from abc import ABC, abstractmethod -__all__ = ["GeneralMaker"] +from ctapipe_io_nectarcam import NectarCAMEventSource -class GeneralMaker(ABC): +from ..data import DataManagement + +__all__ = ["BaseMaker"] + +class BaseMaker(ABC): """Mother class for all the makers, the role of makers is to do computation on the data. """ @abstractmethod @@ -16,4 +20,23 @@ def make(self, *args, **kwargs): Abstract method that needs to be implemented by subclasses. This method is the main one, which computes and does the work. """ - pass \ No newline at end of file + pass + @staticmethod + def load_run(run_number : int,max_events : int = None, run_file = None) : + """Static method to load from $NECTARCAMDATA directory data for specified run with max_events + + Args:self.__run_number = run_number + run_number (int): run_id + maxevents (int, optional): max of events to be loaded. Defaults to 0, to load everythings. + run_file (optional) : if provided, will load this run file + Returns: + List[ctapipe_io_nectarcam.NectarCAMEventSource]: List of EventSource for each run files + """ + if run_file is None : + generic_filename,_ = DataManagement.findrun(run_number) + log.info(f"{str(generic_filename)} will be loaded") + eventsource = NectarCAMEventSource(input_url=generic_filename,max_events=max_events) + else : + log.info(f"{run_file} will be loaded") + eventsource = NectarCAMEventSource(input_url=run_file,max_events=max_events) + return eventsource \ No newline at end of file diff --git a/src/nectarchain/makers/tests/test_waveformsMakers.py b/src/nectarchain/makers/tests/test_waveformsMakers.py new file mode 100644 index 00000000..a6f9036c --- /dev/null +++ b/src/nectarchain/makers/tests/test_waveformsMakers.py @@ -0,0 +1,61 @@ +from nectarchain.makers.waveformsMakers import WaveformsMaker +from nectarchain.data.container import WaveformsContainer,WaveformsContainerIO +from ctapipe.containers import EventType +import numpy as np +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level = logging.DEBUG) +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +class TestWaveformsMaker: + run_number = 3938 + max_events = 1000 + + def test_instance(self) : + waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, + max_events = TestWaveformsMaker.max_events) + assert isinstance(waveformsMaker,WaveformsMaker) + + def test_all_multiple_trigger(self) : + trigger1 = EventType.FLATFIELD + trigger2 = EventType.SKY_PEDESTAL + waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, + max_events = TestWaveformsMaker.max_events) + waveformsContainer_list = waveformsMaker.make(trigger_type = [trigger1,trigger2],restart_from_begining = True) + for waveformsContainer in waveformsContainer_list : + assert isinstance(waveformsContainer,WaveformsContainer) + + + def test_all_trigger_None(self) : + waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, + max_events = TestWaveformsMaker.max_events) + waveformsContainer_list = waveformsMaker.make() + assert isinstance(waveformsContainer_list[0],WaveformsContainer) + + def test_select_waveforms_hg(self) : + waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, + max_events = TestWaveformsMaker.max_events) + waveformsContainer_list = waveformsMaker.make() + pixel_id = np.array([3,67,87]) + WaveformsMaker.select_waveforms_hg(waveformsContainer_list[0],pixel_id) + + def test_sort_WaveformsContainer(self) : + waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, + max_events = TestWaveformsMaker.max_events) + waveformsContainer_list = waveformsMaker.make() + sortWfs = WaveformsMaker.sort(waveformsContainer_list[0],method = 'event_id') + assert np.array_equal(sortWfs.event_id ,np.sort(waveformsContainer_list[0].event_id)) + + def test_write_load_container(self) : + waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, + max_events = TestWaveformsMaker.max_events) + waveformsContainer_list = waveformsMaker.make() + WaveformsContainerIO.write("/tmp/test_wfs_container/",waveformsContainer_list[0],overwrite = True) + loaded_wfs = WaveformsContainerIO.load(f"/tmp/test_wfs_container/waveforms_run{TestWaveformsMaker.run_number}.fits") + +if __name__ == '__main__' : + import logging + logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level = logging.DEBUG) + log = logging.getLogger(__name__) + log.handlers = logging.getLogger('__main__').handlers + TestWaveformsMaker().test_write_load_container() \ No newline at end of file diff --git a/src/nectarchain/makers/waveformsMakers.py b/src/nectarchain/makers/waveformsMakers.py index af357241..dd1910b1 100644 --- a/src/nectarchain/makers/waveformsMakers.py +++ b/src/nectarchain/makers/waveformsMakers.py @@ -8,6 +8,8 @@ from enum import Enum +from tqdm import tqdm + from astropy.io import fits from astropy.table import QTable,Column,Table import astropy.units as u @@ -18,7 +20,7 @@ from ctapipe_io_nectarcam import NectarCAMEventSource -from ..management import DataManagement +from ..data import DataManagement import sys import logging @@ -26,82 +28,59 @@ log = logging.getLogger(__name__) log.handlers = logging.getLogger('__main__').handlers -__all__ = ["WaveformsContainer","WaveformsContainers"] - +from ctapipe.containers import EventType +from ..data.container import WaveformsContainer + +from nectarchain.makers import BaseMaker -class WaveformsContainer() : - """class used to load run and load waveforms from r0 data +__all__ = ["WaveformsMaker"] + +class WaveformsMaker(BaseMaker) : + """class use to make the waveform extraction from event read from r0 data """ TEL_ID = 0 - CAMERA = CameraGeometry.from_name("NectarCam-003") - def __new__(cls,*args,**kwargs) : - obj = object.__new__(cls) - return obj + CAMERA_NAME = "NectarCam-003" + CAMERA = CameraGeometry.from_name(CAMERA_NAME) - def __init__(self,run_number : int,max_events : int = None,nevents : int = -1,run_file = None, init_arrays : bool = False): +#constructors + def __init__(self,run_number : int,max_events : int = None,run_file = None,*args,**kwargs): """construtor Args: run_number (int): id of the run to be loaded maxevents (int, optional): max of events to be loaded. Defaults to 0, to load everythings. nevents (int, optional) : number of events in run if known (parameter used to save computing time) - merge_file (optional) : if True will load all fits.fz files of the run, else merge_file can be integer to select the run fits.fz file according to this number - + run_file (optional) : if provided, will load this run file """ + super().__init__(*args,**kwargs) self.__run_number = run_number self.__run_file = run_file self.__max_events = max_events - self.__reader = WaveformsContainer.load_run(run_number,max_events,run_file = run_file) + self.__reader = WaveformsMaker.load_run(run_number,max_events,run_file = run_file) #from reader members self.__npixels = self.__reader.camera_config.num_pixels self.__nsamples = self.__reader.camera_config.num_samples - self.__geometry = self.__reader.subarray.tel[WaveformsContainer.TEL_ID].camera + self.__geometry = self.__reader.subarray.tel[WaveformsMaker.TEL_ID].camera self.__subarray = self.__reader.subarray self.__pixels_id = self.__reader.camera_config.expected_pixels_id - #set camera properties - log.info(f"N pixels : {self.npixels}") - - #run properties - if nevents != -1 : - self.__nevents = nevents if max_events is None else min(max_events,nevents) #self.check_events() - self.__reader = None - else : - self.__nevents = self.check_events() - - log.info(f"N_events : {self.nevents}") - if init_arrays : - self.__init_arrays() - - def __init_arrays(self,**kwargs) : - log.debug('creation of the EventSource reader') - self.__reader = WaveformsContainer.load_run(self.__run_number,self.__max_events,run_file = self.__run_file) + log.info(f"N pixels : {self.npixels}") - log.debug("create wfs, ucts, event properties and triger pattern arrays") - #define zeros members which will be filled therafter - self.wfs_hg = np.zeros((self.nevents,self.npixels,self.nsamples),dtype = np.uint16) - self.wfs_lg = np.zeros((self.nevents,self.npixels,self.nsamples),dtype = np.uint16) - self.ucts_timestamp = np.zeros((self.nevents),dtype = np.uint64) - self.ucts_busy_counter = np.zeros((self.nevents),dtype = np.uint32) - self.ucts_event_counter = np.zeros((self.nevents),dtype = np.uint32) - self.event_type = np.zeros((self.nevents),dtype = np.uint8) - self.event_id = np.zeros((self.nevents),dtype = np.uint32) - self.trig_pattern_all = np.zeros((self.nevents,self.CAMERA.n_pixels,4),dtype = bool) - #self.trig_pattern = np.zeros((self.nevents,self.npixels),dtype = bool) - #self.multiplicity = np.zeros((self.nevents,self.npixels),dtype = np.uint16) - - self.__broken_pixels_hg = np.zeros((self.npixels),dtype = bool) - self.__broken_pixels_lg = np.zeros((self.npixels),dtype = bool) - - - def __compute_broken_pixels(self,**kwargs) : - log.warning("computation of broken pixels is not yet implemented") - self.__broken_pixels_hg = np.zeros((self.npixels),dtype = bool) - self.__broken_pixels_lg = np.zeros((self.npixels),dtype = bool) + #data we want to compute + self.__wfs_hg = {} + self.__wfs_lg = {} + self.__ucts_timestamp = {} + self.__ucts_busy_counter = {} + self.__ucts_event_counter = {} + self.__event_type = {} + self.__event_id = {} + self.__trig_patter_all = {} + self.__broken_pixels_hg = {} + self.__broken_pixels_lg = {} @staticmethod def create_from_events_list(events_list : list, @@ -111,7 +90,7 @@ def create_from_events_list(events_list : list, subarray, pixels_id : int, ) : - cls = WaveformsContainer.__new__(WaveformsContainer) + cls = __class__.__new__() cls.__run_number = run_number cls.__nevents = len(events_list) @@ -120,232 +99,171 @@ def create_from_events_list(events_list : list, cls.__subarray = subarray cls.__pixels_id = pixels_id - - wfs_hg_tmp=np.zeros((npixels,nsamples),dtype = np.uint16) - wfs_lg_tmp=np.zeros((npixels,nsamples),dtype = np.uint16) + cls.__wfs_hg = {} + cls.__wfs_lg = {} + cls.__ucts_timestamp = {} + cls.__ucts_busy_counter = {} + cls.__ucts_event_counter = {} + cls.__event_type = {} + cls.__event_id = {} + cls.__trig_patter_all = {} + cls.__broken_pixels_hg = {} + cls.__broken_pixels_lg = {} - for i, event in enumerate(events_list): - cls.fill_wfs_from_event(event,i,wfs_hg_tmp,wfs_lg_tmp) - - cls.__compute_broken_pixels() + cls.__init_trigger_type(None) + for event in tqdm(events_list): + cls._make_event(event,None) + return cls.__make_output_container([None]) - @staticmethod - def load_run(run_number : int,max_events : int = None, run_file = None) : - """Static method to load from $NECTARCAMDATA directory data for specified run with max_events - - Args:self.__run_number = run_number - run_number (int): run_id - maxevents (int, optional): max of events to be loaded. Defaults to 0, to load everythings. - merge_file (optional) : if True will load all fits.fz files of the run, else merge_file can be integer to select the run fits.fz file according to this number - Returns: - List[ctapipe_io_nectarcam.NectarCAMEventSource]: List of EventSource for each run files - """ - if run_file is None : - generic_filename,_ = DataManagement.findrun(run_number) - log.info(f"{str(generic_filename)} will be loaded") - eventsource = NectarCAMEventSource(input_url=generic_filename,max_events=max_events) - else : - log.info(f"{run_file} will be loaded") - eventsource = NectarCAMEventSource(input_url=run_file,max_events=max_events) - return eventsource - - def check_events(self): - """method to check triggertype of events and counting number of events in all readers - it prints the trigger type when trigger type change over events (to not write one line by events) - - Returns: - int : number of events - """ - log.info("checking trigger type") - has_changed = True - previous_trigger = None - n_events = 0 - for i,event in enumerate(self.__reader): - log.debug(f"events i has ID : {np.uint16(event.index.event_id)}") - if previous_trigger is not None : - has_changed = (previous_trigger.value != event.trigger.event_type.value) - previous_trigger = event.trigger.event_type - if has_changed : - log.info(f"event {i} is {previous_trigger} : {previous_trigger.value} trigger type") - n_events+=1 - return n_events + + def __init_trigger_type(self,trigger_type,**kwargs) : + name = WaveformsMaker.__get_name_trigger(trigger_type) + log.info(f"initiamization of the waveformsMaker following trigger type : {name}") + self.__wfs_hg[f"{name}"] = [] + self.__wfs_lg[f"{name}"] = [] + self.__ucts_timestamp[f"{name}"] = [] + self.__ucts_busy_counter[f"{name}"] = [] + self.__ucts_event_counter[f"{name}"] = [] + self.__event_type[f"{name}"] = [] + self.__event_id[f"{name}"] = [] + self.__trig_patter_all[f"{name}"] = [] + self.__broken_pixels_hg[f"{name}"] = [] + self.__broken_pixels_lg[f"{name}"] = [] + + def __compute_broken_pixels(self,wfs_hg_event,wfs_lg_event,**kwargs) : + log.warning("computation of broken pixels is not yet implemented") + return np.zeros((self.npixels),dtype = bool),np.zeros((self.npixels),dtype = bool) + + @staticmethod + def __get_name_trigger(trigger : EventType) : + if trigger is None : + name = "None" + else : + name = trigger.name + return name - def load_wfs(self,compute_trigger_patern = False): + def make(self,n_events = np.inf, trigger_type : list = None, restart_from_begining = False): """mathod to extract waveforms data from the EventSource Args: + trigger_type (list[EventType], optional): only events with the asked trigger type will be use. Defaults to None. compute_trigger_patern (bool, optional): To recompute on our side the trigger patern. Defaults to False. """ - if not(hasattr(self, "wfs_hg")) : - self.__init_arrays() - - wfs_hg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) - wfs_lg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) + if ~np.isfinite(n_events) : + log.warning('no needed events number specified, it may cause a memory error') + if isinstance(trigger_type,EventType) or trigger_type is None : + trigger_type = [trigger_type] + + if restart_from_begining : + log.debug('restart from begining : creation of the EventSource reader') + self.__reader = WaveformsMaker.load_run(self.__run_number,self.__max_events,run_file = self.__run_file) + + for _trigger_type in trigger_type : + self.__init_trigger_type(_trigger_type) n_traited_events = 0 for i,event in enumerate(self.__reader): if i%100 == 0: log.info(f"reading event number {i}") - self.fill_wfs_from_event(event,i,wfs_hg_tmp,wfs_lg_tmp) - - self.__compute_broken_pixels() - - #if compute_trigger_patern and np.max(self.trig_pattern) == 0: - # self.compute_trigger_patern() - - def fill_wfs_from_event(self, - event, - i : int, - wfs_hg_tmp : np.ndarray, - wfs_lg_tmp : np.ndarray - ) : - self.event_id[i] = np.uint16(event.index.event_id) - self.ucts_timestamp[i]=event.nectarcam.tel[WaveformsContainer.TEL_ID].evt.ucts_timestamp - self.event_type[i]=event.trigger.event_type.value - self.ucts_busy_counter[i] = event.nectarcam.tel[WaveformsContainer.TEL_ID].evt.ucts_busy_counter - self.ucts_event_counter[i] = event.nectarcam.tel[WaveformsContainer.TEL_ID].evt.ucts_event_counter - self.trig_pattern_all[i] = event.nectarcam.tel[WaveformsContainer.TEL_ID].evt.trigger_pattern.T + for trigger in trigger_type : + if (trigger is None) or (trigger == event.trigger.event_type) : + self._make_event(event,trigger) + n_traited_events += 1 + if n_traited_events >= n_events : + break + + return self.__make_output_container(trigger_type) + + def _make_event(self, + event, + trigger : EventType + ) : + name = WaveformsMaker.__get_name_trigger(trigger) + self.__event_id[f'{name}'].append(np.uint16(event.index.event_id)) + self.__ucts_timestamp[f'{name}'].append(event.nectarcam.tel[WaveformsMaker.TEL_ID].evt.ucts_timestamp) + self.__event_type[f'{name}'].append(event.trigger.event_type.value) + self.__ucts_busy_counter[f'{name}'].append(event.nectarcam.tel[WaveformsMaker.TEL_ID].evt.ucts_busy_counter) + self.__ucts_event_counter[f'{name}'].append(event.nectarcam.tel[WaveformsMaker.TEL_ID].evt.ucts_event_counter) + self.__trig_patter_all[f'{name}'].append(event.nectarcam.tel[WaveformsMaker.TEL_ID].evt.trigger_pattern.T) + + wfs_hg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) + wfs_lg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) for pix in range(self.npixels): wfs_lg_tmp[pix]=event.r0.tel[0].waveform[1,self.pixels_id[pix]] wfs_hg_tmp[pix]=event.r0.tel[0].waveform[0,self.pixels_id[pix]] - self.wfs_hg[i] = wfs_hg_tmp - self.wfs_lg[i] = wfs_lg_tmp - - def write(self,path : str, **kwargs) : - """method to write in an output FITS file the WaveformsContainer. Two files are created, one FITS representing the data - and one HDF5 file representing the subarray configuration - - Args: - path (str): the directory where you want to save data - """ - suffix = kwargs.get("suffix","") - if suffix != "" : suffix = f"_{suffix}" - - log.info(f"saving in {path}") - os.makedirs(path,exist_ok = True) - - hdr = fits.Header() - hdr['RUN'] = self.__run_number - hdr['NEVENTS'] = self.nevents - hdr['NPIXELS'] = self.npixels - hdr['NSAMPLES'] = self.nsamples - hdr['SUBARRAY'] = self.subarray.name - - self.subarray.to_hdf(f"{Path(path)}/subarray_run{self.run_number}{suffix}.hdf5",overwrite=kwargs.get('overwrite',False)) - - - - hdr['COMMENT'] = f"The waveforms containeur for run {self.__run_number} : primary is the pixels id, 2nd HDU : high gain waveforms, 3rd HDU : low gain waveforms, 4th HDU : event properties and 5th HDU trigger paterns." - - primary_hdu = fits.PrimaryHDU(self.pixels_id,header=hdr) - - wfs_hg_hdu = fits.ImageHDU(self.wfs_hg,name = "HG Waveforms") - wfs_lg_hdu = fits.ImageHDU(self.wfs_lg,name = "LG Waveforms") - - - col1 = fits.Column(array = self.event_id, name = "event_id", format = '1J') - col2 = fits.Column(array = self.event_type, name = "event_type", format = '1I') - col3 = fits.Column(array = self.ucts_timestamp, name = "ucts_timestamp", format = '1K') - col4 = fits.Column(array = self.ucts_busy_counter, name = "ucts_busy_counter", format = '1J') - col5 = fits.Column(array = self.ucts_event_counter, name = "ucts_event_counter", format = '1J') - col6 = fits.Column(array = self.multiplicity, name = "multiplicity", format = '1I') - - coldefs = fits.ColDefs([col1, col2, col3, col4, col5, col6]) - event_properties = fits.BinTableHDU.from_columns(coldefs,name = 'event properties') - - col1 = fits.Column(array = self.trig_pattern_all, name = "trig_pattern_all", format = f'{4 * self.CAMERA.n_pixels}L',dim = f'({self.CAMERA.n_pixels},4)') - col2 = fits.Column(array = self.trig_pattern, name = "trig_pattern", format = f'{self.CAMERA.n_pixels}L') - coldefs = fits.ColDefs([col1, col2]) - trigger_patern = fits.BinTableHDU.from_columns(coldefs,name = 'trigger patern') - - hdul = fits.HDUList([primary_hdu, wfs_hg_hdu, wfs_lg_hdu,event_properties,trigger_patern]) - try : - hdul.writeto(Path(path)/f"waveforms_run{self.run_number}{suffix}.fits",overwrite=kwargs.get('overwrite',False)) - log.info(f"run saved in {Path(path)}/waveforms_run{self.run_number}{suffix}.fits") - except OSError as e : - log.warning(e) - except Exception as e : - log.error(e,exc_info = True) - raise e - - + self.__wfs_hg[f'{name}'].append(wfs_hg_tmp.tolist()) + self.__wfs_lg[f'{name}'].append(wfs_lg_tmp.tolist()) + + broken_pixels_hg,broken_pixels_lg = self.__compute_broken_pixels(wfs_hg_tmp,wfs_lg_tmp) + self.__broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) + self.__broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) + + def __make_output_container(self,trigger_type) : + output = [] + for trigger in trigger_type : + waveformsContainer = WaveformsContainer( + run_number = self.run_number, + npixels = self.npixels, + nsamples = self.nsamples, + subarray = self.subarray, + camera = self.CAMERA_NAME, + pixels_id = self.pixels_id, + nevents = self.nevents(trigger), + wfs_hg = self.wfs_hg(trigger), + wfs_lg = self.wfs_lg(trigger), + broken_pixels_hg = self.broken_pixels_hg(trigger), + broken_pixels_lg = self.broken_pixels_lg(trigger), + ucts_timestamp = self.ucts_timestamp(trigger), + ucts_busy_counter = self.ucts_busy_counter(trigger), + ucts_event_counter = self.ucts_event_counter(trigger), + event_type = self.event_type(trigger), + event_id = self.event_id(trigger), + trig_pattern_all = self.trig_pattern_all(trigger), + trig_pattern = self.trig_pattern(trigger), + multiplicity = self.multiplicity(trigger) + ) + output.append(waveformsContainer) + return output @staticmethod - def load(path : str) : - """load WaveformsContainer from FITS file previously written with WaveformsContainer.write() method - Note : 2 files are loaded, the FITS one representing the waveforms data and a HDF5 file representing the subarray configuration. - This second file has to be next to the FITS file. - - Args: - path (str): path of the FITS file - - Returns: - WaveformsContainer: WaveformsContainer instance - """ - log.info(f"loading from {path}") - with fits.open(Path(path)) as hdul : - cls = WaveformsContainer.__new__(WaveformsContainer) - - cls.__run_number = hdul[0].header['RUN'] - cls.__nevents = hdul[0].header['NEVENTS'] - cls.__npixels = hdul[0].header['NPIXELS'] - cls.__nsamples = hdul[0].header['NSAMPLES'] - - cls.__subarray = SubarrayDescription.from_hdf(Path(path.replace('waveforms_','subarray_').replace('fits','hdf5'))) - - - cls.__pixels_id = hdul[0].data - cls.wfs_hg = hdul[1].data - cls.wfs_lg = hdul[2].data - - table_prop = hdul[3].data - cls.event_id = table_prop["event_id"] - cls.event_type = table_prop["event_type"] - cls.ucts_timestamp = table_prop["ucts_timestamp"] - cls.ucts_busy_counter = table_prop["ucts_busy_counter"] - cls.ucts_event_counter = table_prop["ucts_event_counter"] - - table_trigger = hdul[4].data - cls.trig_pattern_all = table_trigger["trig_pattern_all"] - - cls.__compute_broken_pixels() - - return cls - - - - - def sort(self, method = 'event_id') : + def sort(waveformsContainer, method = 'event_id') : + output = WaveformsContainer( + run_number = waveformsContainer.run_number, + npixels = waveformsContainer.npixels, + nsamples = waveformsContainer.nsamples, + subarray = waveformsContainer.subarray, + camera = waveformsContainer.camera, + pixels_id = waveformsContainer.pixels_id, + nevents = waveformsContainer.nevents + ) if method == 'event_id' : - index = np.argsort(self.event_id) - self.ucts_timestamp = self.ucts_timestamp[index] - self.ucts_busy_counter = self.ucts_busy_counter[index] - self.ucts_event_counter = self.ucts_event_counter[index] - self.event_type = self.event_type[index] - self.event_id = self.event_id[index] - self.trig_pattern_all = self.trig_pattern_all[index] - self.wfs_hg = self.wfs_hg[index] - self.wfs_lg = self.wfs_lg[index] + index = np.argsort(waveformsContainer.event_id) + output.ucts_timestamp = waveformsContainer.ucts_timestamp[index] + output.ucts_busy_counter = waveformsContainer.ucts_busy_counter[index] + output.ucts_event_counter = waveformsContainer.ucts_event_counter[index] + output.event_type = waveformsContainer.event_type[index] + output.event_id = waveformsContainer.event_id[index] + output.trig_pattern_all = waveformsContainer.trig_pattern_all[index] + output.trig_pattern = waveformsContainer.trig_pattern[index] + output.multiplicity = waveformsContainer.multiplicity[index] + + output.wfs_hg = waveformsContainer.wfs_hg[index] + output.wfs_lg = waveformsContainer.wfs_lg[index] + output.broken_pixels_hg = waveformsContainer.broken_pixels_hg[index] + output.broken_pixels_lg = waveformsContainer.broken_pixels_lg[index] else : raise ArgumentError(f"{method} is not a valid method for sorting") - - def compute_trigger_patern(self) : - """(preliminary) function to compute the trigger patern - """ - #mean.shape nevents * npixels - mean,std =np.mean(self.wfs_hg,axis = 2),np.std(self.wfs_hg,axis = 2) - self.trig_pattern = self.wfs_hg.max(axis = 2) > (mean + 3 * std) - - - + return output + + @staticmethod ##methods used to display - def display(self,evt,cmap = 'gnuplot2') : + def display(waveformsContainer,evt,geometry, cmap = 'gnuplot2') : """plot camera display Args: @@ -355,12 +273,13 @@ def display(self,evt,cmap = 'gnuplot2') : Returns: CameraDisplay: thoe cameraDisplay plot """ - image = self.wfs_hg.sum(axis=2) - disp = CameraDisplay(geometry=WaveformsContainer.CAMERA, image=image[evt], cmap=cmap) + image = waveformsContainer.wfs_hg.sum(axis=2) + disp = CameraDisplay(geometry=geometry, image=image[evt], cmap=cmap) disp.add_colorbar() return disp - def plot_waveform_hg(self,evt,**kwargs) : + @staticmethod + def plot_waveform_hg(waveformsContainer,evt,**kwargs) : """plot the waveform of the evt Args: @@ -374,11 +293,11 @@ def plot_waveform_hg(self,evt,**kwargs) : ax = kwargs.get('ax') else : fig,ax = plt.subplots(1,1) - ax.plot(self.wfs_hg[evt].T) + ax.plot(waveformsContainer.wfs_hg[evt].T) return fig,ax - - def select_waveforms_hg(self,pixel_id : np.ndarray) : + @staticmethod + def select_waveforms_hg(waveformsContainer,pixel_id : np.ndarray) : """method to extract waveforms HG from a list of pixel ids The output is the waveforms HG with a shape following the size of the input pixel_id argument Pixel in pixel_id which are not present in the WaveformsContaineur pixels_id are skipped @@ -387,14 +306,15 @@ def select_waveforms_hg(self,pixel_id : np.ndarray) : Returns: (np.ndarray): waveforms array in the order of specified pixel_id """ - mask_contain_pixels_id = np.array([pixel in self.pixels_id for pixel in pixel_id],dtype = bool) + mask_contain_pixels_id = np.array([pixel in waveformsContainer.pixels_id for pixel in pixel_id],dtype = bool) for pixel in pixel_id[~mask_contain_pixels_id] : log.warning(f"You asked for pixel_id {pixel} but it is not present in this WaveformsContainer, skip this one") - res = np.array([self.wfs_hg[:,np.where(self.pixels_id == pixel)[0][0],:] for pixel in pixel_id[mask_contain_pixels_id]]) - res = res.transpose(res.shape[1],res.shape[0],res.shape[2]) + res = np.array([waveformsContainer.wfs_hg[:,np.where(waveformsContainer.pixels_id == pixel)[0][0],:] for pixel in pixel_id[mask_contain_pixels_id]]) + res = res.transpose(1,0,2) + ####could be nice to return np.ma.masked_array(data = res, mask = waveformsContainer.broken_pixels_hg.transpose(res.shape[1],res.shape[0],res.shape[2])) return res - - def select_waveforms_lg(self,pixel_id : np.ndarray) : + @staticmethod + def select_waveforms_lg(waveformsContainer,pixel_id : np.ndarray) : """method to extract waveforms LG from a list of pixel ids The output is the waveforms LG with a shape following the size of the input pixel_id argument Pixel in pixel_id which are not present in the WaveformsContaineur pixels_id are skipped @@ -405,183 +325,52 @@ def select_waveforms_lg(self,pixel_id : np.ndarray) : Returns: (np.ndarray): waveforms array in the order of specified pixel_id """ - mask_contain_pixels_id = np.array([pixel in self.pixels_id for pixel in pixel_id],dtype = bool) + mask_contain_pixels_id = np.array([pixel in waveformsContainer.pixels_id for pixel in pixel_id],dtype = bool) for pixel in pixel_id[~mask_contain_pixels_id] : log.warning(f"You asked for pixel_id {pixel} but it is not present in this WaveformsContainer, skip this one") - res = np.array([self.wfs_lg[:,np.where(self.pixels_id == pixel)[0][0],:] for pixel in pixel_id[mask_contain_pixels_id]]) - res = res.transpose(res.shape[1],res.shape[0],res.shape[2]) + res = np.array([waveformsContainer.wfs_lg[:,np.where(waveformsContainer.pixels_id == pixel)[0][0],:] for pixel in pixel_id[mask_contain_pixels_id]]) + res = res.transpose(1,0,2) return res @property - def _run_file(self) : return self.__run_file - + def _run_file(self) : return self.__run_file @property def _max_events(self) : return self.__max_events - @property def reader(self) : return self.__reader - @property def npixels(self) : return self.__npixels - @property def nsamples(self) : return self.__nsamples - @property def geometry(self) : return self.__geometry - @property def subarray(self) : return self.__subarray - @property def pixels_id(self) : return self.__pixels_id - - @property - def nevents(self) : return self.__nevents - @property def run_number(self) : return self.__run_number - - @property - def broken_pixels_hg(self) : return self.__broken_pixels_hg - - @property - def broken_pixels_lg(self) : return self.__broken_pixels_lg - - #physical properties - @property - def multiplicity(self) : return np.uint16(np.count_nonzero(self.trig_pattern,axis = 1)) - - @property - def trig_pattern(self) : return self.trig_pattern_all.any(axis = 1) - - - - - - - -class WaveformsContainers() : - """This class is to be used for computing waveforms of a run treating run files one by one - """ - def __new__(cls,*args,**kwargs) : - """base class constructor - - Returns: - WaveformsContainers : the new object - """ - obj = object.__new__(cls) - return obj - - def __init__(self,run_number : int,max_events : int = None, init_arrays : bool = False) : - """initialize the waveformsContainer list inside the main object - - Args: - run_number (int): the run number - max_events (int, optional): max events we want to read. Defaults to None. - """ - log.info('Initialization of WaveformsContainers') - _,filenames = DataManagement.findrun(run_number) - self.waveformsContainer = [] - self.__nWaveformsContainer = 0 - for i,file in enumerate(filenames) : - self.waveformsContainer.append(WaveformsContainer(run_number,max_events=max_events,run_file=file, init_arrays= init_arrays)) - self.__nWaveformsContainer += 1 - if not(max_events is None) : max_events -= self.waveformsContainer[i].nevents - log.info(f'WaveformsContainer number {i} is created') - if not(max_events is None) and max_events <= 0 : break - - def load_wfs(self,compute_trigger_patern = False,index : int = None) : - """load waveforms from the Eventsource reader - - Args: - compute_trigger_patern (bool, optional): to compute manually the trigger patern. Defaults to False. - index (int, optional): to only load waveforms from EventSource for the waveformsContainer at index, this parameters - can be used to load, write or perform some work step by step. Thus all the event are not nessessary loaded at the same time - In such case memory can be saved. Defaults to None. - - Raises: - IndexError: the index is > nWaveformsContainer or < 0 - """ - if index is None : - for i in range(self.__nWaveformsContainer) : - self.waveformsContainer[i].load_wfs(compute_trigger_patern = compute_trigger_patern) + def nevents(self,trigger) : return len(self.__event_id[WaveformsMaker.__get_name_trigger(trigger)]) + def wfs_hg(self,trigger) : return np.array(self.__wfs_hg[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint16) + def wfs_lg(self,trigger) : return np.array(self.__wfs_lg[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint16) + def broken_pixels_hg(self,trigger) : return np.array(self.__broken_pixels_hg[WaveformsMaker.__get_name_trigger(trigger)],dtype = bool) + def broken_pixels_lg(self,trigger) : return np.array(self.__broken_pixels_lg[WaveformsMaker.__get_name_trigger(trigger)],dtype = bool) + def ucts_timestamp(self,trigger) : return np.array(self.__ucts_timestamp[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint64) + def ucts_busy_counter(self,trigger) : return np.array(self.__ucts_busy_counter[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint32) + def ucts_event_counter(self,trigger) : return np.array(self.__ucts_event_counter[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint32) + def event_type(self,trigger) : return np.array(self.__event_type[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint8) + def event_id(self,trigger) : return np.array(self.__event_id[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint32) + def multiplicity(self,trigger) : + tmp = self.trig_pattern(trigger) + if len(tmp) == 0 : + return np.array([]) else : - if index < self.__nWaveformsContainer and index >= 0 : - self.waveformsContainer[index].load_wfs(compute_trigger_patern = compute_trigger_patern) - else : - raise IndexError(f"index must be >= 0 and < {self.__nWaveformsContainer}") - - - def write(self,path : str, index : int = None, **kwargs) : - """method to write on disk all the waveformsContainer in the list - - Args: - path (str): path where to save the waveforms - index (int, optional): index of the waveforms list you want to write. Defaults to None. - - Raises: - IndexError: the index is > nWaveformsContainer or < 0 - """ - if index is None : - for i in range(self.__nWaveformsContainer) : - self.waveformsContainer[i].write(path,suffix = f"{i:04d}" ,**kwargs) + return np.uint16(np.count_nonzero(tmp,axis = 1)) + def trig_pattern(self,trigger) : + tmp = self.trig_pattern_all(trigger) + if len(tmp) == 0 : + return np.array([]) else : - if index < self.__nWaveformsContainer and index >= 0 : - self.waveformsContainer[index].write(path,suffix = f"{index:04d}" ,**kwargs) - else : - raise IndexError(f"index must be >= 0 and < {self.__nWaveformsContainer}") - - @staticmethod - def load(path : str) : - """method to load the waveforms list from splited fits files - - Args: - path (str): path where data should be, it contain filename without extension - - Raises: - e: File not found + return tmp.any(axis = 2) + def trig_pattern_all(self,trigger) : return np.array(self.__trig_patter_all[f"{WaveformsMaker.__get_name_trigger(trigger)}"],dtype = bool) - Returns: - WaveformsContainers: - """ - log.info(f"loading from {path}") - files = glob.glob(f"{path}_*.fits") - if len(files) == 0 : - e = FileNotFoundError(f"no files found corresponding to {path}_*.fits") - log.error(e) - raise e - else : - cls = WaveformsContainers.__new__(WaveformsContainers) - cls.waveformsContainer = [] - cls.__nWaveformsContainer = len(files) - for file in files : - cls.waveformsContainer.append(WaveformsContainer.load(file)) - return cls - - - - - - @property - def nWaveformsContainer(self) : - """getter giving the number of waveformsContainer into the waveformsContainers instance - - Returns: - int: the number of waveformsContainer - """ - return self.__nWaveformsContainer - - @property - def nevents(self) : - """getter giving the number of events in the waveformsContainers instance""" - return np.sum([self.waveformsContainer[i].nevents for i in range(self.__nWaveformsContainer)]) - - def append(self,waveformsContainer : WaveformsContainer) : - """method to append WaveformsContainer to the waveforms list - - Args: - waveformsContainer (WaveformsContainer): the waveformsContainer to stack - """ - self.waveformsContainer.append(waveformsContainer) - self.__nWaveformsContainer += 1 From 205ac1f093ae6fe6bd40548f4af357d91950066c Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 14 Sep 2023 04:47:13 +0200 Subject: [PATCH 44/62] move charge container into makers module --- .../{data/container/charge.py => makers/chargesMakers.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/nectarchain/{data/container/charge.py => makers/chargesMakers.py} (100%) diff --git a/src/nectarchain/data/container/charge.py b/src/nectarchain/makers/chargesMakers.py similarity index 100% rename from src/nectarchain/data/container/charge.py rename to src/nectarchain/makers/chargesMakers.py From a43bdab4f722f1cfce416221943197e9b492a5da Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 14 Sep 2023 05:19:00 +0200 Subject: [PATCH 45/62] creation of mother class to charge and waveform container --- src/nectarchain/data/container/__init__.py | 2 +- .../data/container/chargesContainer.py | 139 ++++++++++++++++++ src/nectarchain/data/container/core.py | 93 ++++++++++++ .../data/container/waveformsContainer.py | 86 +---------- 4 files changed, 239 insertions(+), 81 deletions(-) create mode 100644 src/nectarchain/data/container/chargesContainer.py create mode 100644 src/nectarchain/data/container/core.py diff --git a/src/nectarchain/data/container/__init__.py b/src/nectarchain/data/container/__init__.py index 2bb52a30..905aff5a 100644 --- a/src/nectarchain/data/container/__init__.py +++ b/src/nectarchain/data/container/__init__.py @@ -1,2 +1,2 @@ from .waveformsContainer import * -from .charge import * \ No newline at end of file +from .chargesContainer import * \ No newline at end of file diff --git a/src/nectarchain/data/container/chargesContainer.py b/src/nectarchain/data/container/chargesContainer.py new file mode 100644 index 00000000..eadce599 --- /dev/null +++ b/src/nectarchain/data/container/chargesContainer.py @@ -0,0 +1,139 @@ +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +import numpy as np +from ctapipe.containers import Field + +from .core import ArrayDataContainer + +class ChargesContainer(ArrayDataContainer): + + charge_hg = Field( + type = np.ndarray, + description = 'The high gain charge') + charge_lg = Field( + type = np.ndarray, + description = 'The low gain charge') + peak_hg = Field( + type = np.ndarray, + description = 'The high gain peak time') + peak_lg = Field( + type = np.ndarray, + description = 'The low gain peak time') + _method = Field( + type = str, + description = 'The charge extraction method') + + +''' + + def write(self,path : Path,**kwargs) : + """method to write in an output FITS file the ChargeContainer. + + Args: + path (str): the directory where you want to save data + """ + suffix = kwargs.get("suffix","") + if suffix != "" : suffix = f"_{suffix}" + + log.info(f"saving in {path}") + os.makedirs(path,exist_ok = True) + + #table = Table(self.charge_hg) + #table.meta["pixels_id"] = self._pixels_id + #table.write(Path(path)/f"charge_hg_run{self.run_number}.ecsv",format='ascii.ecsv',overwrite=kwargs.get('overwrite',False)) + # + #table = Table(self.charge_lg) + #table.meta["pixels_id"] = self._pixels_id + #table.write(Path(path)/f"charge_lg_run{self.run_number}.ecsv",format='ascii.ecsv',overwrite=kwargs.get('overwrite',False)) + # + #table = Table(self.peak_hg) + #table.meta["pixels_id"] = self._pixels_id + #table.write(Path(path)/f"peak_hg_run{self.run_number}.ecsv",format='ascii.ecsv',overwrite=kwargs.get('overwrite',False)) + # + #table = Table(self.peak_lg) + #table.meta["pixels_id"] = self._pixels_id + #table.write(Path(path)/f"peak_lg_run{self.run_number}.ecsv",format='ascii.ecsv',overwrite=kwargs.get('overwrite',False)) + + hdr = fits.Header() + hdr['RUN'] = self._run_number + hdr['NEVENTS'] = self.nevents + hdr['NPIXELS'] = self.npixels + hdr['COMMENT'] = f"The charge containeur for run {self._run_number} with {self._method} method : primary is the pixels id, then you can find HG charge, LG charge, HG peak and LG peak, 2 last HDU are composed of event properties and trigger patern" + + primary_hdu = fits.PrimaryHDU(self.pixels_id,header=hdr) + charge_hg_hdu = fits.ImageHDU(self.charge_hg,name = "HG charge") + charge_lg_hdu = fits.ImageHDU(self.charge_lg,name = "LG charge") + peak_hg_hdu = fits.ImageHDU(self.peak_hg, name = 'HG peak time') + peak_lg_hdu = fits.ImageHDU(self.peak_lg, name = 'LG peak time') + + col1 = fits.Column(array = self.event_id, name = "event_id", format = '1I') + col2 = fits.Column(array = self.event_type, name = "event_type", format = '1I') + col3 = fits.Column(array = self.ucts_timestamp, name = "ucts_timestamp", format = '1K') + col4 = fits.Column(array = self.ucts_busy_counter, name = "ucts_busy_counter", format = '1I') + col5 = fits.Column(array = self.ucts_event_counter, name = "ucts_event_counter", format = '1I') + col6 = fits.Column(array = self.multiplicity, name = "multiplicity", format = '1I') + + coldefs = fits.ColDefs([col1, col2, col3, col4, col5, col6]) + event_properties = fits.BinTableHDU.from_columns(coldefs, name = 'event properties') + + col1 = fits.Column(array = self.trig_pattern_all, name = "trig_pattern_all", format = f'{4 * self.CAMERA.n_pixels}L',dim = f'({self.CAMERA.n_pixels},4)') + col2 = fits.Column(array = self.trig_pattern, name = "trig_pattern", format = f'{self.CAMERA.n_pixels}L') + coldefs = fits.ColDefs([col1, col2]) + trigger_patern = fits.BinTableHDU.from_columns(coldefs, name = 'trigger patern') + + hdul = fits.HDUList([primary_hdu, charge_hg_hdu, charge_lg_hdu,peak_hg_hdu,peak_lg_hdu,event_properties,trigger_patern]) + try : + hdul.writeto(Path(path)/f"charge_run{self.run_number}{suffix}.fits",overwrite=kwargs.get('overwrite',False)) + log.info(f"charge saved in {Path(path)}/charge_run{self.run_number}{suffix}.fits") + except OSError as e : + log.warning(e) + except Exception as e : + log.error(e,exc_info = True) + raise e + + + + + @staticmethod + def from_file(path : Path,run_number : int,**kwargs) : + """load ChargeContainer from FITS file previously written with ChargeContainer.write() method + + Args: + path (str): path of the FITS file + run_number (int) : the run number + + Returns: + ChargeContainer: ChargeContainer instance + """ + if kwargs.get("explicit_filename",False) : + filename = kwargs.get("explicit_filename") + log.info(f"loading {filename}") + else : + log.info(f"loading in {path} run number {run_number}") + filename = Path(path)/f"charge_run{run_number}.fits" + + with fits.open(filename) as hdul : + pixels_id = hdul[0].data + nevents = hdul[0].header['NEVENTS'] + npixels = hdul[0].header['NPIXELS'] + charge_hg = hdul[1].data + charge_lg = hdul[2].data + peak_hg = hdul[3].data + peak_lg = hdul[4].data + + cls = ChargeContainer(charge_hg,charge_lg,peak_hg,peak_lg,run_number,pixels_id,nevents,npixels) + + cls.event_id = hdul[5].data["event_id"] + cls.event_type = hdul[5].data["event_type"] + cls.ucts_timestamp = hdul[5].data["ucts_timestamp"] + cls.ucts_busy_counter = hdul[5].data["ucts_busy_counter"] + cls.ucts_event_counter = hdul[5].data["ucts_event_counter"] + + table_trigger = hdul[6].data + cls.trig_pattern_all = table_trigger["trig_pattern_all"] + + return cls +''' \ No newline at end of file diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py new file mode 100644 index 00000000..fa9b00c0 --- /dev/null +++ b/src/nectarchain/data/container/core.py @@ -0,0 +1,93 @@ +import sys +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +import os +from ctapipe.containers import Container,Field +import numpy as np + + +class ArrayDataContainer(Container): + """ + A container that holds information about waveforms from a specific run. + + Attributes: + run_number (int): The run number associated with the waveforms. + nevents (int): The number of events. + npixels (int): The number of pixels. + camera (str): The name of the camera. + pixels_id (np.ndarray): An array of pixel IDs. + broken_pixels_hg (np.ndarray): An array of high gain broken pixels. + broken_pixels_lg (np.ndarray): An array of low gain broken pixels. + ucts_timestamp (np.ndarray): An array of events' UCTS timestamps. + ucts_busy_counter (np.ndarray): An array of UCTS busy counters. + ucts_event_counter (np.ndarray): An array of UCTS event counters. + event_type (np.ndarray): An array of trigger event types. + event_id (np.ndarray): An array of event IDs. + trig_pattern_all (np.ndarray): An array of trigger patterns. + trig_pattern (np.ndarray): An array of reduced trigger patterns. + multiplicity (np.ndarray): An array of events' multiplicities. + """ + + run_number = Field( + type=int, + description="run number associated to the waveforms", + ) + nevents = Field( + type=int, + description="number of events", + ) + npixels = Field( + type=int, + description="number of effective pixels", + ) + pixels_id = Field( + type=np.ndarray, + description="pixel ids" + ) + broken_pixels_hg = Field( + type=np.ndarray, + description="high gain broken pixels" + ) + broken_pixels_lg = Field( + type=np.ndarray, + description="low gain broken pixels" + ) + camera = Field( + type=str, + description="camera name", + ) + ucts_timestamp = Field( + type=np.ndarray, + description="events ucts timestamp" + ) + ucts_busy_counter = Field( + type=np.ndarray, + description="ucts busy counter" + ) + ucts_event_counter = Field( + type=np.ndarray, + description="ucts event counter" + ) + event_type = Field( + type=np.ndarray, + description="trigger event type" + ) + event_id = Field( + type=np.ndarray, + description="event ids" + ) + trig_pattern_all = Field( + type=np.ndarray, + description="trigger pattern" + ) + trig_pattern = Field( + type=np.ndarray, + description="reduced trigger pattern" + ) + multiplicity = Field( + type=np.ndarray, + description="events multiplicity" + ) \ No newline at end of file diff --git a/src/nectarchain/data/container/waveformsContainer.py b/src/nectarchain/data/container/waveformsContainer.py index 9673f013..1a277b11 100644 --- a/src/nectarchain/data/container/waveformsContainer.py +++ b/src/nectarchain/data/container/waveformsContainer.py @@ -5,8 +5,7 @@ log.handlers = logging.getLogger('__main__').handlers import os -from ctapipe.containers import Container,Field -from ctapipe.io import TableWriter,TableLoader + from ctapipe.instrument.subarray import SubarrayDescription import numpy as np @@ -15,50 +14,25 @@ from enum import Enum from tqdm import tqdm - +from ctapipe.containers import Field from astropy.io import fits from astropy.table import QTable,Column,Table import astropy.units as u from abc import ABC -class WaveformsContainer(Container): +from .core import ArrayDataContainer + +class WaveformsContainer(ArrayDataContainer): """ A container that holds information about waveforms from a specific run. Attributes: - run_number (int): The run number associated with the waveforms. - nevents (int): The number of events. - npixels (int): The number of pixels. nsamples (int): The number of samples in the waveforms. - subarray (str): The name of the subarray. - camera (str): The name of the camera. - pixels_id (np.ndarray): An array of pixel IDs. + subarray (SubarrayDescription): The subarray description instance. wfs_hg (np.ndarray): An array of high gain waveforms. wfs_lg (np.ndarray): An array of low gain waveforms. - broken_pixels_hg (np.ndarray): An array of high gain broken pixels. - broken_pixels_lg (np.ndarray): An array of low gain broken pixels. - ucts_timestamp (np.ndarray): An array of events' UCTS timestamps. - ucts_busy_counter (np.ndarray): An array of UCTS busy counters. - ucts_event_counter (np.ndarray): An array of UCTS event counters. - event_type (np.ndarray): An array of trigger event types. - event_id (np.ndarray): An array of event IDs. - trig_pattern_all (np.ndarray): An array of trigger patterns. - trig_pattern (np.ndarray): An array of reduced trigger patterns. - multiplicity (np.ndarray): An array of events' multiplicities. """ - run_number = Field( - type=int, - description="run number associated to the waveforms", - ) - nevents = Field( - type=int, - description="number of events", - ) - npixels = Field( - type=int, - description="number of effective pixels", - ) nsamples = Field( type=int, description="number of samples in the waveforms", @@ -67,14 +41,6 @@ class WaveformsContainer(Container): type=SubarrayDescription, description="The subarray description" ) - camera = Field( - type=str, - description="camera name", - ) - pixels_id = Field( - type=np.ndarray, - description="pixel ids" - ) wfs_hg = Field( type=np.ndarray, description="high gain waveforms" @@ -83,46 +49,6 @@ class WaveformsContainer(Container): type=np.ndarray, description="low gain waveforms" ) - broken_pixels_hg = Field( - type=np.ndarray, - description="high gain broken pixels" - ) - broken_pixels_lg = Field( - type=np.ndarray, - description="low gain broken pixels" - ) - ucts_timestamp = Field( - type=np.ndarray, - description="events ucts timestamp" - ) - ucts_busy_counter = Field( - type=np.ndarray, - description="ucts busy counter" - ) - ucts_event_counter = Field( - type=np.ndarray, - description="ucts event counter" - ) - event_type = Field( - type=np.ndarray, - description="trigger event type" - ) - event_id = Field( - type=np.ndarray, - description="event ids" - ) - trig_pattern_all = Field( - type=np.ndarray, - description="trigger pattern" - ) - trig_pattern = Field( - type=np.ndarray, - description="reduced trigger pattern" - ) - multiplicity = Field( - type=np.ndarray, - description="events multiplicity" - ) class WaveformsContainerIO(ABC) : From b0ae914e4518e502a319d0fc458197112f5b69fa Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 14 Sep 2023 06:46:23 +0200 Subject: [PATCH 46/62] follow up : factorisation waveforms and chargs Makers --- src/nectarchain/makers/__init__.py | 3 +- src/nectarchain/makers/chargesMakers.py | 116 +---------- src/nectarchain/makers/core.py | 226 +++++++++++++++++++++- src/nectarchain/makers/waveformsMakers.py | 189 ++++-------------- 4 files changed, 267 insertions(+), 267 deletions(-) diff --git a/src/nectarchain/makers/__init__.py b/src/nectarchain/makers/__init__.py index a467a348..1cac4aa2 100644 --- a/src/nectarchain/makers/__init__.py +++ b/src/nectarchain/makers/__init__.py @@ -1,2 +1,3 @@ from .core import * -from .waveformsMakers import * \ No newline at end of file +from .waveformsMakers import * +from .chargesMakers import * diff --git a/src/nectarchain/makers/chargesMakers.py b/src/nectarchain/makers/chargesMakers.py index 1ee7604b..744f9e5e 100644 --- a/src/nectarchain/makers/chargesMakers.py +++ b/src/nectarchain/makers/chargesMakers.py @@ -19,7 +19,9 @@ from numba import guvectorize, float64, int64, bool_ -from .waveformsContainer import WaveformsContainer +from ..data.container import WaveformsContainer + +from .core import ArrayDataMaker @@ -91,7 +93,7 @@ def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): pass -class ChargeContainer() : +class ChargeMaker(ArrayDataMaker) : """ class used to compute charge from waveforms container @@ -147,8 +149,6 @@ class used to compute charge from waveforms container nevents = chargeContainer.nevents method = chargeContainer.method """ - TEL_ID = 0 - CAMERA = CameraGeometry.from_name("NectarCam-003") def __init__(self, charge_hg, charge_lg, peak_hg, peak_lg, run_number, pixels_id, nevents, npixels, method="FullWaveformSum"): """ @@ -212,114 +212,6 @@ def from_waveforms(cls, waveformContainer: WaveformsContainer, method: str = "Fu return chargeContainer - def write(self,path : Path,**kwargs) : - """method to write in an output FITS file the ChargeContainer. - - Args: - path (str): the directory where you want to save data - """ - suffix = kwargs.get("suffix","") - if suffix != "" : suffix = f"_{suffix}" - - log.info(f"saving in {path}") - os.makedirs(path,exist_ok = True) - - #table = Table(self.charge_hg) - #table.meta["pixels_id"] = self._pixels_id - #table.write(Path(path)/f"charge_hg_run{self.run_number}.ecsv",format='ascii.ecsv',overwrite=kwargs.get('overwrite',False)) - # - #table = Table(self.charge_lg) - #table.meta["pixels_id"] = self._pixels_id - #table.write(Path(path)/f"charge_lg_run{self.run_number}.ecsv",format='ascii.ecsv',overwrite=kwargs.get('overwrite',False)) - # - #table = Table(self.peak_hg) - #table.meta["pixels_id"] = self._pixels_id - #table.write(Path(path)/f"peak_hg_run{self.run_number}.ecsv",format='ascii.ecsv',overwrite=kwargs.get('overwrite',False)) - # - #table = Table(self.peak_lg) - #table.meta["pixels_id"] = self._pixels_id - #table.write(Path(path)/f"peak_lg_run{self.run_number}.ecsv",format='ascii.ecsv',overwrite=kwargs.get('overwrite',False)) - - hdr = fits.Header() - hdr['RUN'] = self._run_number - hdr['NEVENTS'] = self.nevents - hdr['NPIXELS'] = self.npixels - hdr['COMMENT'] = f"The charge containeur for run {self._run_number} with {self._method} method : primary is the pixels id, then you can find HG charge, LG charge, HG peak and LG peak, 2 last HDU are composed of event properties and trigger patern" - - primary_hdu = fits.PrimaryHDU(self.pixels_id,header=hdr) - charge_hg_hdu = fits.ImageHDU(self.charge_hg,name = "HG charge") - charge_lg_hdu = fits.ImageHDU(self.charge_lg,name = "LG charge") - peak_hg_hdu = fits.ImageHDU(self.peak_hg, name = 'HG peak time') - peak_lg_hdu = fits.ImageHDU(self.peak_lg, name = 'LG peak time') - - col1 = fits.Column(array = self.event_id, name = "event_id", format = '1I') - col2 = fits.Column(array = self.event_type, name = "event_type", format = '1I') - col3 = fits.Column(array = self.ucts_timestamp, name = "ucts_timestamp", format = '1K') - col4 = fits.Column(array = self.ucts_busy_counter, name = "ucts_busy_counter", format = '1I') - col5 = fits.Column(array = self.ucts_event_counter, name = "ucts_event_counter", format = '1I') - col6 = fits.Column(array = self.multiplicity, name = "multiplicity", format = '1I') - - coldefs = fits.ColDefs([col1, col2, col3, col4, col5, col6]) - event_properties = fits.BinTableHDU.from_columns(coldefs, name = 'event properties') - - col1 = fits.Column(array = self.trig_pattern_all, name = "trig_pattern_all", format = f'{4 * self.CAMERA.n_pixels}L',dim = f'({self.CAMERA.n_pixels},4)') - col2 = fits.Column(array = self.trig_pattern, name = "trig_pattern", format = f'{self.CAMERA.n_pixels}L') - coldefs = fits.ColDefs([col1, col2]) - trigger_patern = fits.BinTableHDU.from_columns(coldefs, name = 'trigger patern') - - hdul = fits.HDUList([primary_hdu, charge_hg_hdu, charge_lg_hdu,peak_hg_hdu,peak_lg_hdu,event_properties,trigger_patern]) - try : - hdul.writeto(Path(path)/f"charge_run{self.run_number}{suffix}.fits",overwrite=kwargs.get('overwrite',False)) - log.info(f"charge saved in {Path(path)}/charge_run{self.run_number}{suffix}.fits") - except OSError as e : - log.warning(e) - except Exception as e : - log.error(e,exc_info = True) - raise e - - - - - @staticmethod - def from_file(path : Path,run_number : int,**kwargs) : - """load ChargeContainer from FITS file previously written with ChargeContainer.write() method - - Args: - path (str): path of the FITS file - run_number (int) : the run number - - Returns: - ChargeContainer: ChargeContainer instance - """ - if kwargs.get("explicit_filename",False) : - filename = kwargs.get("explicit_filename") - log.info(f"loading {filename}") - else : - log.info(f"loading in {path} run number {run_number}") - filename = Path(path)/f"charge_run{run_number}.fits" - - with fits.open(filename) as hdul : - pixels_id = hdul[0].data - nevents = hdul[0].header['NEVENTS'] - npixels = hdul[0].header['NPIXELS'] - charge_hg = hdul[1].data - charge_lg = hdul[2].data - peak_hg = hdul[3].data - peak_lg = hdul[4].data - - cls = ChargeContainer(charge_hg,charge_lg,peak_hg,peak_lg,run_number,pixels_id,nevents,npixels) - - cls.event_id = hdul[5].data["event_id"] - cls.event_type = hdul[5].data["event_type"] - cls.ucts_timestamp = hdul[5].data["ucts_timestamp"] - cls.ucts_busy_counter = hdul[5].data["ucts_busy_counter"] - cls.ucts_event_counter = hdul[5].data["ucts_event_counter"] - - table_trigger = hdul[6].data - cls.trig_pattern_all = table_trigger["trig_pattern_all"] - - return cls - @staticmethod def compute_charge(waveformContainer : WaveformsContainer,channel : int,method : str = "FullWaveformSum" ,**kwargs) : diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 0c91e598..6b53f7e1 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -6,6 +6,12 @@ from abc import ABC, abstractmethod from ctapipe_io_nectarcam import NectarCAMEventSource +import numpy as np +from ctapipe.containers import EventType +from ctapipe.visualization import CameraDisplay +from ctapipe.coordinates import CameraFrame,EngineeringCameraFrame +from ctapipe.instrument import CameraGeometry,SubarrayDescription,TelescopeDescription +import copy from ..data import DataManagement @@ -39,4 +45,222 @@ def load_run(run_number : int,max_events : int = None, run_file = None) : else : log.info(f"{run_file} will be loaded") eventsource = NectarCAMEventSource(input_url=run_file,max_events=max_events) - return eventsource \ No newline at end of file + return eventsource + + +class LoopEventsMaker(BaseMaker): + def __init__(self,run_number : int,max_events : int = None,run_file = None,*args,**kwargs): + """construtor + + Args: + run_number (int): id of the run to be loaded + maxevents (int, optional): max of events to be loaded. Defaults to 0, to load everythings. + nevents (int, optional) : number of events in run if known (parameter used to save computing time) + run_file (optional) : if provided, will load this run file + """ + super().__init__(*args,**kwargs) + + self.__run_number = run_number + self.__run_file = run_file + self.__max_events = max_events + + self.__reader = __class__.load_run(run_number,max_events,run_file = run_file) + + #from reader members + self.__npixels = self.__reader.camera_config.num_pixels + self.__pixels_id = self.__reader.camera_config.expected_pixels_id + + log.info(f"N pixels : {self.npixels}") + + def make(self,n_events = np.inf, restart_from_begining = False) : + if restart_from_begining : + log.debug('restart from begining : creation of the EventSource reader') + self.__reader = __class__.load_run(self.__run_number,self.__max_events,run_file = self.__run_file) + + n_traited_events = 0 + for i,event in enumerate(self.__reader): + if i%100 == 0: + log.info(f"reading event number {i}") + self._make_event(event) + n_traited_events += 1 + if n_traited_events >= n_events : + break + + + @abstractmethod + def _make_event(self, event : EventType) : pass + + @property + def _run_file(self) : return self.__run_file + @property + def _max_events(self) : return self.__max_events + @property + def _reader(self) : return self.__reader + @property + def _npixels(self) : return self.__npixels + @property + def _pixels_id(self) : return self.__pixels_id + @property + def _run_number(self) : return self.__run_number + @property + def reader(self) : return copy.deepcopy(self.__reader) + @property + def npixels(self) : return copy.deepcopy(self.__npixels) + @property + def pixels_id(self) : return copy.deepcopy(self.__pixels_id) + @property + def run_number(self) : return copy.deepcopy(self.__run_number) + + + +class ArrayDataMaker(LoopEventsMaker) : + TEL_ID = 0 + CAMERA_NAME = "NectarCam-003" + CAMERA = CameraGeometry.from_name(CAMERA_NAME) + def __init__(self,run_number : int,max_events : int = None,run_file = None,*args,**kwargs): + """construtor + + Args: + run_number (int): id of the run to be loaded + maxevents (int, optional): max of events to be loaded. Defaults to 0, to load everythings. + nevents (int, optional) : number of events in run if known (parameter used to save computing time) + run_file (optional) : if provided, will load this run file + """ + super().__init__(run_number,max_events,run_file,*args,**kwargs) + + #data we want to compute + self.__ucts_timestamp = {} + self.__ucts_busy_counter = {} + self.__ucts_event_counter = {} + self.__event_type = {} + self.__event_id = {} + self.__trig_patter_all = {} + self.__broken_pixels_hg = {} + self.__broken_pixels_lg = {} + + def _init_trigger_type(self,trigger,**kwargs) : + name = __class__._get_name_trigger(trigger) + self.__ucts_timestamp[f"{name}"] = [] + self.__ucts_busy_counter[f"{name}"] = [] + self.__ucts_event_counter[f"{name}"] = [] + self.__event_type[f"{name}"] = [] + self.__event_id[f"{name}"] = [] + self.__trig_patter_all[f"{name}"] = [] + self.__broken_pixels_hg[f"{name}"] = [] + self.__broken_pixels_lg[f"{name}"] = [] + + def _compute_broken_pixels(self,wfs_hg_event,wfs_lg_event,**kwargs) : + log.warning("computation of broken pixels is not yet implemented") + return np.zeros((self.npixels),dtype = bool),np.zeros((self.npixels),dtype = bool) + + @staticmethod + def _get_name_trigger(trigger : EventType) : + if trigger is None : + name = "None" + else : + name = trigger.name + return name + + def make(self,n_events = np.inf, trigger_type : list = None, restart_from_begining = False) : + """mathod to extract data from the EventSource + + Args: + trigger_type (list[EventType], optional): only events with the asked trigger type will be use. Defaults to None. + compute_trigger_patern (bool, optional): To recompute on our side the trigger patern. Defaults to False. + """ + if ~np.isfinite(n_events) : + log.warning('no needed events number specified, it may cause a memory error') + if isinstance(trigger_type,EventType) or trigger_type is None : + trigger_type = [trigger_type] + for _trigger_type in trigger_type : + self._init_trigger_type(_trigger_type) + + if restart_from_begining : + log.debug('restart from begining : creation of the EventSource reader') + self.__reader = __class__.load_run(self.__run_number,self.__max_events,run_file = self.__run_file) + + n_traited_events = 0 + for i,event in enumerate(self.__reader): + if i%100 == 0: + log.info(f"reading event number {i}") + for trigger in trigger_type : + if (trigger is None) or (trigger == event.trigger.event_type) : + self._make_event(event,trigger) + n_traited_events += 1 + if n_traited_events >= n_events : + break + + return self._make_output_container() + + + def _make_event(self,event,trigger) : + name = __class__._get_name_trigger(trigger) + self.__event_id[f'{name}'].append(np.uint16(event.index.event_id)) + self.__ucts_timestamp[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_timestamp) + self.__event_type[f'{name}'].append(event.trigger.event_type.value) + self.__ucts_busy_counter[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_busy_counter) + self.__ucts_event_counter[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_event_counter) + self.__trig_patter_all[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.trigger_pattern.T) + + broken_pixels_hg,broken_pixels_lg = self._compute_broken_pixels() + self.__broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) + self.__broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) + + @abstractmethod + def _make_output_container(self) : pass + + @staticmethod + def create_from_events_list(events_list : list, + run_number : int, + npixels : int, + nsamples : int, + subarray, + pixels_id : int, + ) : + cls = __class__.__new__() + + cls.__run_number = run_number + cls.__nevents = len(events_list) + cls.__npixels = npixels + cls.__nsamples = nsamples + cls.__subarray = subarray + cls.__pixels_id = pixels_id + + cls.__ucts_timestamp = {} + cls.__ucts_busy_counter = {} + cls.__ucts_event_counter = {} + cls.__event_type = {} + cls.__event_id = {} + cls.__trig_patter_all = {} + cls.__broken_pixels_hg = {} + cls.__broken_pixels_lg = {} + + return cls + + + def nevents(self,trigger) : return len(self.__event_id[__class__._get_name_trigger(trigger)]) + @property + def _broken_pixels_hg(self) : return self.__broken_pixels_hg + def broken_pixels_hg(self,trigger) : return np.array(self.__broken_pixels_hg[__class__._get_name_trigger(trigger)],dtype = bool) + @property + def _broken_pixels_lg(self) : return self.__broken_pixels_lg + def broken_pixels_lg(self,trigger) : return np.array(self.__broken_pixels_lg[__class__._get_name_trigger(trigger)],dtype = bool) + def ucts_timestamp(self,trigger) : return np.array(self.__ucts_timestamp[__class__._get_name_trigger(trigger)],dtype = np.uint64) + def ucts_busy_counter(self,trigger) : return np.array(self.__ucts_busy_counter[__class__._get_name_trigger(trigger)],dtype = np.uint32) + def ucts_event_counter(self,trigger) : return np.array(self.__ucts_event_counter[__class__._get_name_trigger(trigger)],dtype = np.uint32) + def event_type(self,trigger) : return np.array(self.__event_type[__class__._get_name_trigger(trigger)],dtype = np.uint8) + def event_id(self,trigger) : return np.array(self.__event_id[__class__._get_name_trigger(trigger)],dtype = np.uint32) + def multiplicity(self,trigger) : + tmp = self.trig_pattern(trigger) + if len(tmp) == 0 : + return np.array([]) + else : + return np.uint16(np.count_nonzero(tmp,axis = 1)) + def trig_pattern(self,trigger) : + tmp = self.trig_pattern_all(trigger) + if len(tmp) == 0 : + return np.array([]) + else : + return tmp.any(axis = 2) + def trig_pattern_all(self,trigger) : return np.array(self.__trig_patter_all[f"{__class__._get_name_trigger(trigger)}"],dtype = bool) + diff --git a/src/nectarchain/makers/waveformsMakers.py b/src/nectarchain/makers/waveformsMakers.py index dd1910b1..a4776318 100644 --- a/src/nectarchain/makers/waveformsMakers.py +++ b/src/nectarchain/makers/waveformsMakers.py @@ -1,3 +1,8 @@ +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + from argparse import ArgumentError import numpy as np from matplotlib import pyplot as plt @@ -23,24 +28,17 @@ from ..data import DataManagement import sys -import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') -log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers from ctapipe.containers import EventType from ..data.container import WaveformsContainer -from nectarchain.makers import BaseMaker +from .core import ArrayDataMaker __all__ = ["WaveformsMaker"] -class WaveformsMaker(BaseMaker) : +class WaveformsMaker(ArrayDataMaker) : """class use to make the waveform extraction from event read from r0 data """ - TEL_ID = 0 - CAMERA_NAME = "NectarCam-003" - CAMERA = CameraGeometry.from_name(CAMERA_NAME) #constructors def __init__(self,run_number : int,max_events : int = None,run_file = None,*args,**kwargs): @@ -52,35 +50,14 @@ def __init__(self,run_number : int,max_events : int = None,run_file = None,*args nevents (int, optional) : number of events in run if known (parameter used to save computing time) run_file (optional) : if provided, will load this run file """ - super().__init__(*args,**kwargs) + super().__init__(run_number,max_events,run_file,*args,**kwargs) - self.__run_number = run_number - self.__run_file = run_file - self.__max_events = max_events - - self.__reader = WaveformsMaker.load_run(run_number,max_events,run_file = run_file) - - #from reader members - self.__npixels = self.__reader.camera_config.num_pixels - self.__nsamples = self.__reader.camera_config.num_samples - self.__geometry = self.__reader.subarray.tel[WaveformsMaker.TEL_ID].camera - self.__subarray = self.__reader.subarray - self.__pixels_id = self.__reader.camera_config.expected_pixels_id - - - log.info(f"N pixels : {self.npixels}") - - #data we want to compute + self.__geometry = self._reader.subarray.tel[__class__.TEL_ID].camera + self.__subarray = self._reader.subarray + self.__nsamples = self._reader.camera_config.num_samples + self.__wfs_hg = {} self.__wfs_lg = {} - self.__ucts_timestamp = {} - self.__ucts_busy_counter = {} - self.__ucts_event_counter = {} - self.__event_type = {} - self.__event_id = {} - self.__trig_patter_all = {} - self.__broken_pixels_hg = {} - self.__broken_pixels_lg = {} @staticmethod def create_from_events_list(events_list : list, @@ -90,105 +67,44 @@ def create_from_events_list(events_list : list, subarray, pixels_id : int, ) : - cls = __class__.__new__() - - cls.__run_number = run_number - cls.__nevents = len(events_list) - cls.__npixels = npixels + cls = super().create_from_events_list(events_list = events_list, + run_number = run_number, + npixels = npixels, + pixels_id = pixels_id + ) + cls.__nsamples = nsamples cls.__subarray = subarray - cls.__pixels_id = pixels_id cls.__wfs_hg = {} cls.__wfs_lg = {} - cls.__ucts_timestamp = {} - cls.__ucts_busy_counter = {} - cls.__ucts_event_counter = {} - cls.__event_type = {} - cls.__event_id = {} - cls.__trig_patter_all = {} - cls.__broken_pixels_hg = {} - cls.__broken_pixels_lg = {} - cls.__init_trigger_type(None) - + cls._init_trigger_type(None) + for event in tqdm(events_list): cls._make_event(event,None) - return cls.__make_output_container([None]) + return cls._make_output_container([None]) - def __init_trigger_type(self,trigger_type,**kwargs) : - name = WaveformsMaker.__get_name_trigger(trigger_type) - log.info(f"initiamization of the waveformsMaker following trigger type : {name}") + def _init_trigger_type(self,trigger_type,**kwargs) : + super()._init_trigger_type(trigger_type,**kwargs) + name = __class__._get_name_trigger(trigger_type) + log.info(f"initialization of the waveformsMaker following trigger type : {name}") self.__wfs_hg[f"{name}"] = [] self.__wfs_lg[f"{name}"] = [] - self.__ucts_timestamp[f"{name}"] = [] - self.__ucts_busy_counter[f"{name}"] = [] - self.__ucts_event_counter[f"{name}"] = [] - self.__event_type[f"{name}"] = [] - self.__event_id[f"{name}"] = [] - self.__trig_patter_all[f"{name}"] = [] - self.__broken_pixels_hg[f"{name}"] = [] - self.__broken_pixels_lg[f"{name}"] = [] - - def __compute_broken_pixels(self,wfs_hg_event,wfs_lg_event,**kwargs) : - log.warning("computation of broken pixels is not yet implemented") - return np.zeros((self.npixels),dtype = bool),np.zeros((self.npixels),dtype = bool) - - @staticmethod - def __get_name_trigger(trigger : EventType) : - if trigger is None : - name = "None" - else : - name = trigger.name - return name - def make(self,n_events = np.inf, trigger_type : list = None, restart_from_begining = False): - """mathod to extract waveforms data from the EventSource - - Args: - trigger_type (list[EventType], optional): only events with the asked trigger type will be use. Defaults to None. - compute_trigger_patern (bool, optional): To recompute on our side the trigger patern. Defaults to False. - """ - if ~np.isfinite(n_events) : - log.warning('no needed events number specified, it may cause a memory error') - if isinstance(trigger_type,EventType) or trigger_type is None : - trigger_type = [trigger_type] - - if restart_from_begining : - log.debug('restart from begining : creation of the EventSource reader') - self.__reader = WaveformsMaker.load_run(self.__run_number,self.__max_events,run_file = self.__run_file) - - for _trigger_type in trigger_type : - self.__init_trigger_type(_trigger_type) - - n_traited_events = 0 - for i,event in enumerate(self.__reader): - if i%100 == 0: - log.info(f"reading event number {i}") - for trigger in trigger_type : - if (trigger is None) or (trigger == event.trigger.event_type) : - self._make_event(event,trigger) - n_traited_events += 1 - if n_traited_events >= n_events : - break - - return self.__make_output_container(trigger_type) + def _make_event(self, event, trigger : EventType ) : - name = WaveformsMaker.__get_name_trigger(trigger) - self.__event_id[f'{name}'].append(np.uint16(event.index.event_id)) - self.__ucts_timestamp[f'{name}'].append(event.nectarcam.tel[WaveformsMaker.TEL_ID].evt.ucts_timestamp) - self.__event_type[f'{name}'].append(event.trigger.event_type.value) - self.__ucts_busy_counter[f'{name}'].append(event.nectarcam.tel[WaveformsMaker.TEL_ID].evt.ucts_busy_counter) - self.__ucts_event_counter[f'{name}'].append(event.nectarcam.tel[WaveformsMaker.TEL_ID].evt.ucts_event_counter) - self.__trig_patter_all[f'{name}'].append(event.nectarcam.tel[WaveformsMaker.TEL_ID].evt.trigger_pattern.T) + super()._make_event(event = event, + trigger = trigger) + name = __class__._get_name_trigger(trigger) wfs_hg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) wfs_lg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) @@ -200,11 +116,11 @@ def _make_event(self, self.__wfs_hg[f'{name}'].append(wfs_hg_tmp.tolist()) self.__wfs_lg[f'{name}'].append(wfs_lg_tmp.tolist()) - broken_pixels_hg,broken_pixels_lg = self.__compute_broken_pixels(wfs_hg_tmp,wfs_lg_tmp) - self.__broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) - self.__broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) + broken_pixels_hg,broken_pixels_lg = self._compute_broken_pixels(wfs_hg_tmp,wfs_lg_tmp) + self._broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) + self._broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) - def __make_output_container(self,trigger_type) : + def _make_output_container(self,trigger_type) : output = [] for trigger in trigger_type : waveformsContainer = WaveformsContainer( @@ -332,45 +248,12 @@ def select_waveforms_lg(waveformsContainer,pixel_id : np.ndarray) : return res - @property - def _run_file(self) : return self.__run_file - @property - def _max_events(self) : return self.__max_events - @property - def reader(self) : return self.__reader - @property - def npixels(self) : return self.__npixels @property def nsamples(self) : return self.__nsamples @property def geometry(self) : return self.__geometry @property def subarray(self) : return self.__subarray - @property - def pixels_id(self) : return self.__pixels_id - @property - def run_number(self) : return self.__run_number - def nevents(self,trigger) : return len(self.__event_id[WaveformsMaker.__get_name_trigger(trigger)]) - def wfs_hg(self,trigger) : return np.array(self.__wfs_hg[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint16) - def wfs_lg(self,trigger) : return np.array(self.__wfs_lg[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint16) - def broken_pixels_hg(self,trigger) : return np.array(self.__broken_pixels_hg[WaveformsMaker.__get_name_trigger(trigger)],dtype = bool) - def broken_pixels_lg(self,trigger) : return np.array(self.__broken_pixels_lg[WaveformsMaker.__get_name_trigger(trigger)],dtype = bool) - def ucts_timestamp(self,trigger) : return np.array(self.__ucts_timestamp[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint64) - def ucts_busy_counter(self,trigger) : return np.array(self.__ucts_busy_counter[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint32) - def ucts_event_counter(self,trigger) : return np.array(self.__ucts_event_counter[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint32) - def event_type(self,trigger) : return np.array(self.__event_type[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint8) - def event_id(self,trigger) : return np.array(self.__event_id[WaveformsMaker.__get_name_trigger(trigger)],dtype = np.uint32) - def multiplicity(self,trigger) : - tmp = self.trig_pattern(trigger) - if len(tmp) == 0 : - return np.array([]) - else : - return np.uint16(np.count_nonzero(tmp,axis = 1)) - def trig_pattern(self,trigger) : - tmp = self.trig_pattern_all(trigger) - if len(tmp) == 0 : - return np.array([]) - else : - return tmp.any(axis = 2) - def trig_pattern_all(self,trigger) : return np.array(self.__trig_patter_all[f"{WaveformsMaker.__get_name_trigger(trigger)}"],dtype = bool) - + def wfs_hg(self,trigger) : return np.array(self.__wfs_hg[__class__._get_name_trigger(trigger)],dtype = np.uint16) + def wfs_lg(self,trigger) : return np.array(self.__wfs_lg[__class__._get_name_trigger(trigger)],dtype = np.uint16) + \ No newline at end of file From a1eee04643592db263007265c4250b9f54984c4f Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Thu, 14 Sep 2023 15:42:13 +0200 Subject: [PATCH 47/62] display module --- src/nectarchain/display/__init__.py | 1 + src/nectarchain/display/display.py | 75 +++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 src/nectarchain/display/__init__.py create mode 100644 src/nectarchain/display/display.py diff --git a/src/nectarchain/display/__init__.py b/src/nectarchain/display/__init__.py new file mode 100644 index 00000000..0b14e0cf --- /dev/null +++ b/src/nectarchain/display/__init__.py @@ -0,0 +1 @@ +from .display import * \ No newline at end of file diff --git a/src/nectarchain/display/display.py b/src/nectarchain/display/display.py new file mode 100644 index 00000000..673efe1e --- /dev/null +++ b/src/nectarchain/display/display.py @@ -0,0 +1,75 @@ +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +from argparse import ArgumentError +import numpy as np +from matplotlib import pyplot as plt +import copy +import os +import glob +from pathlib import Path + +from enum import Enum + +from tqdm import tqdm + +from astropy.io import fits +from astropy.table import QTable,Column,Table +import astropy.units as u + +from ctapipe.visualization import CameraDisplay +from ctapipe.coordinates import CameraFrame,EngineeringCameraFrame +from ctapipe.instrument import CameraGeometry,SubarrayDescription,TelescopeDescription + +from ctapipe_io_nectarcam import NectarCAMEventSource + +from ..data import DataManagement + +import sys + +from ctapipe.containers import EventType +from ..data.container import WaveformsContainer,ChargesContainer,ArrayDataContainer + +from abc import ABC + +class ContainerDisplay(ABC) : + @staticmethod + def display(container : ArrayDataContainer,evt,geometry, cmap = 'gnuplot2') : + """plot camera display for HIGH GAIN channel + + Args: + evt (int): event index + cmap (str, optional): colormap. Defaults to 'gnuplot2'. + + Returns: + CameraDisplay: thoe cameraDisplay plot + """ + if isinstance(container,ChargesContainer) : + image = container.charges_hg + elif isinstance(container,WaveformsContainer) : + image = container.wfs_hg.sum(axis=2) + else : + log.warning("container can't be displayed") + disp = CameraDisplay(geometry=geometry, image=image[evt], cmap=cmap) + disp.add_colorbar() + return disp + + @staticmethod + def plot_waveform(waveformsContainer : WaveformsContainer,evt,**kwargs) : + """plot the waveform of the evt in the HIGH GAIN channel + + Args: + evt (int): the event index + + Returns: + tuple: the figure and axes + """ + if 'figure' in kwargs.keys() and 'ax' in kwargs.keys() : + fig = kwargs.get('figure') + ax = kwargs.get('ax') + else : + fig,ax = plt.subplots(1,1) + ax.plot(waveformsContainer.wfs_hg[evt].T) + return fig,ax From fd0e78a2b6df238f8e12c15bd5dbf0f859bf4b30 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 15 Sep 2023 00:36:00 +0200 Subject: [PATCH 48/62] finalisation of the refactorisation of the waveformsMaker chargesMaker implementation --- .../data/container/chargesContainer.py | 8 +- src/nectarchain/makers/chargesMakers.py | 588 ++++++------------ src/nectarchain/makers/core.py | 112 +++- src/nectarchain/makers/waveformsMakers.py | 186 +++--- 4 files changed, 362 insertions(+), 532 deletions(-) diff --git a/src/nectarchain/data/container/chargesContainer.py b/src/nectarchain/data/container/chargesContainer.py index eadce599..d555075e 100644 --- a/src/nectarchain/data/container/chargesContainer.py +++ b/src/nectarchain/data/container/chargesContainer.py @@ -10,12 +10,12 @@ class ChargesContainer(ArrayDataContainer): - charge_hg = Field( + charges_hg = Field( type = np.ndarray, - description = 'The high gain charge') - charge_lg = Field( + description = 'The high gain charges') + charges_lg = Field( type = np.ndarray, - description = 'The low gain charge') + description = 'The low gain charges') peak_hg = Field( type = np.ndarray, description = 'The high gain peak time') diff --git a/src/nectarchain/makers/chargesMakers.py b/src/nectarchain/makers/chargesMakers.py index 744f9e5e..294a9000 100644 --- a/src/nectarchain/makers/chargesMakers.py +++ b/src/nectarchain/makers/chargesMakers.py @@ -14,14 +14,19 @@ from ctapipe.instrument import CameraGeometry from ctapipe_io_nectarcam import constants +from ctapipe.containers import EventType +from ctapipe.image import ImageExtractor + from astropy.io import fits from numba import guvectorize, float64, int64, bool_ -from ..data.container import WaveformsContainer +from ..data.container import WaveformsContainer,ChargesContainer from .core import ArrayDataMaker +from .extractor.utils import CtapipeExtractor + @@ -94,124 +99,181 @@ def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): class ChargeMaker(ArrayDataMaker) : - """ - class used to compute charge from waveforms container - - Attributes: - TEL_ID (int): Telescope ID - CAMERA (CameraGeometry): Camera geometry - - Methods: - __init__(self, charge_hg, charge_lg, peak_hg, peak_lg, run_number, pixels_id, nevents, npixels, method="FullWaveformSum"): Initializes a ChargeContainer instance - from_waveforms(cls, waveformContainer, method="FullWaveformSum", **kwargs): Creates a ChargeContainer instance from a WaveformsContainer - write(self, path, **kwargs): Writes the charge data to a FITS file - from_file(path, run_number, **kwargs): Loads charge data from a FITS file - compute_charge(waveformContainer, channel, method="FullWaveformSum", **kwargs): Computes charge from waveforms - histo_hg(self, n_bins=1000, autoscale=True): Computes the histogram of the high-gain (HG) channel charge values - histo_lg(self, n_bins=1000, autoscale=True): Computes the histogram of the low-gain (LG) channel charge values - select_charge_hg(self, pixel_id): Extracts the high-gain (HG) charge values for the specified pixel IDs - select_charge_lg(self, pixel_id): Extracts the low-gain (LG) charge values for the specified pixel IDs - sort(self, method='event_id'): Sorts the charge data based on the specified method - run_number(self): Returns the run number - pixels_id(self): Returns the pixel IDs - npixels(self): Returns the number of pixels - nevents(self): Returns the number of events - method(self): Returns the charge computation method - multiplicity(self): Returns the multiplicity of events - trig_pattern(self): Returns the trigger pattern - - Example Usage: - # Create a ChargeContainer instance from a WaveformsContainer - chargeContainer = ChargeContainer.from_waveforms(waveformContainer) - - # Write the charge data to a FITS file - chargeContainer.write(path) - - # Load charge data from a FITS file - chargeContainer = ChargeContainer.from_file(path, run_number) - - # Compute histograms of the charge values - hist_hg = chargeContainer.histo_hg() - hist_lg = chargeContainer.histo_lg() - - # Select charge values for specific pixels - pixel_ids = [1, 2, 3] - charge_hg = chargeContainer.select_charge_hg(pixel_ids) - charge_lg = chargeContainer.select_charge_lg(pixel_ids) - - # Sort the charge data based on event ID - chargeContainer.sort() - - # Access properties of the ChargeContainer - run_number = chargeContainer.run_number - pixels_id = chargeContainer.pixels_id - npixels = chargeContainer.npixels - nevents = chargeContainer.nevents - method = chargeContainer.method - """ +#constructors + def __init__(self,run_number : int,max_events : int = None,run_file = None,*args,**kwargs): + """construtor - def __init__(self, charge_hg, charge_lg, peak_hg, peak_lg, run_number, pixels_id, nevents, npixels, method="FullWaveformSum"): + Args: + run_number (int): id of the run to be loaded + maxevents (int, optional): max of events to be loaded. Defaults to 0, to load everythings. + nevents (int, optional) : number of events in run if known (parameter used to save computing time) + run_file (optional) : if provided, will load this run file """ - Initializes a ChargeContainer instance with the provided arguments and sets some additional attributes to default values. + super().__init__(run_number,max_events,run_file,*args,**kwargs) + self.__charges_hg = {} + self.__charges_lg = {} + self.__peak_hg = {} + self.__peak_lg = {} + + def _init_trigger_type(self,trigger_type,**kwargs) : + super()._init_trigger_type(trigger_type,**kwargs) + name = __class__._get_name_trigger(trigger_type) + log.info(f"initialization of the chargeMaker following trigger type : {name}") + self.__charges_hg[f"{name}"] = [] + self.__charges_lg[f"{name}"] = [] + self.__peak_hg[f"{name}"] = [] + self.__peak_lg[f"{name}"] = [] + + + def make(self, + n_events = np.inf, + trigger_type : list = None, + restart_from_begining = False, + method: str = "FullWaveformSum", + *args,**kwargs): + kwargs["method"]=method + super().make(n_events=n_events, + trigger_type=trigger_type, + restart_from_begining=restart_from_begining, + *args,**kwargs) + + def _make_event(self, + event, + trigger : EventType, + method: str = "FullWaveformSum", + *args, + **kwargs + ) : + wfs_hg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) + wfs_lg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) + + super()._make_event(event = event, + trigger = trigger, + wfs_hg = wfs_hg_tmp, + wfs_lg = wfs_lg_tmp, + *args, + **kwargs) + name = __class__._get_name_trigger(trigger) + + broken_pixels_hg,broken_pixels_lg = __class__._compute_broken_pixels(wfs_hg_tmp,wfs_lg_tmp) + self._broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) + self._broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) + + + imageExtractor = __class__._get_imageExtractor(method,self._reader.subarray,**kwargs) + + __image = CtapipeExtractor.get_image_peak_time(imageExtractor(wfs_hg_tmp,__class__.TEL_ID,constants.HIGH_GAIN,broken_pixels_hg)) + self.__charges_hg[f"{name}"].append(__image[0].tolist()) + self.__peak_hg[f"{name}"].append(__image[1].tolist()) + + __image = CtapipeExtractor.get_image_peak_time(imageExtractor(wfs_lg_tmp,__class__.TEL_ID,constants.LOW_GAIN,broken_pixels_lg)) + self.__charges_lg[f"{name}"].append(__image[0].tolist()) + self.__peak_lg[f"{name}"].append(__image[1].tolist()) - Args: - charge_hg (array): The high-gain charge values. - charge_lg (array): The low-gain charge values. - peak_hg (array): The high-gain peak time values. - peak_lg (array): The low-gain peak time values. - run_number (int): The run number. - pixels_id (array): The pixel IDs. - nevents (int): The number of events. - npixels (int): The number of pixels. - method (str, optional): The charge computation method. Defaults to "FullWaveformSum". + @staticmethod + def _get_imageExtractor(method,subarray,**kwargs) : + if not(method in list_ctapipe_charge_extractor or method in list_nectarchain_charge_extractor) : + raise ArgumentError(f"method must be in {list_ctapipe_charge_extractor}") - Returns: - None + extractor_kwargs = {} + for key in eval(method).class_own_traits().keys() : + if key in kwargs.keys() : + extractor_kwargs[key] = kwargs[key] + + if "apply_integration_correction" in eval(method).class_own_traits().keys() : #to change the default behavior of ctapipe extractor + extractor_kwargs["apply_integration_correction"] = kwargs.get("apply_integration_correction",False) + + log.debug(f"Extracting charges with method {method} and extractor_kwargs {extractor_kwargs}") + imageExtractor = eval(method)(subarray,**extractor_kwargs) + return imageExtractor + + def _make_output_container(self,trigger_type) : + output = [] + for trigger in trigger_type : + chargesContainer = ChargesContainer( + run_number = self.run_number, + npixels = self.npixels, + camera = self.CAMERA_NAME, + pixels_id = self.pixels_id, + nevents = self.nevents(trigger), + charges_hg = self.charges_hg(trigger), + charges_lg = self.charges_lg(trigger), + peak_hg = self.peak_hg(trigger), + peak_lg = self.peak_lg(trigger), + broken_pixels_hg = self.broken_pixels_hg(trigger), + broken_pixels_lg = self.broken_pixels_lg(trigger), + ucts_timestamp = self.ucts_timestamp(trigger), + ucts_busy_counter = self.ucts_busy_counter(trigger), + ucts_event_counter = self.ucts_event_counter(trigger), + event_type = self.event_type(trigger), + event_id = self.event_id(trigger), + trig_pattern_all = self.trig_pattern_all(trigger), + trig_pattern = self.trig_pattern(trigger), + multiplicity = self.multiplicity(trigger) + ) + output.append(chargesContainer) + return output + + @staticmethod + def sort(chargesContainer :ChargesContainer, method = 'event_id') : + output = ChargesContainer( + run_number = chargesContainer.run_number, + npixels = chargesContainer.npixels, + camera = chargesContainer.camera, + pixels_id = chargesContainer.pixels_id, + nevents = chargesContainer.nevents + ) + if method == 'event_id' : + index = np.argsort(chargesContainer.event_id) + for field in chargesContainer.keys() : + if not(field in ["run_number","npixels","camera","pixels_id","nevents"]) : + output[field] = chargesContainer[field][index] + else : + raise ArgumentError(f"{method} is not a valid method for sorting") + return output + + + @staticmethod + def select_charges_hg(chargesContainer :ChargesContainer,pixel_id : np.ndarray) : + res = __class__.select_container_array_field(container = chargesContainer,pixel_id = pixel_id,field = 'charges_hg') + res = res.transpose(1,0) + return res + + + @staticmethod + def select_charges_lg(chargesContainer : ChargesContainer,pixel_id : np.ndarray) : + res = __class__.select_container_array_field(container = chargesContainer,pixel_id = pixel_id,field = 'charges_lg') + res = res.transpose(1,0) + return res - """ - self.charge_hg = charge_hg - self.charge_lg = charge_lg - self.peak_hg = peak_hg - self.peak_lg = peak_lg - self._run_number = run_number - self._pixels_id = pixels_id - self._method = method - self._nevents = nevents - self._npixels = npixels - - self.ucts_timestamp = np.zeros((self.nevents), dtype=np.uint64) - self.ucts_busy_counter = np.zeros((self.nevents), dtype=np.uint16) - self.ucts_event_counter = np.zeros((self.nevents), dtype=np.uint16) - self.event_type = np.zeros((self.nevents), dtype=np.uint8) - self.event_id = np.zeros((self.nevents), dtype=np.uint16) - self.trig_pattern_all = np.zeros((self.nevents, self.CAMERA.n_pixels, 4), dtype=bool) - - - @classmethod - def from_waveforms(cls, waveformContainer: WaveformsContainer, method: str = "FullWaveformSum", **kwargs) -> 'ChargeContainer': - """Create a new ChargeContainer instance from a WaveformsContainer. - Args: - waveformContainer (WaveformsContainer): The waveforms container object from which to compute the charge values. - method (str, optional): The method to use for charge computation. Defaults to "FullWaveformSum". - Returns: - ChargeContainer: The created ChargeContainer instance with computed charge values and other attributes. - """ - log.info(f"computing hg charge with {method} method") - charge_hg, peak_hg = ChargeContainer.compute_charge(waveformContainer, constants.HIGH_GAIN, method, **kwargs) - charge_hg = np.array(charge_hg, dtype=np.uint16) - log.info(f"computing lg charge with {method} method") - charge_lg, peak_lg = ChargeContainer.compute_charge(waveformContainer, constants.LOW_GAIN, method, **kwargs) - charge_lg = np.array(charge_lg, dtype=np.uint16) - chargeContainer = cls(charge_hg, charge_lg, peak_hg, peak_lg, waveformContainer.run_number, waveformContainer.pixels_id, waveformContainer.nevents, waveformContainer.npixels, method) - chargeContainer.ucts_timestamp = waveformContainer.ucts_timestamp - chargeContainer.ucts_busy_counter = waveformContainer.ucts_busy_counter - chargeContainer.ucts_event_counter = waveformContainer.ucts_event_counter - chargeContainer.event_type = waveformContainer.event_type - chargeContainer.event_id = waveformContainer.event_id - chargeContainer.trig_pattern_all = waveformContainer.trig_pattern_all - return chargeContainer + def charges_hg(self,trigger) : return np.array(self.__charges_hg[__class__._get_name_trigger(trigger)],dtype = np.uint16) + def charges_lg(self,trigger) : return np.array(self.__charges_lg[__class__._get_name_trigger(trigger)],dtype = np.uint16) + def peak_hg(self,trigger) : return np.array(self.__peak_hg[__class__._get_name_trigger(trigger)],dtype = np.uint16) + def peak_lg(self,trigger) : return np.array(self.__peak_lg[__class__._get_name_trigger(trigger)],dtype = np.uint16) + + + @staticmethod + def create_from_waveforms(waveformsContainer: WaveformsContainer, method: str = "FullWaveformSum", **kwargs) -> ChargesContainer: + chargesContainer = ChargesContainer() + + for field in waveformsContainer.keys() : + if not(field in ["subarray","nsamples","wfs_hg","wfs_lg"]) : + chargesContainer[field] = waveformsContainer[field] + + log.info(f"computing hg charge with {method} method") + charges_hg, peak_hg = __class__.compute_charge(waveformsContainer, constants.HIGH_GAIN, method, **kwargs) + charges_hg = np.array(charges_hg, dtype=np.uint16) + log.info(f"computing lg charge with {method} method") + charges_lg, peak_lg = __class__.compute_charge(waveformsContainer, constants.LOW_GAIN, method, **kwargs) + charges_lg = np.array(charges_lg, dtype=np.uint16) + chargesContainer.charges_hg = charges_hg + chargesContainer.charges_lg = charges_lg + chargesContainer.peak_hg = peak_hg + chargesContainer.peak_lg = peak_lg + + return chargesContainer @staticmethod def compute_charge(waveformContainer : WaveformsContainer,channel : int,method : str = "FullWaveformSum" ,**kwargs) : @@ -231,32 +293,35 @@ def compute_charge(waveformContainer : WaveformsContainer,channel : int,method : """ #import is here for fix issue with pytest (TypeError : inference is not possible with python <3.9 (Numba conflict bc there is no inference...)) - from ...makers.extractor.utils import CtapipeExtractor + from .extractor.utils import CtapipeExtractor - if not(method in list_ctapipe_charge_extractor or method in list_nectarchain_charge_extractor) : - raise ArgumentError(f"method must be in {list_ctapipe_charge_extractor}") - - extractor_kwargs = {} - for key in eval(method).class_own_traits().keys() : - if key in kwargs.keys() : - extractor_kwargs[key] = kwargs[key] - - if "apply_integration_correction" in eval(method).class_own_traits().keys() : #to change the default behavior of ctapipe extractor - extractor_kwargs["apply_integration_correction"] = kwargs.get("apply_integration_correction",False) - - log.debug(f"Extracting charges with method {method} and extractor_kwargs {extractor_kwargs}") - ImageExtractor = eval(method)(waveformContainer.subarray,**extractor_kwargs) + imageExtractor = __class__._get_imageExtractor(method = method,subarray = waveformContainer.subarray,**kwargs) if channel == constants.HIGH_GAIN: - out = np.array([CtapipeExtractor.get_image_peak_time(ImageExtractor(waveformContainer.wfs_hg[i],waveformContainer.TEL_ID,channel,waveformContainer.broken_pixels_hg)) for i in range(len(waveformContainer.wfs_hg))]).transpose(1,0,2) + out = np.array([CtapipeExtractor.get_image_peak_time(imageExtractor(waveformContainer.wfs_hg[i],waveformContainer.TEL_ID,channel,waveformContainer.broken_pixels_hg)) for i in range(len(waveformContainer.wfs_hg))]).transpose(1,0,2) return out[0],out[1] elif channel == constants.LOW_GAIN: - out = np.array([CtapipeExtractor.get_image_peak_time(ImageExtractor(waveformContainer.wfs_lg[i],waveformContainer.TEL_ID,channel,waveformContainer.broken_pixels_lg)) for i in range(len(waveformContainer.wfs_lg))]).transpose(1,0,2) + out = np.array([CtapipeExtractor.get_image_peak_time(imageExtractor(waveformContainer.wfs_lg[i],waveformContainer.TEL_ID,channel,waveformContainer.broken_pixels_lg)) for i in range(len(waveformContainer.wfs_lg))]).transpose(1,0,2) return out[0],out[1] else : raise ArgumentError(f"channel must be {constants.LOW_GAIN} or {constants.HIGH_GAIN}") - def histo_hg(self,n_bins : int = 1000,autoscale : bool = True) -> ma.masked_array: + @staticmethod + def histo_hg(chargesContainer : ChargesContainer,n_bins : int = 1000,autoscale : bool = True) -> ma.masked_array: + return __class__._histo(chargesContainer = chargesContainer, + field = 'charges_hg', + n_bins = n_bins, + autoscale = autoscale) + + @staticmethod + def histo_lg(chargesContainer : ChargesContainer,n_bins : int = 1000,autoscale : bool = True) -> ma.masked_array: + return __class__._histo(chargesContainer = chargesContainer, + field = 'charges_lg', + n_bins = n_bins, + autoscale = autoscale) + + @staticmethod + def _histo(chargesContainer : ChargesContainer,field : str, n_bins : int = 1000,autoscale : bool = True) -> ma.masked_array: """method to compute histogram of HG channel Numba is used to compute histograms in vectorized way @@ -267,18 +332,18 @@ def histo_hg(self,n_bins : int = 1000,autoscale : bool = True) -> ma.masked_arra Returns: np.ndarray: masked array of charge histograms (histo,charge) """ - mask_broken_pix = np.array((self.charge_hg == self.charge_hg.mean(axis = 0)).mean(axis=0),dtype = bool) + mask_broken_pix = np.array((chargesContainer[field] == chargesContainer[field].mean(axis = 0)).mean(axis=0),dtype = bool) log.debug(f"there are {mask_broken_pix.sum()} broken pixels (charge stays at same level for each events)") if autoscale : - all_range = np.arange(np.uint16(np.min(self.charge_hg.T[~mask_broken_pix].T)) - 0.5,np.uint16(np.max(self.charge_hg.T[~mask_broken_pix].T)) + 1.5,1) - #hist_ma = ma.masked_array(np.zeros((self.charge_hg.shape[1],all_range.shape[0]),dtype = np.uint16), mask=np.zeros((self.charge_hg.shape[1],all_range.shape[0]),dtype = bool)) - charge_ma = ma.masked_array((all_range.reshape(all_range.shape[0],1) @ np.ones((1,self.charge_hg.shape[1]))).T, mask=np.zeros((self.charge_hg.shape[1],all_range.shape[0]),dtype = bool)) + all_range = np.arange(np.uint16(np.min(chargesContainer[field].T[~mask_broken_pix].T)) - 0.5,np.uint16(np.max(chargesContainer[field].T[~mask_broken_pix].T)) + 1.5,1) + #hist_ma = ma.masked_array(np.zeros((self[field].shape[1],all_range.shape[0]),dtype = np.uint16), mask=np.zeros((self[field].shape[1],all_range.shape[0]),dtype = bool)) + charge_ma = ma.masked_array((all_range.reshape(all_range.shape[0],1) @ np.ones((1,chargesContainer[field].shape[1]))).T, mask=np.zeros((chargesContainer[field].shape[1],all_range.shape[0]),dtype = bool)) broxen_pixels_mask = np.array([mask_broken_pix for i in range(charge_ma.shape[1])]).T #hist_ma.mask = new_data_mask.T start = time.time() - _mask, hist_ma_data = make_histo(self.charge_hg.T, all_range, mask_broken_pix)#, charge_ma.data, charge_ma.mask, hist_ma.data, hist_ma.mask) + _mask, hist_ma_data = make_histo(chargesContainer[field].T, all_range, mask_broken_pix)#, charge_ma.data, charge_ma.mask, hist_ma.data, hist_ma.mask) charge_ma.mask = np.logical_or(_mask,broxen_pixels_mask) hist_ma = ma.masked_array(hist_ma_data,mask = charge_ma.mask) log.debug(f"histogram hg computation time : {time.time() - start} sec") @@ -286,260 +351,9 @@ def histo_hg(self,n_bins : int = 1000,autoscale : bool = True) -> ma.masked_arra return ma.masked_array((hist_ma,charge_ma)) else : - hist = np.array([np.histogram(self.charge_hg.T[i],bins=n_bins)[0] for i in range(self.charge_hg.shape[1])]) - charge = np.array([np.histogram(self.charge_hg.T[i],bins=n_bins)[1] for i in range(self.charge_hg.shape[1])]) + hist = np.array([np.histogram(chargesContainer[field].T[i],bins=n_bins)[0] for i in range(chargesContainer[field].shape[1])]) + charge = np.array([np.histogram(chargesContainer[field].T[i],bins=n_bins)[1] for i in range(chargesContainer[field].shape[1])]) charge_edges = np.array([np.mean(charge.T[i:i+2],axis = 0) for i in range(charge.shape[1]-1)]).T return np.array((hist,charge_edges)) - def histo_lg(self,n_bins: int = 1000,autoscale : bool = True) -> np.ndarray: - """method to compute histogram of LG channel - Numba is used to compute histograms in vectorized way - - Args: - n_bins (int, optional): number of bins in charge (ADC counts). Defaults to 1000. - autoscale (bool, optional): auto detect number of bins by pixels (bin witdh = 1 ADC). Defaults to True. - - Returns: - np.ndarray: masked array of charge histograms (histo,charge) - """ - mask_broken_pix = np.array((self.charge_lg == self.charge_lg.mean(axis = 0)).mean(axis=0),dtype = bool) - log.debug(f"there are {mask_broken_pix.sum()} broken pixels (charge stays at same level for each events)") - - if autoscale : - all_range = np.arange(np.uint16(np.min(self.charge_lg.T[~mask_broken_pix].T)) - 0.5,np.uint16(np.max(self.charge_lg.T[~mask_broken_pix].T)) + 1.5,1) - charge_ma = ma.masked_array((all_range.reshape(all_range.shape[0],1) @ np.ones((1,self.charge_lg.shape[1]))).T, mask=np.zeros((self.charge_lg.shape[1],all_range.shape[0]),dtype = bool)) - - broxen_pixels_mask = np.array([mask_broken_pix for i in range(charge_ma.shape[1])]).T - #hist_ma.mask = new_data_mask.T - start = time.time() - _mask, hist_ma_data = make_histo(self.charge_lg.T, all_range, mask_broken_pix)#, charge_ma.data, charge_ma.mask, hist_ma.data, hist_ma.mask) - charge_ma.mask = np.logical_or(_mask,broxen_pixels_mask) - hist_ma = ma.masked_array(hist_ma_data,mask = charge_ma.mask) - log.debug(f"histogram lg computation time : {time.time() - start} sec") - - return ma.masked_array((hist_ma,charge_ma)) - - else : - hist = np.array([np.histogram(self.charge_lg.T[i],bins=n_bins)[0] for i in range(self.charge_lg.shape[1])]) - charge = np.array([np.histogram(self.charge_lg.T[i],bins=n_bins)[1] for i in range(self.charge_lg.shape[1])]) - charge_edges = np.array([np.mean(charge.T[i:i+2],axis = 0) for i in range(charge.shape[1]-1)]).T - - return np.array((hist,charge_edges)) - - def select_charge_hg(self,pixel_id : np.ndarray) : - """method to extract charge HG from a list of pixel ids - The output is the charge HG with a shape following the size of the input pixel_id argument - Pixel in pixel_id which are not present in the ChargeContaineur pixels_id are skipped - - Args: - pixel_id (np.ndarray): array of pixel ids you want to extract the charge - - Returns: - (np.ndarray): charge array in the order of specified pixel_id - """ - mask_contain_pixels_id = np.array([pixel in self.pixels_id for pixel in pixel_id],dtype = bool) - for pixel in pixel_id[~mask_contain_pixels_id] : log.warning(f"You asked for pixel_id {pixel} but it is not present in this ChargeContainer, skip this one") - return np.array([self.charge_hg.T[np.where(self.pixels_id == pixel)[0][0]] for pixel in pixel_id[mask_contain_pixels_id]]).T - - def select_charge_lg(self,pixel_id : np.ndarray) : - """method to extract charge LG from a list of pixel ids - The output is the charge LG with a shape following the size of the input pixel_id argument - Pixel in pixel_id which are not present in the ChargeContaineur pixels_id are skipped - - Args: - pixel_id (np.ndarray): array of pixel ids you want to extract the charge - - Returns: - (np.ndarray): charge array in the order of specified pixel_id - """ - mask_contain_pixels_id = np.array([pixel in self.pixels_id for pixel in pixel_id],dtype = bool) - for pixel in pixel_id[~mask_contain_pixels_id] : log.warning(f"You asked for pixel_id {pixel} but it is not present in this ChargeContainer, skip this one") - return np.array([self.charge_lg.T[np.where(self.pixels_id == pixel)[0][0]] for pixel in pixel_id[mask_contain_pixels_id]]).T - - def sort(self, method = 'event_id') : - if method == 'event_id' : - log.info('sorting ChargeContaineur with event_id') - index = np.argsort(self.event_id) - self.ucts_timestamp = self.ucts_timestamp[index] - self.ucts_busy_counter = self.ucts_busy_counter[index] - self.ucts_event_counter = self.ucts_event_counter[index] - self.event_type = self.event_type[index] - self.event_id = self.event_id[index] - self.trig_pattern_all = self.trig_pattern_all[index] - self.charge_hg = self.charge_hg[index] - self.charge_lg = self.charge_lg[index] - self.peak_hg = self.peak_hg[index] - self.peak_lg = self.peak_lg[index] - else : - raise ArgumentError(f"{method} is not a valid method for sorting") - - @property - def run_number(self) : return self._run_number - - @property - def pixels_id(self) : return self._pixels_id - - @property - def npixels(self) : return self._npixels - - @property - def nevents(self) : return self._nevents - - @property - def method(self) : return self._method - - #physical properties - @property - def multiplicity(self) : return np.uint16(np.count_nonzero(self.trig_pattern,axis = 1)) - - @property - def trig_pattern(self) : return self.trig_pattern_all.any(axis = 1) - -''' -class ChargeContainers() : - """ - The `ChargeContainers` class is used to store and manipulate a collection of `ChargeContainer` objects. It provides methods for creating, writing, and merging `ChargeContainer` instances. - - Example Usage: - # Create a `ChargeContainers` instance from a `WaveformsContainers` object - waveform_containers = WaveformsContainers() - charge_containers = ChargeContainers.from_waveforms(waveform_containers) - - # Write the `ChargeContainers` to disk - charge_containers.write("path/to/save") - - # Load `ChargeContainers` from a FITS file - charge_containers = ChargeContainers.from_file("path/to/file", run_number=123) - - # Merge the `ChargeContainers` into a single `ChargeContainer` - merged_charge_container = charge_containers.merge() - - Methods: - - `from_waveforms(waveformContainers: WaveformsContainers, **kwargs)`: Creates a `ChargeContainers` instance from a `WaveformsContainers` object. - - `write(path: str, **kwargs)`: Writes each `ChargeContainer` in `ChargeContainers` to disk. - - `from_file(path: Path, run_number: int, **kwargs)`: Loads `ChargeContainers` from FITS files previously written with `write()` method. - - `append(chargeContainer: ChargeContainer)`: Stacks a `ChargeContainer` into the `ChargeContainers` instance. - - `merge() -> ChargeContainer`: Merges the `ChargeContainers` into a single `ChargeContainer`. - - Fields: - - `chargeContainers`: A list to store `ChargeContainer` objects. - - `__nChargeContainer`: The number of `ChargeContainer` objects stored in `chargeContainers`. - - `nChargeContainer`: A property that returns the number of `ChargeContainer` objects in `chargeContainers`. - - `nevents`: A property that returns the total number of events in all `ChargeContainer` objects in `chargeContainers`. - """ - def __init__(self, *args, **kwargs) : - self.chargeContainers = [] - self.__nChargeContainer = 0 - @classmethod - def from_waveforms(cls,waveformContainers : WaveformsContainers,**kwargs) : - """create ChargeContainers from waveformContainers - - Args: - waveformContainers (WaveformsContainers) - - Returns: - chargeContainers (ChargeContainers) - """ - chargeContainers = cls() - for i in range(waveformContainers.nWaveformsContainer) : - chargeContainers.append(ChargeContainer.from_waveforms(waveformContainers.waveformsContainer[i],**kwargs)) - return chargeContainers - - def write(self,path : str, **kwargs) : - """write each ChargeContainer in ChargeContainers on disk - - Args: - path (str): path where data are saved - """ - for i in range(self.__nChargeContainer) : - self.chargeContainers[i].write(path,suffix = f"{i:04d}" ,**kwargs) - - @staticmethod - def from_file(path : Path,run_number : int,**kwargs) : - """load ChargeContainers from FITS file previously written with ChargeContainers.write() method - This method will search all the fits files corresponding to {path}/charge_run{run_number}_*.fits scheme - - Args: - path (str): path with name of the FITS file without .fits extension - run_number (int) : the run number - - Returns: - ChargeContainers: ChargeContainers instance - """ - log.info(f"loading from {path}/charge_run{run_number}_*.fits") - files = glob.glob(f"{path}/charge_run{run_number}_*.fits") - if len(files) == 0 : - e = FileNotFoundError(f"no files found corresponding to {path}_*.fits") - log.error(e) - raise e - else : - cls = ChargeContainers.__new__(ChargeContainers) - cls.chargeContainers = [] - cls.__nChargeContainer = len(files) - for file in files : - cls.chargeContainers.append(ChargeContainer.from_file(path,run_number,explicit_filename = file)) - return cls - - @property - def nChargeContainer(self) : - """getter giving the number of chargeContainer into the ChargeContainers instance - - Returns: - int: the number of chargeContainer - """ - return self.__nChargeContainer - - @property - def nevents(self) : - """number of events into the whole ChargesContainers - - Returns: - int: number of events - """ - return np.sum([self.chargeContainers[i].nevents for i in range(self.__nChargeContainer)]) - - def append(self,chargeContainer : ChargeContainer) : - """method to stack a ChargeContainer into the ChargeContainers - - Args: - chargeContainer (ChargeContainer): the data to be stacked into - """ - self.chargeContainers.append(chargeContainer) - self.__nChargeContainer += 1 - - - def merge(self) -> ChargeContainer : - """method to merge a ChargeContainers into one single ChargeContainer - - Returns: - ChargeContainer: the merged object - """ - cls = ChargeContainer.__new__(ChargeContainer) - cls.charge_hg = np.concatenate([chargecontainer.charge_hg for chargecontainer in self.chargeContainers],axis = 0) - cls.charge_lg = np.concatenate([chargecontainer.charge_lg for chargecontainer in self.chargeContainers],axis = 0) - cls.peak_hg = np.concatenate([chargecontainer.peak_hg for chargecontainer in self.chargeContainers],axis = 0) - cls.peak_lg = np.concatenate([chargecontainer.peak_lg for chargecontainer in self.chargeContainers],axis = 0) - - if np.all([chargecontainer.run_number == self.chargeContainers[0].run_number for chargecontainer in self.chargeContainers]) : - cls._run_number = self.chargeContainers[0].run_number - if np.all([chargecontainer.pixels_id == self.chargeContainers[0].pixels_id for chargecontainer in self.chargeContainers]) : - cls._pixels_id = self.chargeContainers[0].pixels_id - if np.all([chargecontainer.method == self.chargeContainers[0].method for chargecontainer in self.chargeContainers]): - cls._method = self.chargeContainers[0].method - cls._nevents = np.sum([chargecontainer.nevents for chargecontainer in self.chargeContainers]) - if np.all([chargecontainer.npixels == self.chargeContainers[0].npixels for chargecontainer in self.chargeContainers]): - cls._npixels = self.chargeContainers[0].npixels - - - cls.ucts_timestamp = np.concatenate([chargecontainer.ucts_timestamp for chargecontainer in self.chargeContainers ]) - cls.ucts_busy_counter = np.concatenate([chargecontainer.ucts_busy_counter for chargecontainer in self.chargeContainers ]) - cls.ucts_event_counter = np.concatenate([chargecontainer.ucts_event_counter for chargecontainer in self.chargeContainers ]) - cls.event_type = np.concatenate([chargecontainer.event_type for chargecontainer in self.chargeContainers ]) - cls.event_id = np.concatenate([chargecontainer.event_id for chargecontainer in self.chargeContainers ]) - cls.trig_pattern_all = np.concatenate([chargecontainer.trig_pattern_all for chargecontainer in self.chargeContainers ],axis = 0) - - cls.sort() - - return cls -''' \ No newline at end of file diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 6b53f7e1..98202515 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -11,9 +11,13 @@ from ctapipe.visualization import CameraDisplay from ctapipe.coordinates import CameraFrame,EngineeringCameraFrame from ctapipe.instrument import CameraGeometry,SubarrayDescription,TelescopeDescription +from ctapipe_io_nectarcam import constants + +import tqdm import copy from ..data import DataManagement +from..data.container import ArrayDataContainer __all__ = ["BaseMaker"] @@ -72,7 +76,7 @@ def __init__(self,run_number : int,max_events : int = None,run_file = None,*args log.info(f"N pixels : {self.npixels}") - def make(self,n_events = np.inf, restart_from_begining = False) : + def make(self,n_events = np.inf, restart_from_begining = False,*args,**kwargs) : if restart_from_begining : log.debug('restart from begining : creation of the EventSource reader') self.__reader = __class__.load_run(self.__run_number,self.__max_events,run_file = self.__run_file) @@ -81,7 +85,7 @@ def make(self,n_events = np.inf, restart_from_begining = False) : for i,event in enumerate(self.__reader): if i%100 == 0: log.info(f"reading event number {i}") - self._make_event(event) + self._make_event(event,*args,**kwargs) n_traited_events += 1 if n_traited_events >= n_events : break @@ -149,9 +153,15 @@ def _init_trigger_type(self,trigger,**kwargs) : self.__broken_pixels_hg[f"{name}"] = [] self.__broken_pixels_lg[f"{name}"] = [] - def _compute_broken_pixels(self,wfs_hg_event,wfs_lg_event,**kwargs) : + @staticmethod + def _compute_broken_pixels(wfs_hg_event,wfs_lg_event,**kwargs) : + log.warning("computation of broken pixels is not yet implemented") + return np.zeros((wfs_hg_event.shape[0]),dtype = bool),np.zeros((wfs_hg_event.shape[0]),dtype = bool) + + @staticmethod + def _compute_broken_pixels_event(event : EventType,pixels_id : np.ndarray,**kwargs) : log.warning("computation of broken pixels is not yet implemented") - return np.zeros((self.npixels),dtype = bool),np.zeros((self.npixels),dtype = bool) + return np.zeros((event.r0.tel[0].waveform[constants.HIGH_GAIN][pixels_id]),dtype = bool),np.zeros((event.r0.tel[0].waveform[constants.LOW_GAIN][pixels_id]),dtype = bool) @staticmethod def _get_name_trigger(trigger : EventType) : @@ -161,7 +171,7 @@ def _get_name_trigger(trigger : EventType) : name = trigger.name return name - def make(self,n_events = np.inf, trigger_type : list = None, restart_from_begining = False) : + def make(self,n_events = np.inf, trigger_type : list = None, restart_from_begining = False,*args,**kwargs) : """mathod to extract data from the EventSource Args: @@ -185,15 +195,15 @@ def make(self,n_events = np.inf, trigger_type : list = None, restart_from_begini log.info(f"reading event number {i}") for trigger in trigger_type : if (trigger is None) or (trigger == event.trigger.event_type) : - self._make_event(event,trigger) + self._make_event(event,trigger,*args,**kwargs) n_traited_events += 1 if n_traited_events >= n_events : break - return self._make_output_container() + return self._make_output_container(trigger_type) - def _make_event(self,event,trigger) : + def _make_event(self,event,trigger,*args,**kwargs) : name = __class__._get_name_trigger(trigger) self.__event_id[f'{name}'].append(np.uint16(event.index.event_id)) self.__ucts_timestamp[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_timestamp) @@ -202,39 +212,75 @@ def _make_event(self,event,trigger) : self.__ucts_event_counter[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_event_counter) self.__trig_patter_all[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.trigger_pattern.T) - broken_pixels_hg,broken_pixels_lg = self._compute_broken_pixels() + broken_pixels_hg,broken_pixels_lg = __class__._compute_broken_pixels() self.__broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) self.__broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) + + get_wfs_hg = kwargs.get("wfs_hg",False) + get_wfs_lg = kwargs.get("wfs_lg",False) + + if get_wfs_hg or get_wfs_lg : + for pix in range(self.npixels): + if get_wfs_hg : + get_wfs_hg[pix]=event.r0.tel[0].waveform[constants.HIGH_GAIN,self.pixels_id[pix]] + if get_wfs_lg : + get_wfs_lg[pix]=event.r0.tel[0].waveform[constants.LOW_GAIN,self.pixels_id[pix]] + @abstractmethod def _make_output_container(self) : pass @staticmethod - def create_from_events_list(events_list : list, - run_number : int, - npixels : int, - nsamples : int, - subarray, - pixels_id : int, - ) : - cls = __class__.__new__() - - cls.__run_number = run_number - cls.__nevents = len(events_list) - cls.__npixels = npixels - cls.__nsamples = nsamples - cls.__subarray = subarray - cls.__pixels_id = pixels_id - - cls.__ucts_timestamp = {} - cls.__ucts_busy_counter = {} - cls.__ucts_event_counter = {} - cls.__event_type = {} - cls.__event_id = {} - cls.__trig_patter_all = {} - cls.__broken_pixels_hg = {} - cls.__broken_pixels_lg = {} + def select_container_array_field(container :ArrayDataContainer,pixel_id : np.ndarray,field : str) : + """method to extract waveforms HG from a list of pixel ids + The output is the waveforms HG with a shape following the size of the input pixel_id argument + Pixel in pixel_id which are not present in the WaveformsContaineur pixels_id are skipped + Args: + pixel_id (np.ndarray): array of pixel ids you want to extract the waveforms + Returns: + (np.ndarray): waveforms array in the order of specified pixel_id + """ + mask_contain_pixels_id = np.array([pixel in container.pixels_id for pixel in pixel_id],dtype = bool) + for pixel in pixel_id[~mask_contain_pixels_id] : log.warning(f"You asked for pixel_id {pixel} but it is not present in this container, skip this one") + res = np.array([container[field][:,np.where(container.pixels_id == pixel)[0][0],:] for pixel in pixel_id[mask_contain_pixels_id]]) + ####could be nice to return np.ma.masked_array(data = res, mask = container.broken_pixels_hg.transpose(res.shape[1],res.shape[0],res.shape[2])) + return res + + + + @staticmethod + def merge(container_a : ArrayDataContainer,container_b : ArrayDataContainer) -> ArrayDataContainer : + """method to merge 2 ArrayDataContainer into one single ArrayDataContainer + + Returns: + ArrayDataContainer: the merged object + """ + cls = ChargeContainer.__new__(ChargeContainer) + cls.charge_hg = np.concatenate([chargecontainer.charge_hg for chargecontainer in self.chargeContainers],axis = 0) + cls.charge_lg = np.concatenate([chargecontainer.charge_lg for chargecontainer in self.chargeContainers],axis = 0) + cls.peak_hg = np.concatenate([chargecontainer.peak_hg for chargecontainer in self.chargeContainers],axis = 0) + cls.peak_lg = np.concatenate([chargecontainer.peak_lg for chargecontainer in self.chargeContainers],axis = 0) + + if np.all([chargecontainer.run_number == self.chargeContainers[0].run_number for chargecontainer in self.chargeContainers]) : + cls._run_number = self.chargeContainers[0].run_number + if np.all([chargecontainer.pixels_id == self.chargeContainers[0].pixels_id for chargecontainer in self.chargeContainers]) : + cls._pixels_id = self.chargeContainers[0].pixels_id + if np.all([chargecontainer.method == self.chargeContainers[0].method for chargecontainer in self.chargeContainers]): + cls._method = self.chargeContainers[0].method + cls._nevents = np.sum([chargecontainer.nevents for chargecontainer in self.chargeContainers]) + if np.all([chargecontainer.npixels == self.chargeContainers[0].npixels for chargecontainer in self.chargeContainers]): + cls._npixels = self.chargeContainers[0].npixels + + + cls.ucts_timestamp = np.concatenate([chargecontainer.ucts_timestamp for chargecontainer in self.chargeContainers ]) + cls.ucts_busy_counter = np.concatenate([chargecontainer.ucts_busy_counter for chargecontainer in self.chargeContainers ]) + cls.ucts_event_counter = np.concatenate([chargecontainer.ucts_event_counter for chargecontainer in self.chargeContainers ]) + cls.event_type = np.concatenate([chargecontainer.event_type for chargecontainer in self.chargeContainers ]) + cls.event_id = np.concatenate([chargecontainer.event_id for chargecontainer in self.chargeContainers ]) + cls.trig_pattern_all = np.concatenate([chargecontainer.trig_pattern_all for chargecontainer in self.chargeContainers ],axis = 0) + cls.sort() + return cls diff --git a/src/nectarchain/makers/waveformsMakers.py b/src/nectarchain/makers/waveformsMakers.py index a4776318..82534e5c 100644 --- a/src/nectarchain/makers/waveformsMakers.py +++ b/src/nectarchain/makers/waveformsMakers.py @@ -24,6 +24,8 @@ from ctapipe.instrument import CameraGeometry,SubarrayDescription,TelescopeDescription from ctapipe_io_nectarcam import NectarCAMEventSource +from ctapipe_io_nectarcam import constants + from ..data import DataManagement @@ -63,28 +65,55 @@ def __init__(self,run_number : int,max_events : int = None,run_file = None,*args def create_from_events_list(events_list : list, run_number : int, npixels : int, - nsamples : int, - subarray, + nsamples :int, + subarray : SubarrayDescription, pixels_id : int, ) : - cls = super().create_from_events_list(events_list = events_list, - run_number = run_number, - npixels = npixels, - pixels_id = pixels_id - ) - - cls.__nsamples = nsamples - cls.__subarray = subarray - - cls.__wfs_hg = {} - cls.__wfs_lg = {} - - cls._init_trigger_type(None) - - for event in tqdm(events_list): - cls._make_event(event,None) - - return cls._make_output_container([None]) + container = WaveformsContainer() + container.run_number = run_number + container.npixels = npixels + container.nsamples = nsamples + container.subarray = subarray + container.camera = __class__.CAMERA_NAME + container.pixels_id = pixels_id + + ucts_timestamp = [] + ucts_busy_counter = [] + ucts_event_counter = [] + event_type = [] + event_id = [] + trig_pattern_all = [] + wfs_hg = [] + wfs_lg = [] + + for event in tqdm(events_list) : + ucts_timestamp.append(event.ucts_timestamp) + ucts_busy_counter.append(event.ucts_busy_counter) + ucts_event_counter.append(event.ucts_event_counter) + event_type.append(event.event_type) + event_id.append(event.event_id) + trig_pattern_all.append(event.trig_pattern_all) + broken_pixels = __class__._compute_broken_pixels_event(event,pixels_id) + + wfs_hg.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][pixels_id]) + wfs_lg.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][pixels_id]) + + container.wfs_hg = np.array(wfs_hg,dtype = np.uint16) + container.wfs_lg = np.array(wfs_lg,dtype = np.uint16) + + container.ucts_timestamp = np.array(ucts_timestamp,dtype = np.uint64) + container.ucts_busy_counter = np.array(ucts_busy_counter,dtype = np.uint32) + container.ucts_event_counter = np.array(ucts_event_counter,dtype = np.uint32) + container.event_type = np.array(event_type,dtype = np.uint8) + container.event_id = np.array(event_id,dtype = np.uint32) + container.trig_pattern_all = np.array(trig_pattern_all,dtype =bool ) + container.trig_pattern = container.trig_pattern_all.any(axis = 2) + container.multiplicity = np.uint16(np.count_nonzero(container.trig_pattern,axis = 1)) + + broken_pixels = __class__._compute_broken_pixels() + container.broken_pixels_hg = broken_pixels[0] + container.broken_pixels_lg = broken_pixels[1] + return container def _init_trigger_type(self,trigger_type,**kwargs) : @@ -93,30 +122,27 @@ def _init_trigger_type(self,trigger_type,**kwargs) : log.info(f"initialization of the waveformsMaker following trigger type : {name}") self.__wfs_hg[f"{name}"] = [] self.__wfs_lg[f"{name}"] = [] - - - def _make_event(self, event, - trigger : EventType + trigger : EventType, + *args, + **kwargs ) : - super()._make_event(event = event, - trigger = trigger) - name = __class__._get_name_trigger(trigger) - wfs_hg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) wfs_lg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) - for pix in range(self.npixels): - wfs_lg_tmp[pix]=event.r0.tel[0].waveform[1,self.pixels_id[pix]] - wfs_hg_tmp[pix]=event.r0.tel[0].waveform[0,self.pixels_id[pix]] + super()._make_event(event = event, + trigger = trigger, + wfs_hg = wfs_hg_tmp, + wfs_lg = wfs_lg_tmp) + name = __class__._get_name_trigger(trigger) self.__wfs_hg[f'{name}'].append(wfs_hg_tmp.tolist()) self.__wfs_lg[f'{name}'].append(wfs_lg_tmp.tolist()) - broken_pixels_hg,broken_pixels_lg = self._compute_broken_pixels(wfs_hg_tmp,wfs_lg_tmp) + broken_pixels_hg,broken_pixels_lg = __class__._compute_broken_pixels(wfs_hg_tmp,wfs_lg_tmp) self._broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) self._broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) @@ -148,7 +174,7 @@ def _make_output_container(self,trigger_type) : return output @staticmethod - def sort(waveformsContainer, method = 'event_id') : + def sort(waveformsContainer :WaveformsContainer, method = 'event_id') : output = WaveformsContainer( run_number = waveformsContainer.run_number, npixels = waveformsContainer.npixels, @@ -160,90 +186,34 @@ def sort(waveformsContainer, method = 'event_id') : ) if method == 'event_id' : index = np.argsort(waveformsContainer.event_id) - output.ucts_timestamp = waveformsContainer.ucts_timestamp[index] - output.ucts_busy_counter = waveformsContainer.ucts_busy_counter[index] - output.ucts_event_counter = waveformsContainer.ucts_event_counter[index] - output.event_type = waveformsContainer.event_type[index] - output.event_id = waveformsContainer.event_id[index] - output.trig_pattern_all = waveformsContainer.trig_pattern_all[index] - output.trig_pattern = waveformsContainer.trig_pattern[index] - output.multiplicity = waveformsContainer.multiplicity[index] - - output.wfs_hg = waveformsContainer.wfs_hg[index] - output.wfs_lg = waveformsContainer.wfs_lg[index] - output.broken_pixels_hg = waveformsContainer.broken_pixels_hg[index] - output.broken_pixels_lg = waveformsContainer.broken_pixels_lg[index] + for field in waveformsContainer() : + if not(field in ["run_number","npixels","nsamples","subarray","camera","pixels_id","nevents"]) : + output[field] = waveformsContainer[field][index] + #output.ucts_busy_counter = waveformsContainer.ucts_busy_counter[index] + #output.ucts_event_counter = waveformsContainer.ucts_event_counter[index] + #output.event_type = waveformsContainer.event_type[index] + #output.event_id = waveformsContainer.event_id[index] + #output.trig_pattern_all = waveformsContainer.trig_pattern_all[index] + #output.trig_pattern = waveformsContainer.trig_pattern[index] + #output.multiplicity = waveformsContainer.multiplicity[index] +# + #output.wfs_hg = waveformsContainer.wfs_hg[index] + #output.wfs_lg = waveformsContainer.wfs_lg[index] + #output.broken_pixels_hg = waveformsContainer.broken_pixels_hg[index] + #output.broken_pixels_lg = waveformsContainer.broken_pixels_lg[index] else : raise ArgumentError(f"{method} is not a valid method for sorting") return output - - @staticmethod - ##methods used to display - def display(waveformsContainer,evt,geometry, cmap = 'gnuplot2') : - """plot camera display - - Args: - evt (int): event index - cmap (str, optional): colormap. Defaults to 'gnuplot2'. - - Returns: - CameraDisplay: thoe cameraDisplay plot - """ - image = waveformsContainer.wfs_hg.sum(axis=2) - disp = CameraDisplay(geometry=geometry, image=image[evt], cmap=cmap) - disp.add_colorbar() - return disp - - @staticmethod - def plot_waveform_hg(waveformsContainer,evt,**kwargs) : - """plot the waveform of the evt - - Args: - evt (int): the event index - - Returns: - tuple: the figure and axes - """ - if 'figure' in kwargs.keys() and 'ax' in kwargs.keys() : - fig = kwargs.get('figure') - ax = kwargs.get('ax') - else : - fig,ax = plt.subplots(1,1) - ax.plot(waveformsContainer.wfs_hg[evt].T) - return fig,ax @staticmethod - def select_waveforms_hg(waveformsContainer,pixel_id : np.ndarray) : - """method to extract waveforms HG from a list of pixel ids - The output is the waveforms HG with a shape following the size of the input pixel_id argument - Pixel in pixel_id which are not present in the WaveformsContaineur pixels_id are skipped - Args: - pixel_id (np.ndarray): array of pixel ids you want to extract the waveforms - Returns: - (np.ndarray): waveforms array in the order of specified pixel_id - """ - mask_contain_pixels_id = np.array([pixel in waveformsContainer.pixels_id for pixel in pixel_id],dtype = bool) - for pixel in pixel_id[~mask_contain_pixels_id] : log.warning(f"You asked for pixel_id {pixel} but it is not present in this WaveformsContainer, skip this one") - res = np.array([waveformsContainer.wfs_hg[:,np.where(waveformsContainer.pixels_id == pixel)[0][0],:] for pixel in pixel_id[mask_contain_pixels_id]]) + def select_waveforms_hg(waveformsContainer:WaveformsContainer,pixel_id : np.ndarray) : + res = __class__.select_container_array_field(container = waveformsContainer,pixel_id = pixel_id,field = 'wfs_lg') res = res.transpose(1,0,2) - ####could be nice to return np.ma.masked_array(data = res, mask = waveformsContainer.broken_pixels_hg.transpose(res.shape[1],res.shape[0],res.shape[2])) return res @staticmethod - def select_waveforms_lg(waveformsContainer,pixel_id : np.ndarray) : - """method to extract waveforms LG from a list of pixel ids - The output is the waveforms LG with a shape following the size of the input pixel_id argument - Pixel in pixel_id which are not present in the WaveformsContaineur pixels_id are skipped - - Args: - pixel_id (np.ndarray): array of pixel ids you want to extract the waveforms - - Returns: - (np.ndarray): waveforms array in the order of specified pixel_id - """ - mask_contain_pixels_id = np.array([pixel in waveformsContainer.pixels_id for pixel in pixel_id],dtype = bool) - for pixel in pixel_id[~mask_contain_pixels_id] : log.warning(f"You asked for pixel_id {pixel} but it is not present in this WaveformsContainer, skip this one") - res = np.array([waveformsContainer.wfs_lg[:,np.where(waveformsContainer.pixels_id == pixel)[0][0],:] for pixel in pixel_id[mask_contain_pixels_id]]) + def select_waveforms_lg(waveformsContainer:WaveformsContainer,pixel_id : np.ndarray) : + res = __class__.select_container_array_field(container = waveformsContainer,pixel_id = pixel_id,field = 'wfs_hg') res = res.transpose(1,0,2) return res From f5d0180f3c69201b4267527b3984ba247bf251c6 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 15 Sep 2023 05:02:33 +0200 Subject: [PATCH 49/62] ChargesContainers I/O cleaning unit test for WaveformsMaker and ChargesMaker --- .../data/container/chargesContainer.py | 140 +++++++++--------- src/nectarchain/data/container/core.py | 1 - src/nectarchain/makers/chargesMakers.py | 62 ++++---- src/nectarchain/makers/core.py | 107 ++++++------- src/nectarchain/makers/extractor/__init__.py | 2 +- .../makers/tests/test_chargesMakers.py | 128 ++++++++++++++++ src/nectarchain/makers/tests/test_core.py | 33 +++++ .../makers/tests/test_waveformsMakers.py | 65 ++++++-- src/nectarchain/makers/waveformsMakers.py | 102 +++++-------- 9 files changed, 400 insertions(+), 240 deletions(-) create mode 100644 src/nectarchain/makers/tests/test_chargesMakers.py create mode 100644 src/nectarchain/makers/tests/test_core.py diff --git a/src/nectarchain/data/container/chargesContainer.py b/src/nectarchain/data/container/chargesContainer.py index d555075e..df7fb1af 100644 --- a/src/nectarchain/data/container/chargesContainer.py +++ b/src/nectarchain/data/container/chargesContainer.py @@ -5,6 +5,10 @@ import numpy as np from ctapipe.containers import Field +from abc import ABC +import os +from pathlib import Path +from astropy.io import fits from .core import ArrayDataContainer @@ -22,14 +26,13 @@ class ChargesContainer(ArrayDataContainer): peak_lg = Field( type = np.ndarray, description = 'The low gain peak time') - _method = Field( + method = Field( type = str, - description = 'The charge extraction method') + description = 'The charge extraction method used') -''' - - def write(self,path : Path,**kwargs) : +class ChargesContainerIO(ABC) : + def write(path : Path, containers : ChargesContainer,**kwargs) : """method to write in an output FITS file the ChargeContainer. Args: @@ -41,64 +44,53 @@ def write(self,path : Path,**kwargs) : log.info(f"saving in {path}") os.makedirs(path,exist_ok = True) - #table = Table(self.charge_hg) - #table.meta["pixels_id"] = self._pixels_id - #table.write(Path(path)/f"charge_hg_run{self.run_number}.ecsv",format='ascii.ecsv',overwrite=kwargs.get('overwrite',False)) - # - #table = Table(self.charge_lg) - #table.meta["pixels_id"] = self._pixels_id - #table.write(Path(path)/f"charge_lg_run{self.run_number}.ecsv",format='ascii.ecsv',overwrite=kwargs.get('overwrite',False)) - # - #table = Table(self.peak_hg) - #table.meta["pixels_id"] = self._pixels_id - #table.write(Path(path)/f"peak_hg_run{self.run_number}.ecsv",format='ascii.ecsv',overwrite=kwargs.get('overwrite',False)) - # - #table = Table(self.peak_lg) - #table.meta["pixels_id"] = self._pixels_id - #table.write(Path(path)/f"peak_lg_run{self.run_number}.ecsv",format='ascii.ecsv',overwrite=kwargs.get('overwrite',False)) - hdr = fits.Header() - hdr['RUN'] = self._run_number - hdr['NEVENTS'] = self.nevents - hdr['NPIXELS'] = self.npixels - hdr['COMMENT'] = f"The charge containeur for run {self._run_number} with {self._method} method : primary is the pixels id, then you can find HG charge, LG charge, HG peak and LG peak, 2 last HDU are composed of event properties and trigger patern" - - primary_hdu = fits.PrimaryHDU(self.pixels_id,header=hdr) - charge_hg_hdu = fits.ImageHDU(self.charge_hg,name = "HG charge") - charge_lg_hdu = fits.ImageHDU(self.charge_lg,name = "LG charge") - peak_hg_hdu = fits.ImageHDU(self.peak_hg, name = 'HG peak time') - peak_lg_hdu = fits.ImageHDU(self.peak_lg, name = 'LG peak time') - - col1 = fits.Column(array = self.event_id, name = "event_id", format = '1I') - col2 = fits.Column(array = self.event_type, name = "event_type", format = '1I') - col3 = fits.Column(array = self.ucts_timestamp, name = "ucts_timestamp", format = '1K') - col4 = fits.Column(array = self.ucts_busy_counter, name = "ucts_busy_counter", format = '1I') - col5 = fits.Column(array = self.ucts_event_counter, name = "ucts_event_counter", format = '1I') - col6 = fits.Column(array = self.multiplicity, name = "multiplicity", format = '1I') + hdr['RUN'] = containers.run_number + hdr['NEVENTS'] = containers.nevents + hdr['NPIXELS'] = containers.npixels + hdr['METHOD'] = containers.method + hdr['CAMERA'] = containers.camera + + + hdr['COMMENT'] = f"The charge containeur for run {containers.run_number} with {containers.method} method : primary is the pixels id, then you can find HG charge, LG charge, HG peak and LG peak, 2 last HDU are composed of event properties and trigger patern" + + primary_hdu = fits.PrimaryHDU(containers.pixels_id,header=hdr) + charge_hg_hdu = fits.ImageHDU(containers.charges_hg,name = "HG charge") + charge_lg_hdu = fits.ImageHDU(containers.charges_lg,name = "LG charge") + peak_hg_hdu = fits.ImageHDU(containers.peak_hg, name = 'HG peak time') + peak_lg_hdu = fits.ImageHDU(containers.peak_lg, name = 'LG peak time') + + col1 = fits.Column(array = containers.broken_pixels_hg, name = "HG broken pixels", format = f'{containers.broken_pixels_hg.shape[1]}L') + col2 = fits.Column(array = containers.broken_pixels_lg, name = "LG broken pixels", format = f'{containers.broken_pixels_lg.shape[1]}L') + coldefs = fits.ColDefs([col1, col2]) + broken_pixels = fits.BinTableHDU.from_columns(coldefs,name = 'trigger patern') + + col1 = fits.Column(array = containers.event_id, name = "event_id", format = '1I') + col2 = fits.Column(array = containers.event_type, name = "event_type", format = '1I') + col3 = fits.Column(array = containers.ucts_timestamp, name = "ucts_timestamp", format = '1K') + col4 = fits.Column(array = containers.ucts_busy_counter, name = "ucts_busy_counter", format = '1I') + col5 = fits.Column(array = containers.ucts_event_counter, name = "ucts_event_counter", format = '1I') + col6 = fits.Column(array = containers.multiplicity, name = "multiplicity", format = '1I') coldefs = fits.ColDefs([col1, col2, col3, col4, col5, col6]) event_properties = fits.BinTableHDU.from_columns(coldefs, name = 'event properties') - col1 = fits.Column(array = self.trig_pattern_all, name = "trig_pattern_all", format = f'{4 * self.CAMERA.n_pixels}L',dim = f'({self.CAMERA.n_pixels},4)') - col2 = fits.Column(array = self.trig_pattern, name = "trig_pattern", format = f'{self.CAMERA.n_pixels}L') + col1 = fits.Column(array = containers.trig_pattern_all, name = "trig_pattern_all", format = f'{4 * containers.trig_pattern_all.shape[1]}L',dim = f'({ containers.trig_pattern_all.shape[1]},4)') + col2 = fits.Column(array = containers.trig_pattern, name = "trig_pattern", format = f'{containers.trig_pattern_all.shape[1]}L') coldefs = fits.ColDefs([col1, col2]) trigger_patern = fits.BinTableHDU.from_columns(coldefs, name = 'trigger patern') - hdul = fits.HDUList([primary_hdu, charge_hg_hdu, charge_lg_hdu,peak_hg_hdu,peak_lg_hdu,event_properties,trigger_patern]) + hdul = fits.HDUList([primary_hdu, charge_hg_hdu, charge_lg_hdu,peak_hg_hdu,peak_lg_hdu,broken_pixels,event_properties,trigger_patern]) try : - hdul.writeto(Path(path)/f"charge_run{self.run_number}{suffix}.fits",overwrite=kwargs.get('overwrite',False)) - log.info(f"charge saved in {Path(path)}/charge_run{self.run_number}{suffix}.fits") + hdul.writeto(Path(path)/f"charge_run{containers.run_number}{suffix}.fits",overwrite=kwargs.get('overwrite',False)) + log.info(f"charge saved in {Path(path)}/charge_run{containers.run_number}{suffix}.fits") except OSError as e : log.warning(e) except Exception as e : log.error(e,exc_info = True) raise e - - - - @staticmethod - def from_file(path : Path,run_number : int,**kwargs) : + def load(path : Path,run_number : int,**kwargs) : """load ChargeContainer from FITS file previously written with ChargeContainer.write() method Args: @@ -116,24 +108,34 @@ def from_file(path : Path,run_number : int,**kwargs) : filename = Path(path)/f"charge_run{run_number}.fits" with fits.open(filename) as hdul : - pixels_id = hdul[0].data - nevents = hdul[0].header['NEVENTS'] - npixels = hdul[0].header['NPIXELS'] - charge_hg = hdul[1].data - charge_lg = hdul[2].data - peak_hg = hdul[3].data - peak_lg = hdul[4].data - - cls = ChargeContainer(charge_hg,charge_lg,peak_hg,peak_lg,run_number,pixels_id,nevents,npixels) - - cls.event_id = hdul[5].data["event_id"] - cls.event_type = hdul[5].data["event_type"] - cls.ucts_timestamp = hdul[5].data["ucts_timestamp"] - cls.ucts_busy_counter = hdul[5].data["ucts_busy_counter"] - cls.ucts_event_counter = hdul[5].data["ucts_event_counter"] - - table_trigger = hdul[6].data - cls.trig_pattern_all = table_trigger["trig_pattern_all"] - - return cls -''' \ No newline at end of file + containers = ChargesContainer() + containers.run_number = hdul[0].header['RUN'] + containers.nevents = hdul[0].header['NEVENTS'] + containers.npixels = hdul[0].header['NPIXELS'] + containers.method = hdul[0].header['METHOD'] + containers.camera = hdul[0].header['CAMERA'] + + containers.pixels_id = hdul[0].data + containers.charges_hg = hdul[1].data + containers.charges_lg = hdul[2].data + containers.peak_hg = hdul[3].data + containers.peak_lg = hdul[4].data + + broken_pixels = hdul[5].data + containers.broken_pixels_hg = broken_pixels["HG broken pixels"] + containers.broken_pixels_lg = broken_pixels["LG broken pixels"] + + + + table_prop = hdul[6].data + containers.event_id = table_prop["event_id"] + containers.event_type = table_prop["event_type"] + containers.ucts_timestamp = table_prop["ucts_timestamp"] + containers.ucts_busy_counter = table_prop["ucts_busy_counter"] + containers.ucts_event_counter = table_prop["ucts_event_counter"] + containers.multiplicity = table_prop["multiplicity"] + + table_trigger = hdul[7].data + containers.trig_pattern_all = table_trigger["trig_pattern_all"] + containers.trig_pattern = table_trigger["trig_pattern"] + return containers diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index fa9b00c0..6471b3a6 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -4,7 +4,6 @@ log = logging.getLogger(__name__) log.handlers = logging.getLogger('__main__').handlers -import os from ctapipe.containers import Container,Field import numpy as np diff --git a/src/nectarchain/makers/chargesMakers.py b/src/nectarchain/makers/chargesMakers.py index 294a9000..73557826 100644 --- a/src/nectarchain/makers/chargesMakers.py +++ b/src/nectarchain/makers/chargesMakers.py @@ -6,19 +6,19 @@ from argparse import ArgumentError import numpy as np import numpy.ma as ma -from pathlib import Path -import glob import time -import os - -from ctapipe.instrument import CameraGeometry from ctapipe_io_nectarcam import constants from ctapipe.containers import EventType -from ctapipe.image import ImageExtractor - +from ctapipe.image.extractor import (FullWaveformSum, + FixedWindowSum, + GlobalPeakWindowSum, + LocalPeakWindowSum, + SlidingWindowMaxSum, + NeighborPeakWindowSum, + BaselineSubtractedNeighborPeakWindowSum, + TwoPassWindowSum) -from astropy.io import fits from numba import guvectorize, float64, int64, bool_ @@ -28,9 +28,7 @@ from .extractor.utils import CtapipeExtractor - - -__all__ = ['ChargeContainer'] +__all__ = ['ChargesMaker'] list_ctapipe_charge_extractor = ["FullWaveformSum", "FixedWindowSum", @@ -98,7 +96,7 @@ def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): pass -class ChargeMaker(ArrayDataMaker) : +class ChargesMaker(ArrayDataMaker) : #constructors def __init__(self,run_number : int,max_events : int = None,run_file = None,*args,**kwargs): """construtor @@ -118,7 +116,7 @@ def __init__(self,run_number : int,max_events : int = None,run_file = None,*args def _init_trigger_type(self,trigger_type,**kwargs) : super()._init_trigger_type(trigger_type,**kwargs) name = __class__._get_name_trigger(trigger_type) - log.info(f"initialization of the chargeMaker following trigger type : {name}") + log.info(f"initialization of the ChargesMaker following trigger type : {name}") self.__charges_hg[f"{name}"] = [] self.__charges_lg[f"{name}"] = [] self.__peak_hg[f"{name}"] = [] @@ -132,7 +130,7 @@ def make(self, method: str = "FullWaveformSum", *args,**kwargs): kwargs["method"]=method - super().make(n_events=n_events, + return super().make(n_events=n_events, trigger_type=trigger_type, restart_from_begining=restart_from_begining, *args,**kwargs) @@ -144,18 +142,14 @@ def _make_event(self, *args, **kwargs ) : - wfs_hg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) - wfs_lg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) - - super()._make_event(event = event, - trigger = trigger, - wfs_hg = wfs_hg_tmp, - wfs_lg = wfs_lg_tmp, - *args, - **kwargs) + + wfs_hg_tmp,wfs_lg_tmp = super()._make_event(event = event, + trigger = trigger, + return_wfs = True, + *args,**kwargs) name = __class__._get_name_trigger(trigger) - broken_pixels_hg,broken_pixels_lg = __class__._compute_broken_pixels(wfs_hg_tmp,wfs_lg_tmp) + broken_pixels_hg,broken_pixels_lg = __class__._compute_broken_pixels_event(event,self._pixels_id) self._broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) self._broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) @@ -187,14 +181,15 @@ def _get_imageExtractor(method,subarray,**kwargs) : imageExtractor = eval(method)(subarray,**extractor_kwargs) return imageExtractor - def _make_output_container(self,trigger_type) : + def _make_output_container(self,trigger_type,method : str) : output = [] for trigger in trigger_type : chargesContainer = ChargesContainer( - run_number = self.run_number, - npixels = self.npixels, + run_number = self._run_number, + npixels = self._npixels, camera = self.CAMERA_NAME, - pixels_id = self.pixels_id, + pixels_id = self._pixels_id, + method = method, nevents = self.nevents(trigger), charges_hg = self.charges_hg(trigger), charges_lg = self.charges_lg(trigger), @@ -221,7 +216,9 @@ def sort(chargesContainer :ChargesContainer, method = 'event_id') : npixels = chargesContainer.npixels, camera = chargesContainer.camera, pixels_id = chargesContainer.pixels_id, - nevents = chargesContainer.nevents + nevents = chargesContainer.nevents, + method = chargesContainer.method + ) if method == 'event_id' : index = np.argsort(chargesContainer.event_id) @@ -261,7 +258,7 @@ def create_from_waveforms(waveformsContainer: WaveformsContainer, method: str = for field in waveformsContainer.keys() : if not(field in ["subarray","nsamples","wfs_hg","wfs_lg"]) : chargesContainer[field] = waveformsContainer[field] - + log.info(f"computing hg charge with {method} method") charges_hg, peak_hg = __class__.compute_charge(waveformsContainer, constants.HIGH_GAIN, method, **kwargs) charges_hg = np.array(charges_hg, dtype=np.uint16) @@ -272,6 +269,7 @@ def create_from_waveforms(waveformsContainer: WaveformsContainer, method: str = chargesContainer.charges_lg = charges_lg chargesContainer.peak_hg = peak_hg chargesContainer.peak_lg = peak_lg + chargesContainer.method = method return chargesContainer @@ -298,10 +296,10 @@ def compute_charge(waveformContainer : WaveformsContainer,channel : int,method : imageExtractor = __class__._get_imageExtractor(method = method,subarray = waveformContainer.subarray,**kwargs) if channel == constants.HIGH_GAIN: - out = np.array([CtapipeExtractor.get_image_peak_time(imageExtractor(waveformContainer.wfs_hg[i],waveformContainer.TEL_ID,channel,waveformContainer.broken_pixels_hg)) for i in range(len(waveformContainer.wfs_hg))]).transpose(1,0,2) + out = np.array([CtapipeExtractor.get_image_peak_time(imageExtractor(waveformContainer.wfs_hg[i],__class__.TEL_ID,channel,waveformContainer.broken_pixels_hg)) for i in range(len(waveformContainer.wfs_hg))]).transpose(1,0,2) return out[0],out[1] elif channel == constants.LOW_GAIN: - out = np.array([CtapipeExtractor.get_image_peak_time(imageExtractor(waveformContainer.wfs_lg[i],waveformContainer.TEL_ID,channel,waveformContainer.broken_pixels_lg)) for i in range(len(waveformContainer.wfs_lg))]).transpose(1,0,2) + out = np.array([CtapipeExtractor.get_image_peak_time(imageExtractor(waveformContainer.wfs_lg[i],__class__.TEL_ID,channel,waveformContainer.broken_pixels_lg)) for i in range(len(waveformContainer.wfs_lg))]).transpose(1,0,2) return out[0],out[1] else : raise ArgumentError(f"channel must be {constants.LOW_GAIN} or {constants.HIGH_GAIN}") diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 98202515..46cdf553 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -7,17 +7,16 @@ from ctapipe_io_nectarcam import NectarCAMEventSource import numpy as np +import copy + from ctapipe.containers import EventType -from ctapipe.visualization import CameraDisplay -from ctapipe.coordinates import CameraFrame,EngineeringCameraFrame -from ctapipe.instrument import CameraGeometry,SubarrayDescription,TelescopeDescription +from ctapipe.instrument import CameraGeometry from ctapipe_io_nectarcam import constants -import tqdm -import copy + from ..data import DataManagement -from..data.container import ArrayDataContainer +from ..data.container.core import ArrayDataContainer __all__ = ["BaseMaker"] @@ -100,6 +99,12 @@ def _run_file(self) : return self.__run_file def _max_events(self) : return self.__max_events @property def _reader(self) : return self.__reader + @_reader.setter + def _reader(self,value) : + if isinstance(value,NectarCAMEventSource) : + self.__reader = value + else : + raise TypeError("The reader must be a NectarCAMEventSource") @property def _npixels(self) : return self.__npixels @property @@ -131,6 +136,7 @@ def __init__(self,run_number : int,max_events : int = None,run_file = None,*args run_file (optional) : if provided, will load this run file """ super().__init__(run_number,max_events,run_file,*args,**kwargs) + self.__nsamples = self._reader.camera_config.num_samples #data we want to compute self.__ucts_timestamp = {} @@ -154,14 +160,14 @@ def _init_trigger_type(self,trigger,**kwargs) : self.__broken_pixels_lg[f"{name}"] = [] @staticmethod - def _compute_broken_pixels(wfs_hg_event,wfs_lg_event,**kwargs) : + def _compute_broken_pixels(wfs_hg,wfs_lg,**kwargs) : log.warning("computation of broken pixels is not yet implemented") - return np.zeros((wfs_hg_event.shape[0]),dtype = bool),np.zeros((wfs_hg_event.shape[0]),dtype = bool) + return np.zeros((wfs_hg.shape[:-1]),dtype = bool),np.zeros((wfs_hg.shape[:-1]),dtype = bool) @staticmethod - def _compute_broken_pixels_event(event : EventType,pixels_id : np.ndarray,**kwargs) : + def _compute_broken_pixels_event(event : EventType,pixels_id,**kwargs) : log.warning("computation of broken pixels is not yet implemented") - return np.zeros((event.r0.tel[0].waveform[constants.HIGH_GAIN][pixels_id]),dtype = bool),np.zeros((event.r0.tel[0].waveform[constants.LOW_GAIN][pixels_id]),dtype = bool) + return np.zeros((len(pixels_id)),dtype = bool),np.zeros((len(pixels_id)),dtype = bool) @staticmethod def _get_name_trigger(trigger : EventType) : @@ -187,10 +193,10 @@ def make(self,n_events = np.inf, trigger_type : list = None, restart_from_begini if restart_from_begining : log.debug('restart from begining : creation of the EventSource reader') - self.__reader = __class__.load_run(self.__run_number,self.__max_events,run_file = self.__run_file) + self._reader = __class__.load_run(self._run_number,self._max_events,run_file = self._run_file) n_traited_events = 0 - for i,event in enumerate(self.__reader): + for i,event in enumerate(self._reader): if i%100 == 0: log.info(f"reading event number {i}") for trigger in trigger_type : @@ -200,7 +206,7 @@ def make(self,n_events = np.inf, trigger_type : list = None, restart_from_begini if n_traited_events >= n_events : break - return self._make_output_container(trigger_type) + return self._make_output_container(trigger_type,*args,**kwargs) def _make_event(self,event,trigger,*args,**kwargs) : @@ -211,20 +217,11 @@ def _make_event(self,event,trigger,*args,**kwargs) : self.__ucts_busy_counter[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_busy_counter) self.__ucts_event_counter[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_event_counter) self.__trig_patter_all[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.trigger_pattern.T) - - broken_pixels_hg,broken_pixels_lg = __class__._compute_broken_pixels() - self.__broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) - self.__broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) - get_wfs_hg = kwargs.get("wfs_hg",False) - get_wfs_lg = kwargs.get("wfs_lg",False) - - if get_wfs_hg or get_wfs_lg : - for pix in range(self.npixels): - if get_wfs_hg : - get_wfs_hg[pix]=event.r0.tel[0].waveform[constants.HIGH_GAIN,self.pixels_id[pix]] - if get_wfs_lg : - get_wfs_lg[pix]=event.r0.tel[0].waveform[constants.LOW_GAIN,self.pixels_id[pix]] + if kwargs.get("return_wfs",False) : + get_wfs_hg=event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id] + get_wfs_lg=event.r0.tel[0].waveform[constants.LOW_GAIN][self.pixels_id] + return get_wfs_hg,get_wfs_lg @abstractmethod @@ -232,17 +229,9 @@ def _make_output_container(self) : pass @staticmethod def select_container_array_field(container :ArrayDataContainer,pixel_id : np.ndarray,field : str) : - """method to extract waveforms HG from a list of pixel ids - The output is the waveforms HG with a shape following the size of the input pixel_id argument - Pixel in pixel_id which are not present in the WaveformsContaineur pixels_id are skipped - Args: - pixel_id (np.ndarray): array of pixel ids you want to extract the waveforms - Returns: - (np.ndarray): waveforms array in the order of specified pixel_id - """ mask_contain_pixels_id = np.array([pixel in container.pixels_id for pixel in pixel_id],dtype = bool) for pixel in pixel_id[~mask_contain_pixels_id] : log.warning(f"You asked for pixel_id {pixel} but it is not present in this container, skip this one") - res = np.array([container[field][:,np.where(container.pixels_id == pixel)[0][0],:] for pixel in pixel_id[mask_contain_pixels_id]]) + res = np.array([np.take(container[field],np.where(container.pixels_id == pixel)[0][0],axis = 1) for pixel in pixel_id[mask_contain_pixels_id]]) ####could be nice to return np.ma.masked_array(data = res, mask = container.broken_pixels_hg.transpose(res.shape[1],res.shape[0],res.shape[2])) return res @@ -255,35 +244,33 @@ def merge(container_a : ArrayDataContainer,container_b : ArrayDataContainer) -> Returns: ArrayDataContainer: the merged object """ - cls = ChargeContainer.__new__(ChargeContainer) - cls.charge_hg = np.concatenate([chargecontainer.charge_hg for chargecontainer in self.chargeContainers],axis = 0) - cls.charge_lg = np.concatenate([chargecontainer.charge_lg for chargecontainer in self.chargeContainers],axis = 0) - cls.peak_hg = np.concatenate([chargecontainer.peak_hg for chargecontainer in self.chargeContainers],axis = 0) - cls.peak_lg = np.concatenate([chargecontainer.peak_lg for chargecontainer in self.chargeContainers],axis = 0) - - if np.all([chargecontainer.run_number == self.chargeContainers[0].run_number for chargecontainer in self.chargeContainers]) : - cls._run_number = self.chargeContainers[0].run_number - if np.all([chargecontainer.pixels_id == self.chargeContainers[0].pixels_id for chargecontainer in self.chargeContainers]) : - cls._pixels_id = self.chargeContainers[0].pixels_id - if np.all([chargecontainer.method == self.chargeContainers[0].method for chargecontainer in self.chargeContainers]): - cls._method = self.chargeContainers[0].method - cls._nevents = np.sum([chargecontainer.nevents for chargecontainer in self.chargeContainers]) - if np.all([chargecontainer.npixels == self.chargeContainers[0].npixels for chargecontainer in self.chargeContainers]): - cls._npixels = self.chargeContainers[0].npixels - - - cls.ucts_timestamp = np.concatenate([chargecontainer.ucts_timestamp for chargecontainer in self.chargeContainers ]) - cls.ucts_busy_counter = np.concatenate([chargecontainer.ucts_busy_counter for chargecontainer in self.chargeContainers ]) - cls.ucts_event_counter = np.concatenate([chargecontainer.ucts_event_counter for chargecontainer in self.chargeContainers ]) - cls.event_type = np.concatenate([chargecontainer.event_type for chargecontainer in self.chargeContainers ]) - cls.event_id = np.concatenate([chargecontainer.event_id for chargecontainer in self.chargeContainers ]) - cls.trig_pattern_all = np.concatenate([chargecontainer.trig_pattern_all for chargecontainer in self.chargeContainers ],axis = 0) + if type(container_a) != type(container_b) : + raise Exception("The containers have to be instnace of the same class") + + if np.array_equal(container_a.pixels_id,container_b.pixels_id) : + raise Exception("The containers have not the same pixels ids") - cls.sort() + merged_container = container_a.__class__.__new__() - return cls + for field in container_a.keys() : + if ~isinstance(container_a[field],np.ndarray) : + if container_a[field] != container_b[field] : + raise Exception(f"merge impossible because of {field} filed (values are {container_a[field]} and {container_b[field]}") + + for field in container_a.keys() : + if isinstance(container_a[field],np.ndarray) : + merged_container[field] = np.concatenate(container_a[field],container_a[field],axis = 0) + else : + merged_container[field] = container_a[field] + return merged_container + + + @property + def nsamples(self) : return copy.deepcopy(self.__nsamples) + @property + def _nsamples(self) : return self.__nsamples def nevents(self,trigger) : return len(self.__event_id[__class__._get_name_trigger(trigger)]) @property def _broken_pixels_hg(self) : return self.__broken_pixels_hg diff --git a/src/nectarchain/makers/extractor/__init__.py b/src/nectarchain/makers/extractor/__init__.py index b8e298f9..6a54cdfe 100644 --- a/src/nectarchain/makers/extractor/__init__.py +++ b/src/nectarchain/makers/extractor/__init__.py @@ -1 +1 @@ -from .charge_extractor import * \ No newline at end of file +#from .charge_extractor import * \ No newline at end of file diff --git a/src/nectarchain/makers/tests/test_chargesMakers.py b/src/nectarchain/makers/tests/test_chargesMakers.py new file mode 100644 index 00000000..a83b9e52 --- /dev/null +++ b/src/nectarchain/makers/tests/test_chargesMakers.py @@ -0,0 +1,128 @@ +from nectarchain.makers import ChargesMaker,WaveformsMaker +from nectarchain.data.container import ChargesContainer,ChargesContainerIO +from ctapipe.containers import EventType +import numpy as np +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level = logging.DEBUG) +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +class TestChargesMaker: + run_number = 3938 + max_events = 100 + + def test_instance(self) : + chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, + max_events = TestChargesMaker.max_events) + assert isinstance(chargesMaker,ChargesMaker) + + def test_shape_valid(self): + chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, + max_events = TestChargesMaker.max_events) + chargesContainer = chargesMaker.make()[0] + + assert chargesContainer.nevents <= TestChargesMaker.max_events + assert chargesContainer.run_number == TestChargesMaker.run_number + assert chargesContainer.ucts_timestamp.shape == (chargesContainer.nevents,) + assert chargesContainer.ucts_busy_counter.shape == (chargesContainer.nevents,) + assert chargesContainer.ucts_event_counter.shape == (chargesContainer.nevents,) + assert chargesContainer.event_type.shape == (chargesContainer.nevents,) + assert chargesContainer.event_id.shape == (chargesContainer.nevents,) + assert chargesContainer.trig_pattern_all.shape[0] == chargesContainer.nevents + assert chargesContainer.trig_pattern_all.shape[2] == 4 + + assert chargesContainer.trig_pattern.shape[0] == chargesContainer.nevents + assert chargesContainer.multiplicity.shape == (chargesContainer.nevents,) + + assert chargesContainer.charges_hg.mean() != 0 + assert chargesContainer.charges_lg.mean() != 0 + assert chargesContainer.peak_hg.mean() != 0 + assert chargesContainer.peak_lg.mean() != 0 + assert chargesContainer.charges_hg.shape == (chargesContainer.nevents,chargesContainer.npixels) + assert chargesContainer.charges_lg.shape == (chargesContainer.nevents,chargesContainer.npixels) + assert chargesContainer.peak_hg.shape == (chargesContainer.nevents,chargesContainer.npixels) + assert chargesContainer.peak_lg.shape == (chargesContainer.nevents,chargesContainer.npixels) + + assert chargesContainer.broken_pixels_hg.shape == (chargesContainer.nevents,chargesContainer.npixels) + assert chargesContainer.broken_pixels_lg.shape == (chargesContainer.nevents,chargesContainer.npixels) + + def test_make_restart_eventsource(self) : + chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, + max_events = TestChargesMaker.max_events) + chargesContainer_list = chargesMaker.make(restart_from_begining = True) + assert isinstance(chargesContainer_list[0],ChargesContainer) + + def test_make_LocalPeakWindowSum(self) : + chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, + max_events = TestChargesMaker.max_events) + chargesContainer_list = chargesMaker.make(method = "LocalPeakWindowSum",window_shift = -4, window_length = 16) + assert isinstance(chargesContainer_list[0],ChargesContainer) + + def test_all_multiple_trigger(self) : + trigger1 = EventType.FLATFIELD + trigger2 = EventType.SKY_PEDESTAL + chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, + max_events = TestChargesMaker.max_events) + chargesContainer_list = chargesMaker.make(trigger_type = [trigger1,trigger2]) + for chargesContainer in chargesContainer_list : + assert isinstance(chargesContainer,ChargesContainer) + + + def test_all_trigger_None(self) : + chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, + max_events = TestChargesMaker.max_events) + chargesContainer_list = chargesMaker.make() + assert isinstance(chargesContainer_list[0],ChargesContainer) + + def test_create_from_waveforms(self) : + waveformsMaker = WaveformsMaker(run_number = TestChargesMaker.run_number, + max_events = TestChargesMaker.max_events) + waveformsContainer_list = waveformsMaker.make() + chargesContainer = ChargesMaker.create_from_waveforms(waveformsContainer_list[0],method = "LocalPeakWindowSum",window_shift = -4, window_length = 16) + assert isinstance(chargesContainer,ChargesContainer) + + def test_select_charges(self) : + chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, + max_events = TestChargesMaker.max_events) + chargesContainer_list = chargesMaker.make() + pixel_id = np.array([3,67,87]) + assert isinstance(ChargesMaker.select_charges_hg(chargesContainer_list[0],pixel_id),np.ndarray) + assert isinstance(ChargesMaker.select_charges_lg(chargesContainer_list[0],pixel_id),np.ndarray) + + def test_histo(self) : + chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, + max_events = TestChargesMaker.max_events) + chargesContainer_list = chargesMaker.make() + histo = ChargesMaker.histo_hg(chargesContainer_list[0]) + assert isinstance(histo,np.ndarray) + assert histo.mean() != 0 + assert histo.shape[0] == 2 + assert histo.shape[1] == chargesContainer_list[0].npixels + histo = ChargesMaker.histo_lg(chargesContainer_list[0]) + assert isinstance(histo,np.ndarray) + assert histo.mean() != 0 + assert histo.shape[0] == 2 + assert histo.shape[1] == chargesContainer_list[0].npixels + + + def test_sort_ChargesContainer(self) : + chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, + max_events = TestChargesMaker.max_events) + chargesContainer_list = chargesMaker.make() + sortWfs = ChargesMaker.sort(chargesContainer_list[0],method = 'event_id') + assert np.array_equal(sortWfs.event_id ,np.sort(chargesContainer_list[0].event_id)) + + def test_write_load_container(self) : + chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, + max_events = TestChargesMaker.max_events) + chargesContainer_list = chargesMaker.make() + ChargesContainerIO.write("/tmp/test_charge_container/",chargesContainer_list[0],overwrite = True) + loaded_charge = ChargesContainerIO.load(f"/tmp/test_charge_container/",run_number = TestChargesMaker.run_number) + assert np.array_equal(chargesContainer_list[0].charges_hg,loaded_charge.charges_hg) + +if __name__ == '__main__' : + import logging + logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level = logging.DEBUG) + log = logging.getLogger(__name__) + log.handlers = logging.getLogger('__main__').handlers + TestChargesMaker().test_write_load_container() \ No newline at end of file diff --git a/src/nectarchain/makers/tests/test_core.py b/src/nectarchain/makers/tests/test_core.py new file mode 100644 index 00000000..242dabef --- /dev/null +++ b/src/nectarchain/makers/tests/test_core.py @@ -0,0 +1,33 @@ +from nectarchain.makers import ChargesMaker,WaveformsMaker +from nectarchain.data.container import ChargesContainer +from ..core import ArrayDataMaker +import logging +logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level = logging.DEBUG) +log = logging.getLogger(__name__) +log.handlers = logging.getLogger('__main__').handlers + +class TestArrayDataMaker: + run_number = 3938 + max_events = 100 + + def test_merge(self) : + chargesMaker = ChargesMaker(run_number = TestArrayDataMaker.run_number, + max_events = TestArrayDataMaker.max_events) + charges_1 = chargesMaker.make() + chargesMaker_2 = ChargesMaker(run_number = TestArrayDataMaker.run_number, + max_events = TestArrayDataMaker.max_events) + charges_2 = chargesMaker_2.make() + + merged = ArrayDataMaker.merge(charges_1,charges_2) + assert isinstance(merged,ChargesContainer) + + def test_merge_different_container(self) : + chargesMaker = ChargesMaker(run_number = TestArrayDataMaker.run_number, + max_events = TestArrayDataMaker.max_events) + charges_1 = chargesMaker.make() + wfsMaker_2 = WaveformsMaker(run_number = TestArrayDataMaker.run_number, + max_events = TestArrayDataMaker.max_events) + wfs_2 = wfsMaker_2.make() + merged = ArrayDataMaker.merge(charges_1,wfs_2) + + \ No newline at end of file diff --git a/src/nectarchain/makers/tests/test_waveformsMakers.py b/src/nectarchain/makers/tests/test_waveformsMakers.py index a6f9036c..06a222eb 100644 --- a/src/nectarchain/makers/tests/test_waveformsMakers.py +++ b/src/nectarchain/makers/tests/test_waveformsMakers.py @@ -1,5 +1,5 @@ from nectarchain.makers.waveformsMakers import WaveformsMaker -from nectarchain.data.container import WaveformsContainer,WaveformsContainerIO +from nectarchain.data.container import WaveformsContainer,WaveformsContainerIO,ArrayDataContainer from ctapipe.containers import EventType import numpy as np import logging @@ -9,12 +9,39 @@ class TestWaveformsMaker: run_number = 3938 - max_events = 1000 + max_events = 100 def test_instance(self) : waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, max_events = TestWaveformsMaker.max_events) assert isinstance(waveformsMaker,WaveformsMaker) + + def test_shape_valid(self): + waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, + max_events = TestWaveformsMaker.max_events) + waveformsContainer = waveformsMaker.make()[0] + + assert waveformsContainer.nevents <= TestWaveformsMaker.max_events + assert waveformsContainer.run_number == TestWaveformsMaker.run_number + assert waveformsContainer.ucts_timestamp.shape == (waveformsContainer.nevents,) + assert waveformsContainer.ucts_busy_counter.shape == (waveformsContainer.nevents,) + assert waveformsContainer.ucts_event_counter.shape == (waveformsContainer.nevents,) + assert waveformsContainer.event_type.shape == (waveformsContainer.nevents,) + assert waveformsContainer.event_id.shape == (waveformsContainer.nevents,) + assert waveformsContainer.trig_pattern_all.shape[0] == waveformsContainer.nevents + assert waveformsContainer.trig_pattern_all.shape[2] == 4 + + assert waveformsContainer.trig_pattern.shape[0] == waveformsContainer.nevents + assert waveformsContainer.multiplicity.shape == (waveformsContainer.nevents,) + + assert waveformsContainer.wfs_hg.mean() != 0 + assert waveformsContainer.wfs_lg.mean() != 0 + assert waveformsContainer.wfs_hg.shape == (waveformsContainer.nevents,waveformsContainer.npixels,waveformsContainer.nsamples) + assert waveformsContainer.wfs_lg.shape == (waveformsContainer.nevents,waveformsContainer.npixels,waveformsContainer.nsamples) + assert waveformsContainer.broken_pixels_hg.shape == (waveformsContainer.nevents,waveformsContainer.npixels) + assert waveformsContainer.broken_pixels_lg.shape == (waveformsContainer.nevents,waveformsContainer.npixels) + + def test_all_multiple_trigger(self) : trigger1 = EventType.FLATFIELD @@ -24,6 +51,7 @@ def test_all_multiple_trigger(self) : waveformsContainer_list = waveformsMaker.make(trigger_type = [trigger1,trigger2],restart_from_begining = True) for waveformsContainer in waveformsContainer_list : assert isinstance(waveformsContainer,WaveformsContainer) + assert waveformsContainer.wfs_hg.mean() != 0 def test_all_trigger_None(self) : @@ -37,7 +65,9 @@ def test_select_waveforms_hg(self) : max_events = TestWaveformsMaker.max_events) waveformsContainer_list = waveformsMaker.make() pixel_id = np.array([3,67,87]) - WaveformsMaker.select_waveforms_hg(waveformsContainer_list[0],pixel_id) + assert isinstance(WaveformsMaker.select_waveforms_hg(waveformsContainer_list[0],pixel_id),np.ndarray) + assert isinstance(WaveformsMaker.select_waveforms_lg(waveformsContainer_list[0],pixel_id),np.ndarray) + def test_sort_WaveformsContainer(self) : waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, @@ -47,15 +77,30 @@ def test_sort_WaveformsContainer(self) : assert np.array_equal(sortWfs.event_id ,np.sort(waveformsContainer_list[0].event_id)) def test_write_load_container(self) : - waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, - max_events = TestWaveformsMaker.max_events) - waveformsContainer_list = waveformsMaker.make() - WaveformsContainerIO.write("/tmp/test_wfs_container/",waveformsContainer_list[0],overwrite = True) - loaded_wfs = WaveformsContainerIO.load(f"/tmp/test_wfs_container/waveforms_run{TestWaveformsMaker.run_number}.fits") + waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, + max_events = TestWaveformsMaker.max_events) + waveformsContainer_list = waveformsMaker.make() + WaveformsContainerIO.write("/tmp/test_wfs_container/",waveformsContainer_list[0],overwrite = True) + loaded_wfs = WaveformsContainerIO.load(f"/tmp/test_wfs_container/waveforms_run{TestWaveformsMaker.run_number}.fits") + assert np.array_equal(waveformsContainer_list[0].wfs_hg,loaded_wfs.wfs_hg) + + def test_create_from_events_list(self) : + waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, + max_events = TestWaveformsMaker.max_events) + events_list = [] + for i,event in enumerate(waveformsMaker._reader) : + events_list.append(event) + waveformsContainer = WaveformsMaker.create_from_events_list(events_list, + waveformsMaker.run_number, + waveformsMaker.npixels, + waveformsMaker.nsamples, + waveformsMaker.subarray, + waveformsMaker.pixels_id) + assert isinstance(waveformsContainer,WaveformsContainer) + if __name__ == '__main__' : import logging logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level = logging.DEBUG) log = logging.getLogger(__name__) - log.handlers = logging.getLogger('__main__').handlers - TestWaveformsMaker().test_write_load_container() \ No newline at end of file + log.handlers = logging.getLogger('__main__').handlers \ No newline at end of file diff --git a/src/nectarchain/makers/waveformsMakers.py b/src/nectarchain/makers/waveformsMakers.py index 82534e5c..aa4c8bc6 100644 --- a/src/nectarchain/makers/waveformsMakers.py +++ b/src/nectarchain/makers/waveformsMakers.py @@ -5,35 +5,14 @@ from argparse import ArgumentError import numpy as np -from matplotlib import pyplot as plt -import copy -import os -import glob -from pathlib import Path - -from enum import Enum - from tqdm import tqdm +import copy -from astropy.io import fits -from astropy.table import QTable,Column,Table -import astropy.units as u - -from ctapipe.visualization import CameraDisplay -from ctapipe.coordinates import CameraFrame,EngineeringCameraFrame -from ctapipe.instrument import CameraGeometry,SubarrayDescription,TelescopeDescription - -from ctapipe_io_nectarcam import NectarCAMEventSource +from ctapipe.instrument import SubarrayDescription from ctapipe_io_nectarcam import constants - - -from ..data import DataManagement - -import sys - from ctapipe.containers import EventType -from ..data.container import WaveformsContainer +from ..data.container import WaveformsContainer from .core import ArrayDataMaker __all__ = ["WaveformsMaker"] @@ -56,7 +35,6 @@ def __init__(self,run_number : int,max_events : int = None,run_file = None,*args self.__geometry = self._reader.subarray.tel[__class__.TEL_ID].camera self.__subarray = self._reader.subarray - self.__nsamples = self._reader.camera_config.num_samples self.__wfs_hg = {} self.__wfs_lg = {} @@ -69,13 +47,14 @@ def create_from_events_list(events_list : list, subarray : SubarrayDescription, pixels_id : int, ) : - container = WaveformsContainer() - container.run_number = run_number - container.npixels = npixels - container.nsamples = nsamples - container.subarray = subarray - container.camera = __class__.CAMERA_NAME - container.pixels_id = pixels_id + container = WaveformsContainer( + run_number = run_number, + npixels = npixels, + nsamples = nsamples, + subarray = subarray, + camera = __class__.CAMERA_NAME, + pixels_id = pixels_id, + ) ucts_timestamp = [] ucts_busy_counter = [] @@ -87,14 +66,12 @@ def create_from_events_list(events_list : list, wfs_lg = [] for event in tqdm(events_list) : - ucts_timestamp.append(event.ucts_timestamp) - ucts_busy_counter.append(event.ucts_busy_counter) - ucts_event_counter.append(event.ucts_event_counter) - event_type.append(event.event_type) - event_id.append(event.event_id) - trig_pattern_all.append(event.trig_pattern_all) - broken_pixels = __class__._compute_broken_pixels_event(event,pixels_id) - + ucts_timestamp.append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_timestamp) + ucts_busy_counter.append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_busy_counter) + ucts_event_counter.append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_event_counter) + event_type.append(event.trigger.event_type.value) + event_id.append(event.index.event_id) + trig_pattern_all.append(event.nectarcam.tel[__class__.TEL_ID].evt.trigger_pattern.T) wfs_hg.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][pixels_id]) wfs_lg.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][pixels_id]) @@ -110,7 +87,7 @@ def create_from_events_list(events_list : list, container.trig_pattern = container.trig_pattern_all.any(axis = 2) container.multiplicity = np.uint16(np.count_nonzero(container.trig_pattern,axis = 1)) - broken_pixels = __class__._compute_broken_pixels() + broken_pixels = __class__._compute_broken_pixels(container.wfs_hg,container.wfs_lg) container.broken_pixels_hg = broken_pixels[0] container.broken_pixels_lg = broken_pixels[1] return container @@ -133,16 +110,16 @@ def _make_event(self, wfs_hg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) wfs_lg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) - super()._make_event(event = event, - trigger = trigger, - wfs_hg = wfs_hg_tmp, - wfs_lg = wfs_lg_tmp) + wfs_hg_tmp,wfs_lg_tmp = super()._make_event(event = event, + trigger = trigger, + return_wfs = True, + *args,**kwargs) name = __class__._get_name_trigger(trigger) self.__wfs_hg[f'{name}'].append(wfs_hg_tmp.tolist()) self.__wfs_lg[f'{name}'].append(wfs_lg_tmp.tolist()) - broken_pixels_hg,broken_pixels_lg = __class__._compute_broken_pixels(wfs_hg_tmp,wfs_lg_tmp) + broken_pixels_hg,broken_pixels_lg = __class__._compute_broken_pixels_event(event,self._pixels_id) self._broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) self._broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) @@ -150,12 +127,12 @@ def _make_output_container(self,trigger_type) : output = [] for trigger in trigger_type : waveformsContainer = WaveformsContainer( - run_number = self.run_number, - npixels = self.npixels, - nsamples = self.nsamples, - subarray = self.subarray, + run_number = self._run_number, + npixels = self._npixels, + nsamples = self._nsamples, + subarray = self._subarray, camera = self.CAMERA_NAME, - pixels_id = self.pixels_id, + pixels_id = self._pixels_id, nevents = self.nevents(trigger), wfs_hg = self.wfs_hg(trigger), wfs_lg = self.wfs_lg(trigger), @@ -186,21 +163,9 @@ def sort(waveformsContainer :WaveformsContainer, method = 'event_id') : ) if method == 'event_id' : index = np.argsort(waveformsContainer.event_id) - for field in waveformsContainer() : + for field in waveformsContainer.keys() : if not(field in ["run_number","npixels","nsamples","subarray","camera","pixels_id","nevents"]) : output[field] = waveformsContainer[field][index] - #output.ucts_busy_counter = waveformsContainer.ucts_busy_counter[index] - #output.ucts_event_counter = waveformsContainer.ucts_event_counter[index] - #output.event_type = waveformsContainer.event_type[index] - #output.event_id = waveformsContainer.event_id[index] - #output.trig_pattern_all = waveformsContainer.trig_pattern_all[index] - #output.trig_pattern = waveformsContainer.trig_pattern[index] - #output.multiplicity = waveformsContainer.multiplicity[index] -# - #output.wfs_hg = waveformsContainer.wfs_hg[index] - #output.wfs_lg = waveformsContainer.wfs_lg[index] - #output.broken_pixels_hg = waveformsContainer.broken_pixels_hg[index] - #output.broken_pixels_lg = waveformsContainer.broken_pixels_lg[index] else : raise ArgumentError(f"{method} is not a valid method for sorting") return output @@ -218,12 +183,15 @@ def select_waveforms_lg(waveformsContainer:WaveformsContainer,pixel_id : np.ndar return res + + @property + def _geometry(self) : return self.__geometry @property - def nsamples(self) : return self.__nsamples + def _subarray(self) : return self.__subarray @property - def geometry(self) : return self.__geometry + def geometry(self) : return copy.deepcopy(self.__geometry) @property - def subarray(self) : return self.__subarray + def subarray(self) : return copy.deepcopy(self.__subarray) def wfs_hg(self,trigger) : return np.array(self.__wfs_hg[__class__._get_name_trigger(trigger)],dtype = np.uint16) def wfs_lg(self,trigger) : return np.array(self.__wfs_lg[__class__._get_name_trigger(trigger)],dtype = np.uint16) \ No newline at end of file From f1c3298db9ba2ec6dbd80a198da43babefefb0de Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 15 Sep 2023 05:17:44 +0200 Subject: [PATCH 50/62] update calibration class following COntainers changes --- .../calibration/gain/FlatFieldSPEMakers.py | 10 +++++---- .../calibration/gain/PhotoStatisticMakers.py | 22 ++++++++++--------- src/nectarchain/makers/core.py | 2 +- 3 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py index c138a6cc..18a670a2 100644 --- a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py @@ -33,7 +33,9 @@ from .gainMakers import GainMaker -from ....data.container import ChargeContainer +from ....data.container import ChargesContainer + +from ...chargesMakers import ChargesMaker from .parameters import Parameter, Parameters @@ -280,13 +282,13 @@ def __init__(self,charge,counts,*args,**kwargs) -> None: self._results.add_column(Column(np.zeros((self.npixels),dtype = np.float64),"pvalue",unit = u.dimensionless_unscaled)) @classmethod - def create_from_chargeContainer(cls, signal : ChargeContainer,**kwargs) : - histo = signal.histo_hg(autoscale = True) + def create_from_chargesContainer(cls, signal : ChargesContainer,**kwargs) : + histo = ChargesMaker.histo_hg(signal,autoscale = True) return cls(charge = histo[1],counts = histo[0],pixels_id = signal.pixels_id,**kwargs) @classmethod def create_from_run_number(cls, run_number : int, **kwargs) : - raise NotImplementedError() + raise NotImplementedError("Need to implement here the use of the WaveformsMaker and ChargesMaker to produce the chargesContainer to be pass into the __ini__") #getters and setters @property diff --git a/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py b/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py index 53dbe0a0..2b453b42 100644 --- a/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py +++ b/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py @@ -18,7 +18,9 @@ from ctapipe_io_nectarcam import constants -from ....data.container import ChargeContainer +from ....data.container import ChargesContainer,ChargesContainerIO + +from ...chargesMakers import ChargesMaker from .gainMakers import GainMaker @@ -64,8 +66,8 @@ def __init__(self, @classmethod def create_from_chargeContainer(cls, - FFcharge : ChargeContainer, - Pedcharge : ChargeContainer, + FFcharge : ChargesContainer, + Pedcharge : ChargesContainer, coefCharge_FF_Ped, SPE_resolution, **kwargs) : @@ -99,7 +101,7 @@ def __readSPE(SPEresults) : return table['resolution'][table['is_valid']].value,table['pixels_id'][table['is_valid']].value @staticmethod - def __get_charges_FF_Ped_reshaped( FFcharge : ChargeContainer, Pedcharge : ChargeContainer, SPE_resolution, SPE_pixels_id) : + def __get_charges_FF_Ped_reshaped( FFcharge : ChargesContainer, Pedcharge : ChargesContainer, SPE_resolution, SPE_pixels_id) : log.info("reshape of SPE, Ped and FF data with intersection of pixel ids") out = {} @@ -110,10 +112,10 @@ def __get_charges_FF_Ped_reshaped( FFcharge : ChargeContainer, Pedcharge : Charg out["SPE_resolution"] = SPE_resolution[mask_SPE] out["pixels_id"] = SPEFFPed_intersection - out["FFcharge_hg"] = FFcharge.select_charge_hg(SPEFFPed_intersection) - out["FFcharge_lg"] = FFcharge.select_charge_lg(SPEFFPed_intersection) - out["Pedcharge_hg"] = Pedcharge.select_charge_hg(SPEFFPed_intersection) - out["Pedcharge_lg"] = Pedcharge.select_charge_lg(SPEFFPed_intersection) + out["FFcharge_hg"] = ChargesMaker.select_charges_hg(FFcharge,SPEFFPed_intersection) + out["FFcharge_lg"] = ChargesMaker.select_charges_lg(FFcharge,SPEFFPed_intersection) + out["Pedcharge_hg"] = ChargesMaker.select_charges_hg(Pedcharge,SPEFFPed_intersection) + out["Pedcharge_lg"] = ChargesMaker.select_charges_lg(Pedcharge,SPEFFPed_intersection) log.info(f"data have {len(SPEFFPed_intersection)} pixels in common") return out @@ -134,7 +136,7 @@ def __readFF(FFRun,**kwargs) : coefCharge_FF_Ped = 1 if isinstance(FFRun,int) : try : - FFcharge = ChargeContainer.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{method}",FFRun) + FFcharge = ChargesContainerIO.load(f"{os.environ['NECTARCAMDATA']}/charges/{method}",FFRun) log.info(f'charges have ever been computed for FF run {FFRun}') except Exception as e : log.error("charge have not been yet computed") @@ -151,7 +153,7 @@ def __readPed(PedRun,**kwargs) : method = 'FullWaveformSum'#kwargs.get('method','std') if isinstance(PedRun,int) : try : - Pedcharge = ChargeContainer.from_file(f"{os.environ['NECTARCAMDATA']}/charges/{method}",PedRun) + Pedcharge = ChargesContainerIO.load(f"{os.environ['NECTARCAMDATA']}/charges/{method}",PedRun) log.info(f'charges have ever been computed for Ped run {PedRun}') except Exception as e : log.error("charge have not been yet computed") diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 46cdf553..308122c1 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -18,7 +18,7 @@ from ..data import DataManagement from ..data.container.core import ArrayDataContainer -__all__ = ["BaseMaker"] +__all__ = ["ArrayDataMaker"] class BaseMaker(ABC): """Mother class for all the makers, the role of makers is to do computation on the data. From 71976ef9209419b8c0271f8c040558aed4f7dd78 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 15 Sep 2023 05:24:01 +0200 Subject: [PATCH 51/62] bugfix according to unit tests --- src/nectarchain/makers/calibration/tests/test_core.py | 5 ----- src/nectarchain/makers/chargesMakers.py | 4 ++-- src/nectarchain/makers/waveformsMakers.py | 2 +- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/src/nectarchain/makers/calibration/tests/test_core.py b/src/nectarchain/makers/calibration/tests/test_core.py index a06eccd5..a9e8a01e 100644 --- a/src/nectarchain/makers/calibration/tests/test_core.py +++ b/src/nectarchain/makers/calibration/tests/test_core.py @@ -51,8 +51,3 @@ def test_change_pixels_id_attribute(self): assert np.equal(calibration_maker._pixels_id,new_pixels_id).all() - - # Tests that an instance of CalibrationMaker cannot be created with an empty list of pixel ids as input. - def test_create_instance_with_empty_pixel_ids(self): - with pytest.raises(TypeError): - gain_maker = CalibrationMakerforTest([]) diff --git a/src/nectarchain/makers/chargesMakers.py b/src/nectarchain/makers/chargesMakers.py index 73557826..afd8d325 100644 --- a/src/nectarchain/makers/chargesMakers.py +++ b/src/nectarchain/makers/chargesMakers.py @@ -181,7 +181,7 @@ def _get_imageExtractor(method,subarray,**kwargs) : imageExtractor = eval(method)(subarray,**extractor_kwargs) return imageExtractor - def _make_output_container(self,trigger_type,method : str) : + def _make_output_container(self,trigger_type,method : str,*args,**kwargs) : output = [] for trigger in trigger_type : chargesContainer = ChargesContainer( @@ -223,7 +223,7 @@ def sort(chargesContainer :ChargesContainer, method = 'event_id') : if method == 'event_id' : index = np.argsort(chargesContainer.event_id) for field in chargesContainer.keys() : - if not(field in ["run_number","npixels","camera","pixels_id","nevents"]) : + if not(field in ["run_number","npixels","camera","pixels_id","nevents","method"]) : output[field] = chargesContainer[field][index] else : raise ArgumentError(f"{method} is not a valid method for sorting") diff --git a/src/nectarchain/makers/waveformsMakers.py b/src/nectarchain/makers/waveformsMakers.py index aa4c8bc6..ecc01603 100644 --- a/src/nectarchain/makers/waveformsMakers.py +++ b/src/nectarchain/makers/waveformsMakers.py @@ -123,7 +123,7 @@ def _make_event(self, self._broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) self._broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) - def _make_output_container(self,trigger_type) : + def _make_output_container(self,trigger_type,*args,**kwargs) : output = [] for trigger in trigger_type : waveformsContainer = WaveformsContainer( From 7ba1ab3d9f983f25f2f654e32b9655f29d109bb1 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 15 Sep 2023 16:31:36 +0200 Subject: [PATCH 52/62] docstrings container + unit test --- src/nectarchain/data/container/__init__.py | 3 +- .../data/container/chargesContainer.py | 233 +++++++++------- src/nectarchain/data/container/core.py | 1 + .../data/container/tests/test_charge.py | 252 ++++-------------- .../data/container/tests/test_waveforms.py | 76 ++++++ .../data/container/waveformsContainer.py | 68 ++++- 6 files changed, 330 insertions(+), 303 deletions(-) diff --git a/src/nectarchain/data/container/__init__.py b/src/nectarchain/data/container/__init__.py index 905aff5a..9df719d0 100644 --- a/src/nectarchain/data/container/__init__.py +++ b/src/nectarchain/data/container/__init__.py @@ -1,2 +1,3 @@ from .waveformsContainer import * -from .chargesContainer import * \ No newline at end of file +from .chargesContainer import * +from .core import * \ No newline at end of file diff --git a/src/nectarchain/data/container/chargesContainer.py b/src/nectarchain/data/container/chargesContainer.py index df7fb1af..d23634f4 100644 --- a/src/nectarchain/data/container/chargesContainer.py +++ b/src/nectarchain/data/container/chargesContainer.py @@ -4,7 +4,7 @@ log.handlers = logging.getLogger('__main__').handlers import numpy as np -from ctapipe.containers import Field +from ctapipe.container import Field from abc import ABC import os from pathlib import Path @@ -13,129 +13,172 @@ from .core import ArrayDataContainer class ChargesContainer(ArrayDataContainer): - - charges_hg = Field( - type = np.ndarray, - description = 'The high gain charges') - charges_lg = Field( - type = np.ndarray, - description = 'The low gain charges') - peak_hg = Field( - type = np.ndarray, - description = 'The high gain peak time') - peak_lg = Field( - type = np.ndarray, - description = 'The low gain peak time') - method = Field( - type = str, - description = 'The charge extraction method used') - + """ + A container that holds information about charges from a specific run. + + Fields: + charges_hg (np.ndarray): An array of high gain charges. + charges_lg (np.ndarray): An array of low gain charges. + peak_hg (np.ndarray): An array of high gain peak time. + peak_lg (np.ndarray): An array of low gain peak time. + method (str): The charge extraction method used. + """ + + charges_hg = Field( + type=np.ndarray(), + description='The high gain charges' + ) + charges_lg = Field( + type=np.ndarray, + description='The low gain charges' + ) + peak_hg = Field( + type=np.ndarray, + description='The high gain peak time' + ) + peak_lg = Field( + type=np.ndarray, + description='The low gain peak time' + ) + method = Field( + type=str, + description='The charge extraction method used' + ) class ChargesContainerIO(ABC) : - def write(path : Path, containers : ChargesContainer,**kwargs) : - """method to write in an output FITS file the ChargeContainer. - + """ + The `ChargesContainerIO` class provides methods for writing and loading `ChargesContainer` instances to/from FITS files. + + Example Usage: + # Writing a ChargesContainer instance to a FITS file + chargesContainer = ChargesContainer() + ChargesContainerIO.write(path, chargesContainer) + + # Loading a ChargesContainer instance from a FITS file + chargesContainer = ChargesContainerIO.load(path, run_number) + + Main functionalities: + - Writing a `ChargesContainer` instance to a FITS file. + - Loading a `ChargesContainer` instance from a FITS file. + + Methods: + - `write(path: Path, container: ChargesContainer, **kwargs) -> None`: Writes a `ChargesContainer` instance to a FITS file. The method takes a file path and the `ChargesContainer` instance as input. Additional keyword arguments can be provided to customize the file name and overwrite behavior. + - `load(path: Path, run_number: int, **kwargs) -> ChargesContainer`: Loads a `ChargesContainer` instance from a FITS file. The method takes a file path and the run number as input. Additional keyword arguments can be provided to specify an explicit file name. + + Fields: + The `ChargesContainerIO` class does not have any fields. + """ + @staticmethod + def write(path : Path, container : ChargesContainer,**kwargs) -> None: + """Write a ChargesContainer instance to a FITS file. Args: - path (str): the directory where you want to save data + path (str): The directory where the FITS file will be saved. + container (ChargesContainer): The ChargesContainer instance to be written to the FITS file. + **kwargs: Additional keyword arguments for customization. + Keyword Args: + suffix (str): A suffix to be added to the file name (default: ""). + overwrite (bool): Whether to overwrite the file if it already exists (default: False). + Returns: + None: This method does not return any value. + Raises: + OSError: If there is an error while writing the FITS file. + Exception: If there is any other exception during the writing process. + Example: + chargesContainer = ChargesContainer() + ChargesContainerIO.write(path, chargesContainer, suffix="v1", overwrite=True) """ suffix = kwargs.get("suffix","") if suffix != "" : suffix = f"_{suffix}" - log.info(f"saving in {path}") os.makedirs(path,exist_ok = True) - hdr = fits.Header() - hdr['RUN'] = containers.run_number - hdr['NEVENTS'] = containers.nevents - hdr['NPIXELS'] = containers.npixels - hdr['METHOD'] = containers.method - hdr['CAMERA'] = containers.camera + hdr['RUN'] = container.run_number + hdr['NEVENTS'] = container.nevents + hdr['NPIXELS'] = container.npixels + hdr['METHOD'] = container.method + hdr['CAMERA'] = container.camera + hdr['COMMENT'] = f"The charge containeur for run {container.run_number} with {container.method} method : primary is the pixels id, then you can find HG charge, LG charge, HG peak and LG peak, 2 last HDU are composed of event properties and trigger patern" - - hdr['COMMENT'] = f"The charge containeur for run {containers.run_number} with {containers.method} method : primary is the pixels id, then you can find HG charge, LG charge, HG peak and LG peak, 2 last HDU are composed of event properties and trigger patern" - - primary_hdu = fits.PrimaryHDU(containers.pixels_id,header=hdr) - charge_hg_hdu = fits.ImageHDU(containers.charges_hg,name = "HG charge") - charge_lg_hdu = fits.ImageHDU(containers.charges_lg,name = "LG charge") - peak_hg_hdu = fits.ImageHDU(containers.peak_hg, name = 'HG peak time') - peak_lg_hdu = fits.ImageHDU(containers.peak_lg, name = 'LG peak time') - - col1 = fits.Column(array = containers.broken_pixels_hg, name = "HG broken pixels", format = f'{containers.broken_pixels_hg.shape[1]}L') - col2 = fits.Column(array = containers.broken_pixels_lg, name = "LG broken pixels", format = f'{containers.broken_pixels_lg.shape[1]}L') + primary_hdu = fits.PrimaryHDU(container.pixels_id,header=hdr) + charge_hg_hdu = fits.ImageHDU(container.charges_hg,name = "HG charge") + charge_lg_hdu = fits.ImageHDU(container.charges_lg,name = "LG charge") + peak_hg_hdu = fits.ImageHDU(container.peak_hg, name = 'HG peak time') + peak_lg_hdu = fits.ImageHDU(container.peak_lg, name = 'LG peak time') + + col1 = fits.Column(array = container.broken_pixels_hg, name = "HG broken pixels", format = f'{container.broken_pixels_hg.shape[1]}L') + col2 = fits.Column(array = container.broken_pixels_lg, name = "LG broken pixels", format = f'{container.broken_pixels_lg.shape[1]}L') coldefs = fits.ColDefs([col1, col2]) broken_pixels = fits.BinTableHDU.from_columns(coldefs,name = 'trigger patern') - col1 = fits.Column(array = containers.event_id, name = "event_id", format = '1I') - col2 = fits.Column(array = containers.event_type, name = "event_type", format = '1I') - col3 = fits.Column(array = containers.ucts_timestamp, name = "ucts_timestamp", format = '1K') - col4 = fits.Column(array = containers.ucts_busy_counter, name = "ucts_busy_counter", format = '1I') - col5 = fits.Column(array = containers.ucts_event_counter, name = "ucts_event_counter", format = '1I') - col6 = fits.Column(array = containers.multiplicity, name = "multiplicity", format = '1I') - + col1 = fits.Column(array = container.event_id, name = "event_id", format = '1I') + col2 = fits.Column(array = container.event_type, name = "event_type", format = '1I') + col3 = fits.Column(array = container.ucts_timestamp, name = "ucts_timestamp", format = '1K') + col4 = fits.Column(array = container.ucts_busy_counter, name = "ucts_busy_counter", format = '1I') + col5 = fits.Column(array = container.ucts_event_counter, name = "ucts_event_counter", format = '1I') + col6 = fits.Column(array = container.multiplicity, name = "multiplicity", format = '1I') coldefs = fits.ColDefs([col1, col2, col3, col4, col5, col6]) event_properties = fits.BinTableHDU.from_columns(coldefs, name = 'event properties') - - col1 = fits.Column(array = containers.trig_pattern_all, name = "trig_pattern_all", format = f'{4 * containers.trig_pattern_all.shape[1]}L',dim = f'({ containers.trig_pattern_all.shape[1]},4)') - col2 = fits.Column(array = containers.trig_pattern, name = "trig_pattern", format = f'{containers.trig_pattern_all.shape[1]}L') + + col1 = fits.Column(array = container.trig_pattern_all, name = "trig_pattern_all", format = f'{4 * container.trig_pattern_all.shape[1]}L',dim = f'({ container.trig_pattern_all.shape[1]},4)') + col2 = fits.Column(array = container.trig_pattern, name = "trig_pattern", format = f'{container.trig_pattern_all.shape[1]}L') coldefs = fits.ColDefs([col1, col2]) trigger_patern = fits.BinTableHDU.from_columns(coldefs, name = 'trigger patern') - + hdul = fits.HDUList([primary_hdu, charge_hg_hdu, charge_lg_hdu,peak_hg_hdu,peak_lg_hdu,broken_pixels,event_properties,trigger_patern]) try : - hdul.writeto(Path(path)/f"charge_run{containers.run_number}{suffix}.fits",overwrite=kwargs.get('overwrite',False)) - log.info(f"charge saved in {Path(path)}/charge_run{containers.run_number}{suffix}.fits") + hdul.writeto(Path(path)/f"charge_run{container.run_number}{suffix}.fits",overwrite=kwargs.get('overwrite',False)) + log.info(f"charge saved in {Path(path)}/charge_run{container.run_number}{suffix}.fits") except OSError as e : log.warning(e) except Exception as e : log.error(e,exc_info = True) raise e - def load(path : Path,run_number : int,**kwargs) : - """load ChargeContainer from FITS file previously written with ChargeContainer.write() method - + @staticmethod + def load(path: str, run_number: int, **kwargs) -> ChargesContainer: + """Load a ChargesContainer instance from a FITS file. + This method opens a FITS file and retrieves the necessary data to create a ChargesContainer instance. + The FITS file should have been previously written using the write method of the ChargesContainerIO class. Args: - path (str): path of the FITS file - run_number (int) : the run number - + path (str): The path of the FITS file. + run_number (int): The run number. + **kwargs: Additional keyword arguments. + explicit_filename (str): If provided, the explicit filename to load. Returns: - ChargeContainer: ChargeContainer instance + ChargesContainer: The loaded ChargesContainer instance. + Example: + chargesContainer = ChargesContainerIO.load(path, run_number) """ - if kwargs.get("explicit_filename",False) : + if kwargs.get("explicit_filename", False): filename = kwargs.get("explicit_filename") - log.info(f"loading {filename}") - else : - log.info(f"loading in {path} run number {run_number}") - filename = Path(path)/f"charge_run{run_number}.fits" - - with fits.open(filename) as hdul : - containers = ChargesContainer() - containers.run_number = hdul[0].header['RUN'] - containers.nevents = hdul[0].header['NEVENTS'] - containers.npixels = hdul[0].header['NPIXELS'] - containers.method = hdul[0].header['METHOD'] - containers.camera = hdul[0].header['CAMERA'] - - containers.pixels_id = hdul[0].data - containers.charges_hg = hdul[1].data - containers.charges_lg = hdul[2].data - containers.peak_hg = hdul[3].data - containers.peak_lg = hdul[4].data - + log.info(f"Loading {filename}") + else: + log.info(f"Loading in {path} run number {run_number}") + filename = Path(path) / f"charge_run{run_number}.fits" + + with fits.open(filename) as hdul: + container = ChargesContainer() + container.run_number = hdul[0].header['RUN'] + container.nevents = hdul[0].header['NEVENTS'] + container.npixels = hdul[0].header['NPIXELS'] + container.method = hdul[0].header['METHOD'] + container.camera = hdul[0].header['CAMERA'] + container.pixels_id = hdul[0].data + container.charges_hg = hdul[1].data + container.charges_lg = hdul[2].data + container.peak_hg = hdul[3].data + container.peak_lg = hdul[4].data broken_pixels = hdul[5].data - containers.broken_pixels_hg = broken_pixels["HG broken pixels"] - containers.broken_pixels_lg = broken_pixels["LG broken pixels"] - - - + container.broken_pixels_hg = broken_pixels["HG broken pixels"] + container.broken_pixels_lg = broken_pixels["LG broken pixels"] table_prop = hdul[6].data - containers.event_id = table_prop["event_id"] - containers.event_type = table_prop["event_type"] - containers.ucts_timestamp = table_prop["ucts_timestamp"] - containers.ucts_busy_counter = table_prop["ucts_busy_counter"] - containers.ucts_event_counter = table_prop["ucts_event_counter"] - containers.multiplicity = table_prop["multiplicity"] - + container.event_id = table_prop["event_id"] + container.event_type = table_prop["event_type"] + container.ucts_timestamp = table_prop["ucts_timestamp"] + container.ucts_busy_counter = table_prop["ucts_busy_counter"] + container.ucts_event_counter = table_prop["ucts_event_counter"] + container.multiplicity = table_prop["multiplicity"] table_trigger = hdul[7].data - containers.trig_pattern_all = table_trigger["trig_pattern_all"] - containers.trig_pattern = table_trigger["trig_pattern"] - return containers + container.trig_pattern_all = table_trigger["trig_pattern_all"] + container.trig_pattern = table_trigger["trig_pattern"] + return container diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index 6471b3a6..d54140c4 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -7,6 +7,7 @@ from ctapipe.containers import Container,Field import numpy as np +__all__= ["ArrayDataContainer"] class ArrayDataContainer(Container): """ diff --git a/src/nectarchain/data/container/tests/test_charge.py b/src/nectarchain/data/container/tests/test_charge.py index 8018ed09..b15ebaa1 100644 --- a/src/nectarchain/data/container/tests/test_charge.py +++ b/src/nectarchain/data/container/tests/test_charge.py @@ -1,43 +1,51 @@ -from nectarchain.data.container.charge import ChargeContainers, ChargeContainer -from nectarchain.data.container.waveforms import WaveformsContainers +from ..chargesContainer import ChargesContainer,ChargesContainerIO +from ..waveformsContainer import WaveformsContainer +from ....makers import ChargesMaker import glob import numpy as np def create_fake_chargeContainer() : - pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) - nevents = 40 - npixels = 10 - rng = np.random.default_rng() - charge_hg = rng.integers(low=0, high=1000, size= (nevents,npixels)) - charge_lg = rng.integers(low=0, high=1000, size= (nevents,npixels)) - peak_hg = rng.integers(low=0, high=60, size= (nevents,npixels)) - peak_lg = rng.integers(low=0, high=60, size= (nevents,npixels)) - run_number = 1234 - return ChargeContainer( - charge_hg = charge_hg , - charge_lg = charge_lg, - peak_hg = peak_hg, - peak_lg = peak_lg, - run_number = run_number, - pixels_id = pixels_id, - nevents = nevents, - npixels = npixels + nevents = TestChargesContainer.nevents, + npixels = TestChargesContainer.npixels, + rng = np.random.default_rng(), + return ChargesContainer( + pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]), + nevents =nevents, + npixels =npixels, + charge_hg = rng.integers(low=0, high=1000, size= (nevents,npixels)), + charge_lg = rng.integers(low=0, high=1000, size= (nevents,npixels)), + peak_hg = rng.integers(low=0, high=60, size= (nevents,npixels)), + peak_lg = rng.integers(low=0, high=60, size= (nevents,npixels)), + run_number = TestChargesContainer.run_number, + camera = 'TEST', + broken_pixels_hg = rng.integers(low=0, high=1, size= (nevents,npixels)), + broken_pixels_lg = rng.integers(low=0, high=1, size= (nevents,npixels)), + ucts_timestamp =rng.integers(low=0, high=100, size= (nevents)), + ucts_busy_counter =rng.integers(low=0, high=100, size= (nevents)), + ucts_event_counter =rng.integers(low=0, high=100, size= (nevents)), + event_type =rng.integers(low=0, high=1, size= (nevents)), + event_id =rng.integers(low=0, high=1000, size= (nevents)), + trig_pattern_all = rng.integers(low=0, high=1, size= (nevents,npixels,4)), + trig_pattern = rng.integers(low=0, high=1, size= (nevents,npixels)), + multiplicity =rng.integers(low=0, high=1, size= (nevents)) ) -class TestChargeContainer: - +class TestChargesContainer: + run_number = 1234 + nevents = 140 + npixels = 10 # Tests that a ChargeContainer object can be created with valid input parameters. def test_create_charge_container(self): pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) - nevents = 40 - npixels = 10 + nevents = TestChargesContainer.nevents + npixels = TestChargesContainer.npixels + run_number = TestChargesContainer.run_number charge_hg = np.random.randn(nevents,npixels) charge_lg = np.random.randn(nevents,npixels) peak_hg = np.random.randn(nevents,npixels) peak_lg = np.random.randn(nevents,npixels) - run_number = 1234 method = 'FullWaveformSum' - charge_container = ChargeContainer( + charge_container = ChargesContainer( charge_hg = charge_hg , charge_lg = charge_lg, peak_hg = peak_hg, @@ -49,8 +57,8 @@ def test_create_charge_container(self): method = method ) - assert np.allclose(charge_container.charge_hg,charge_hg) - assert np.allclose(charge_container.charge_lg,charge_lg) + assert np.allclose(charge_container.charges_hg,charge_hg) + assert np.allclose(charge_container.charges_lg,charge_lg) assert np.allclose(charge_container.peak_hg,peak_hg) assert np.allclose(charge_container.peak_lg,peak_lg) assert charge_container.run_number == run_number @@ -71,56 +79,39 @@ def test_create_charge_container(self): # Tests that the ChargeContainer object can be written to a file and the file is created. def test_write_charge_container(self, tmp_path = "/tmp"): charge_container = create_fake_chargeContainer() + tmp_path += f"/{np.random.randn(1)}" - charge_container.write(tmp_path) + ChargesContainerIO.write(tmp_path,charge_container) - assert len(glob.glob(f"{tmp_path}/charge_run1234.fits")) == 1 + assert len(glob.glob(f"{tmp_path}/charge_run{TestChargesContainer.run_number}.fits")) == 1 # Tests that a ChargeContainer object can be loaded from a file and the object is correctly initialized. def test_load_charge_container(self, tmp_path = "/tmp"): - pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) - nevents = 40 - npixels = 10 - charge_hg = np.random.randn(nevents,npixels) - charge_lg = np.random.randn(nevents,npixels) - peak_hg = np.random.randn(nevents,npixels) - peak_lg = np.random.randn(nevents,npixels) - run_number = np.random.randn(1)[0] - method = 'FullWaveformSum' - charge_container = ChargeContainer( - charge_hg = charge_hg , - charge_lg = charge_lg, - peak_hg = peak_hg, - peak_lg = peak_lg, - run_number = run_number, - pixels_id = pixels_id, - nevents = nevents, - npixels = npixels, - method = method - ) + charge_container = create_fake_chargeContainer() + tmp_path += f"/{np.random.randn(1)}" - charge_container.write(tmp_path) + ChargesContainerIO.write(tmp_path,charge_container) - loaded_charge_container = ChargeContainer.from_file(tmp_path, run_number) + loaded_charge_container = ChargesContainerIO.load(tmp_path,TestChargesContainer.run_number ) - assert isinstance(loaded_charge_container, ChargeContainer) - assert np.allclose(loaded_charge_container.charge_hg,charge_hg) - assert np.allclose(loaded_charge_container.charge_lg,charge_lg) - assert np.allclose(loaded_charge_container.peak_hg,peak_hg) - assert np.allclose(loaded_charge_container.peak_lg,peak_lg) - assert loaded_charge_container.run_number == run_number - assert loaded_charge_container.pixels_id.tolist() == pixels_id.tolist() - assert loaded_charge_container.nevents == nevents - assert loaded_charge_container.npixels == npixels - assert loaded_charge_container.method == method + assert isinstance(loaded_charge_container, ChargesContainer) + assert np.allclose(loaded_charge_container.charges_hg,charge_container.charges_hg) + assert np.allclose(loaded_charge_container.charges_lg,charge_container.charges_lg) + assert np.allclose(loaded_charge_container.peak_hg,charge_container.peak_hg) + assert np.allclose(loaded_charge_container.peak_lg,charge_container.peak_lg) + assert loaded_charge_container.run_number == charge_container.run_number + assert loaded_charge_container.pixels_id.tolist() == charge_container.pixels_id.tolist() + assert loaded_charge_container.nevents == charge_container.nevents + assert loaded_charge_container.npixels == charge_container.npixels + assert loaded_charge_container.method == charge_container.method # Tests that the ChargeContainer object can be sorted by event_id and the object is sorted accordingly. def test_sort_charge_container(self): charge_container = create_fake_chargeContainer() - charge_container.sort() + sorted_charge_container = ChargesMaker.sort(charge_container) - assert charge_container.event_id.tolist() == sorted(charge_container.event_id.tolist()) + assert sorted_charge_container.event_id.tolist() == sorted(charge_container.event_id.tolist()) # Tests that the run_number, pixels_id, npixels, nevents, method, multiplicity, and trig_pattern properties of the ChargeContainer object can be accessed and the values are correct. def test_access_properties(self): @@ -133,7 +124,7 @@ def test_access_properties(self): peak_lg = np.random.randn(nevents,npixels) run_number = 1234 method = 'FullWaveformSum' - charge_container = ChargeContainer( + charge_container = ChargesContainer( charge_hg = charge_hg , charge_lg = charge_lg, peak_hg = peak_hg, @@ -151,133 +142,4 @@ def test_access_properties(self): assert charge_container.nevents == nevents assert charge_container.method == method assert charge_container.multiplicity.shape == (nevents,) - assert charge_container.trig_pattern.shape == (nevents,4) - - -class TestChargeContainers: - - # Tests that an instance of ChargeContainers can be created with default arguments - def test_create_instance_with_default_arguments(self): - charge_containers = ChargeContainers() - assert len(charge_containers.chargeContainers) == 0 - assert charge_containers.nChargeContainer == 0 - - # Tests that an instance of ChargeContainers can be created from a WaveformsContainers instance - #def test_create_instance_from_waveforms(self): - # waveform_containers = WaveformsContainers(run_number = 1234) - # charge_containers = ChargeContainers.from_waveforms(waveform_containers) - # assert len(charge_containers.chargeContainers) == waveform_containers.nWaveformsContainer - # assert charge_containers.nChargeContainer == waveform_containers.nWaveformsContainer - - # Tests that ChargeContainers can be written to disk - def test_write_to_disk(self, tmpdir): - charge_containers = ChargeContainers() - - charge_container = create_fake_chargeContainer() - - charge_containers.append(charge_container) - path = str(tmpdir) - charge_containers.write(path) - assert len(glob.glob(f"{path}/charge_run*1234*.fits")) == 1 - - # Tests that ChargeContainers can be loaded from disk - def test_load_from_disk(self, tmpdir): - charge_containers = ChargeContainers() - - pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) - nevents = 40 - npixels = 10 - charge_hg = np.random.randn(nevents,npixels) - charge_lg = np.random.randn(nevents,npixels) - peak_hg = np.random.randn(nevents,npixels) - peak_lg = np.random.randn(nevents,npixels) - run_number = np.random.randn(1)[0] - - charge_container = ChargeContainer( - charge_hg = charge_hg , - charge_lg = charge_lg, - peak_hg = peak_hg, - peak_lg = peak_lg, - run_number = run_number, - pixels_id = pixels_id, - nevents = nevents, - npixels = npixels - ) - charge_containers.append(charge_container) - path = str(tmpdir) - charge_containers.write(path) - loaded_charge_containers = ChargeContainers.from_file(path, run_number) - assert len(loaded_charge_containers.chargeContainers) == 1 - assert loaded_charge_containers.nChargeContainer == 1 - - # Tests that a ChargeContainer can be appended to a ChargeContainers instance - def test_append_charge_container(self): - charge_containers = ChargeContainers() - pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) - nevents = 40 - npixels = 10 - charge_hg = np.random.randn(nevents,npixels) - charge_lg = np.random.randn(nevents,npixels) - peak_hg = np.random.randn(nevents,npixels) - peak_lg = np.random.randn(nevents,npixels) - run_number = 1234 - - charge_container = ChargeContainer( - charge_hg = charge_hg , - charge_lg = charge_lg, - peak_hg = peak_hg, - peak_lg = peak_lg, - run_number = run_number, - pixels_id = pixels_id, - nevents = nevents, - npixels = npixels - ) - charge_containers.append(charge_container) - assert len(charge_containers.chargeContainers) == 1 - assert charge_containers.nChargeContainer == 1 - - # Tests that a ChargeContainers instance can be merged into a single ChargeContainer - def test_merge_into_single_charge_container(self): - charge_containers = ChargeContainers() - pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) - nevents = 40 - npixels = 10 - charge_hg = np.random.randn(nevents,npixels) - charge_lg = np.random.randn(nevents,npixels) - peak_hg = np.random.randn(nevents,npixels) - peak_lg = np.random.randn(nevents,npixels) - run_number = 1234 - - charge_container1 = ChargeContainer( - charge_hg = charge_hg , - charge_lg = charge_lg, - peak_hg = peak_hg, - peak_lg = peak_lg, - run_number = run_number, - pixels_id = pixels_id, - nevents = nevents, - npixels = npixels - ) - pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) - nevents = 40 - npixels = 10 - charge_hg = np.random.randn(nevents,npixels) - charge_lg = np.random.randn(nevents,npixels) - peak_hg = np.random.randn(nevents,npixels) - peak_lg = np.random.randn(nevents,npixels) - run_number = 1234 - - charge_container2 = ChargeContainer( - charge_hg = charge_hg , - charge_lg = charge_lg, - peak_hg = peak_hg, - peak_lg = peak_lg, - run_number = run_number, - pixels_id = pixels_id, - nevents = nevents, - npixels = npixels - ) - charge_containers.append(charge_container1) - charge_containers.append(charge_container2) - merged_charge_container = charge_containers.merge() - assert merged_charge_container.nevents == charge_container1.nevents + charge_container2.nevents \ No newline at end of file + assert charge_container.trig_pattern.shape == (nevents,4) \ No newline at end of file diff --git a/src/nectarchain/data/container/tests/test_waveforms.py b/src/nectarchain/data/container/tests/test_waveforms.py index e69de29b..d3689f79 100644 --- a/src/nectarchain/data/container/tests/test_waveforms.py +++ b/src/nectarchain/data/container/tests/test_waveforms.py @@ -0,0 +1,76 @@ +from ..waveformsContainer import WaveformsContainer,WaveformsContainerIO +from ....makers import WaveformsMaker +import glob +import numpy as np + +def create_fake_waveformsContainer() : + nevents = TestWaveformsContainer.nevents, + npixels = TestWaveformsContainer.npixels, + nsamples = TestWaveformsContainer.nsamples, + rng = np.random.default_rng(), + + return WaveformsContainer( + pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]), + nevents =nevents, + npixels =npixels, + wfs_hg = rng.integers(low=0, high=1000, size= (nevents,npixels,nsamples)), + wfs_lg = rng.integers(low=0, high=1000, size= (nevents,npixels,nsamples)), + run_number = TestWaveformsContainer.run_number, + camera = 'TEST', + broken_pixels_hg = rng.integers(low=0, high=1, size= (nevents,npixels)), + broken_pixels_lg = rng.integers(low=0, high=1, size= (nevents,npixels)), + ucts_timestamp =rng.integers(low=0, high=100, size= (nevents)), + ucts_busy_counter =rng.integers(low=0, high=100, size= (nevents)), + ucts_event_counter =rng.integers(low=0, high=100, size= (nevents)), + event_type =rng.integers(low=0, high=1, size= (nevents)), + event_id =rng.integers(low=0, high=1000, size= (nevents)), + trig_pattern_all = rng.integers(low=0, high=1, size= (nevents,npixels,4)), + trig_pattern = rng.integers(low=0, high=1, size= (nevents,npixels)), + multiplicity =rng.integers(low=0, high=1, size= (nevents)) + ) + +class TestWaveformsContainer: + run_number = 1234 + nevents = 140 + npixels = 10 + nsamples = 5 + # Tests that a ChargeContainer object can be created with valid input parameters. + def test_create_waveform_container(self): + waveform_container = create_fake_waveformsContainer() + assert isinstance(waveform_container,WaveformsContainer) + + + # Tests that the ChargeContainer object can be written to a file and the file is created. + def test_write_waveform_container(self, tmp_path = "/tmp"): + waveform_container = create_fake_waveformsContainer() + tmp_path += f"/{np.random.randn(1)}" + + WaveformsContainerIO.write(tmp_path,waveform_container) + + assert len(glob.glob(f"{tmp_path}/*_run{TestWaveformsContainer.run_number}.fits")) == 1 + + # Tests that a ChargeContainer object can be loaded from a file and the object is correctly initialized. + def test_load_waveform_container(self, tmp_path = "/tmp"): + waveform_container = create_fake_waveformsContainer() + tmp_path += f"/{np.random.randn(1)}" + + WaveformsContainerIO.write(tmp_path,waveform_container) + + loaded_waveform_container = WaveformsContainerIO.load(tmp_path,TestWaveformsContainer.run_number ) + + assert isinstance(loaded_waveform_container, WaveformsContainer) + assert np.allclose(loaded_waveform_container.wfs_hg,waveform_container.charges_hg) + assert np.allclose(loaded_waveform_container.wfs_lg,waveform_container.charges_lg) + assert loaded_waveform_container.run_number == waveform_container.run_number + assert loaded_waveform_container.pixels_id.tolist() == waveform_container.pixels_id.tolist() + assert loaded_waveform_container.nevents == waveform_container.nevents + assert loaded_waveform_container.npixels == waveform_container.npixels + assert loaded_waveform_container.nsamples == waveform_container.nsamples + + # Tests that the ChargeContainer object can be sorted by event_id and the object is sorted accordingly. + def test_sort_waveform_container(self): + waveform_container = create_fake_waveformsContainer() + + sorted_waveform_container = WaveformsMaker.sort(waveform_container) + + assert sorted_waveform_container.event_id.tolist() == sorted(waveform_container.event_id.tolist()) \ No newline at end of file diff --git a/src/nectarchain/data/container/waveformsContainer.py b/src/nectarchain/data/container/waveformsContainer.py index 1a277b11..57563e80 100644 --- a/src/nectarchain/data/container/waveformsContainer.py +++ b/src/nectarchain/data/container/waveformsContainer.py @@ -26,7 +26,7 @@ class WaveformsContainer(ArrayDataContainer): """ A container that holds information about waveforms from a specific run. - Attributes: + Fields: nsamples (int): The number of samples in the waveforms. subarray (SubarrayDescription): The subarray description instance. wfs_hg (np.ndarray): An array of high gain waveforms. @@ -51,15 +51,55 @@ class WaveformsContainer(ArrayDataContainer): ) + class WaveformsContainerIO(ABC) : + """ + The `WaveformsContainerIO` class provides methods for writing and loading `WaveformsContainer` instances to/from FITS files. It also includes a method for writing the subarray configuration to an HDF5 file. + + Example Usage: + # Writing a WaveformsContainer instance to a FITS file + container = WaveformsContainer() + # ... populate the container with data ... + WaveformsContainerIO.write('/path/to/output', container) + + # Loading a WaveformsContainer instance from a FITS file + container = WaveformsContainerIO.load('/path/to/input.fits') + + Main functionalities: + - Writing a `WaveformsContainer` instance to a FITS file, including the waveforms data and metadata. + - Loading a `WaveformsContainer` instance from a FITS file, including the waveforms data and metadata. + - Writing the subarray configuration to an HDF5 file. + + Methods: + - `write(path: str, containers: WaveformsContainer, **kwargs) -> None`: Writes a `WaveformsContainer` instance to a FITS file. The method also creates an HDF5 file representing the subarray configuration. + - `load(path: str) -> WaveformsContainer`: Loads a `WaveformsContainer` instance from a FITS file. The method also loads the subarray configuration from the corresponding HDF5 file. + + Fields: + None. + """ @staticmethod - def write(path : str, containers : WaveformsContainer, **kwargs): - '''method to write in an output FITS file the WaveformsContainer. Two files are created, one FITS representing the data - and one HDF5 file representing the subarray configuration + def write(path : str, containers : WaveformsContainer, **kwargs) -> None: + '''Write the WaveformsContainer data to an output FITS file. + + This method creates two files: one FITS file representing the waveform data and one HDF5 file representing the subarray configuration. Args: - path (str): the directory where you want to save data + path (str): The directory where you want to save the data. + containers (WaveformsContainer): The WaveformsContainer instance containing the data to be saved. + **kwargs: Additional keyword arguments for customization (optional). + + Keyword Args: + suffix (str, optional): A suffix to append to the output file names (default is ''). + overwrite (bool, optional): If True, overwrite the output files if they already exist (default is False). + Returns: + None: This method does not return any value. + Raises: + OSError: If there is an error while writing the FITS file. + Exception: If there is any other exception during the writing process. + Example: + waveformsContainer = WaveformsContainer() + WaveformsContainerIO.write(path, waveformsContainer, suffix="v1", overwrite=True) ''' suffix = kwargs.get("suffix","") if suffix != "" : suffix = f"_{suffix}" @@ -118,16 +158,20 @@ def write(path : str, containers : WaveformsContainer, **kwargs): raise e @staticmethod - def load(path : str) : - '''load WaveformsContainer from FITS file previously written with WaveformsContainer.write() method - Note : 2 files are loaded, the FITS one representing the waveforms data and a HDF5 file representing the subarray configuration. - This second file has to be next to the FITS file. - + def load(path : str) -> WaveformsContainer: + '''Load a WaveformsContainer from a FITS file previously written with WaveformsContainerIO.write() method. + + Note: Two files are loaded—the FITS file representing the waveform data and an HDF5 file representing the subarray configuration. + The HDF5 file should be located next to the FITS file. + Args: - path (str): path of the FITS file + path (str): The path to the FITS file containing the waveform data. Returns: - WaveformsContainer: WaveformsContainer instance + WaveformsContainer: A WaveformsContainer instance loaded from the specified file. + Example: + waveformsContainer = WaveformsContainerIO.load(path, run_number) + """ ''' log.info(f"loading from {path}") with fits.open(Path(path)) as hdul : From 2ed495c01cd65bfdaee1f416129cc5ad8a7b0c95 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 15 Sep 2023 16:33:04 +0200 Subject: [PATCH 53/62] make container module visible --- src/nectarchain/data/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/nectarchain/data/__init__.py b/src/nectarchain/data/__init__.py index 92ba4ff4..4db71b78 100644 --- a/src/nectarchain/data/__init__.py +++ b/src/nectarchain/data/__init__.py @@ -1 +1,2 @@ -from .management import * \ No newline at end of file +from .management import * +from .container import * \ No newline at end of file From 973a28c96de8959bb1a25bda4270532562bb2c03 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Fri, 15 Sep 2023 23:55:20 +0200 Subject: [PATCH 54/62] small bugfix + fix unit test for makers --- .../data/container/chargesContainer.py | 4 +- .../data/container/tests/test_charge.py | 43 +++++++++---------- .../data/container/tests/test_waveforms.py | 30 +++++++------ .../data/container/waveformsContainer.py | 17 +++++--- .../gain/tests/test_FlatFieldSPEMakers.py | 14 +++--- .../makers/tests/test_chargesMakers.py | 2 +- src/nectarchain/makers/tests/test_core.py | 3 +- .../makers/tests/test_waveformsMakers.py | 2 +- 8 files changed, 62 insertions(+), 53 deletions(-) diff --git a/src/nectarchain/data/container/chargesContainer.py b/src/nectarchain/data/container/chargesContainer.py index d23634f4..90bac147 100644 --- a/src/nectarchain/data/container/chargesContainer.py +++ b/src/nectarchain/data/container/chargesContainer.py @@ -4,7 +4,7 @@ log.handlers = logging.getLogger('__main__').handlers import numpy as np -from ctapipe.container import Field +from ctapipe.containers import Field from abc import ABC import os from pathlib import Path @@ -25,7 +25,7 @@ class ChargesContainer(ArrayDataContainer): """ charges_hg = Field( - type=np.ndarray(), + type=np.ndarray, description='The high gain charges' ) charges_lg = Field( diff --git a/src/nectarchain/data/container/tests/test_charge.py b/src/nectarchain/data/container/tests/test_charge.py index b15ebaa1..da559baa 100644 --- a/src/nectarchain/data/container/tests/test_charge.py +++ b/src/nectarchain/data/container/tests/test_charge.py @@ -1,19 +1,18 @@ -from ..chargesContainer import ChargesContainer,ChargesContainerIO -from ..waveformsContainer import WaveformsContainer -from ....makers import ChargesMaker +from nectarchain.data.container import ChargesContainer,ChargesContainerIO +from nectarchain.makers import ChargesMaker import glob import numpy as np def create_fake_chargeContainer() : - nevents = TestChargesContainer.nevents, - npixels = TestChargesContainer.npixels, - rng = np.random.default_rng(), + nevents = TestChargesContainer.nevents + npixels = TestChargesContainer.npixels + rng = np.random.default_rng() return ChargesContainer( pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]), nevents =nevents, npixels =npixels, - charge_hg = rng.integers(low=0, high=1000, size= (nevents,npixels)), - charge_lg = rng.integers(low=0, high=1000, size= (nevents,npixels)), + charges_hg = rng.integers(low=0, high=1000, size= (nevents,npixels)), + charges_lg = rng.integers(low=0, high=1000, size= (nevents,npixels)), peak_hg = rng.integers(low=0, high=60, size= (nevents,npixels)), peak_lg = rng.integers(low=0, high=60, size= (nevents,npixels)), run_number = TestChargesContainer.run_number, @@ -40,14 +39,14 @@ def test_create_charge_container(self): nevents = TestChargesContainer.nevents npixels = TestChargesContainer.npixels run_number = TestChargesContainer.run_number - charge_hg = np.random.randn(nevents,npixels) - charge_lg = np.random.randn(nevents,npixels) + charges_hg = np.random.randn(nevents,npixels) + charges_lg = np.random.randn(nevents,npixels) peak_hg = np.random.randn(nevents,npixels) peak_lg = np.random.randn(nevents,npixels) method = 'FullWaveformSum' charge_container = ChargesContainer( - charge_hg = charge_hg , - charge_lg = charge_lg, + charges_hg = charges_hg , + charges_lg = charges_lg, peak_hg = peak_hg, peak_lg = peak_lg, run_number = run_number, @@ -57,8 +56,8 @@ def test_create_charge_container(self): method = method ) - assert np.allclose(charge_container.charges_hg,charge_hg) - assert np.allclose(charge_container.charges_lg,charge_lg) + assert np.allclose(charge_container.charges_hg,charges_hg) + assert np.allclose(charge_container.charges_lg,charges_lg) assert np.allclose(charge_container.peak_hg,peak_hg) assert np.allclose(charge_container.peak_lg,peak_lg) assert charge_container.run_number == run_number @@ -79,7 +78,7 @@ def test_create_charge_container(self): # Tests that the ChargeContainer object can be written to a file and the file is created. def test_write_charge_container(self, tmp_path = "/tmp"): charge_container = create_fake_chargeContainer() - tmp_path += f"/{np.random.randn(1)}" + tmp_path += f"/{np.random.randn(1)[0]}" ChargesContainerIO.write(tmp_path,charge_container) @@ -88,7 +87,7 @@ def test_write_charge_container(self, tmp_path = "/tmp"): # Tests that a ChargeContainer object can be loaded from a file and the object is correctly initialized. def test_load_charge_container(self, tmp_path = "/tmp"): charge_container = create_fake_chargeContainer() - tmp_path += f"/{np.random.randn(1)}" + tmp_path += f"/{np.random.randn(1)[0]}" ChargesContainerIO.write(tmp_path,charge_container) @@ -118,15 +117,15 @@ def test_access_properties(self): pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) nevents = 40 npixels = 10 - charge_hg = np.random.randn(nevents,npixels) - charge_lg = np.random.randn(nevents,npixels) + charges_hg = np.random.randn(nevents,npixels) + charges_lg = np.random.randn(nevents,npixels) peak_hg = np.random.randn(nevents,npixels) peak_lg = np.random.randn(nevents,npixels) run_number = 1234 method = 'FullWaveformSum' charge_container = ChargesContainer( - charge_hg = charge_hg , - charge_lg = charge_lg, + charges_hg = charges_hg , + charges_lg = charges_lg, peak_hg = peak_hg, peak_lg = peak_lg, run_number = run_number, @@ -140,6 +139,4 @@ def test_access_properties(self): assert charge_container.pixels_id.tolist() == pixels_id.tolist() assert charge_container.npixels == npixels assert charge_container.nevents == nevents - assert charge_container.method == method - assert charge_container.multiplicity.shape == (nevents,) - assert charge_container.trig_pattern.shape == (nevents,4) \ No newline at end of file + assert charge_container.method == method \ No newline at end of file diff --git a/src/nectarchain/data/container/tests/test_waveforms.py b/src/nectarchain/data/container/tests/test_waveforms.py index d3689f79..a74668c6 100644 --- a/src/nectarchain/data/container/tests/test_waveforms.py +++ b/src/nectarchain/data/container/tests/test_waveforms.py @@ -1,13 +1,15 @@ -from ..waveformsContainer import WaveformsContainer,WaveformsContainerIO -from ....makers import WaveformsMaker +from nectarchain.data.container import WaveformsContainer,WaveformsContainerIO +from nectarchain.makers import WaveformsMaker +from ctapipe.instrument import SubarrayDescription import glob import numpy as np def create_fake_waveformsContainer() : - nevents = TestWaveformsContainer.nevents, - npixels = TestWaveformsContainer.npixels, - nsamples = TestWaveformsContainer.nsamples, - rng = np.random.default_rng(), + nevents = TestWaveformsContainer.nevents + npixels = TestWaveformsContainer.npixels + nsamples = TestWaveformsContainer.nsamples + rng = np.random.default_rng() + faked_subarray = SubarrayDescription(name = 'TEST') return WaveformsContainer( pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]), @@ -17,6 +19,7 @@ def create_fake_waveformsContainer() : wfs_lg = rng.integers(low=0, high=1000, size= (nevents,npixels,nsamples)), run_number = TestWaveformsContainer.run_number, camera = 'TEST', + subarray = faked_subarray, broken_pixels_hg = rng.integers(low=0, high=1, size= (nevents,npixels)), broken_pixels_lg = rng.integers(low=0, high=1, size= (nevents,npixels)), ucts_timestamp =rng.integers(low=0, high=100, size= (nevents)), @@ -43,7 +46,7 @@ def test_create_waveform_container(self): # Tests that the ChargeContainer object can be written to a file and the file is created. def test_write_waveform_container(self, tmp_path = "/tmp"): waveform_container = create_fake_waveformsContainer() - tmp_path += f"/{np.random.randn(1)}" + tmp_path += f"/{np.random.randn(1)[0]}" WaveformsContainerIO.write(tmp_path,waveform_container) @@ -52,15 +55,15 @@ def test_write_waveform_container(self, tmp_path = "/tmp"): # Tests that a ChargeContainer object can be loaded from a file and the object is correctly initialized. def test_load_waveform_container(self, tmp_path = "/tmp"): waveform_container = create_fake_waveformsContainer() - tmp_path += f"/{np.random.randn(1)}" + tmp_path += f"/{np.random.randn(1)[0]}" WaveformsContainerIO.write(tmp_path,waveform_container) - loaded_waveform_container = WaveformsContainerIO.load(tmp_path,TestWaveformsContainer.run_number ) + loaded_waveform_container = WaveformsContainerIO.load(tmp_path,TestWaveformsContainer.run_number) assert isinstance(loaded_waveform_container, WaveformsContainer) - assert np.allclose(loaded_waveform_container.wfs_hg,waveform_container.charges_hg) - assert np.allclose(loaded_waveform_container.wfs_lg,waveform_container.charges_lg) + assert np.allclose(loaded_waveform_container.wfs_hg,waveform_container.wfs_hg) + assert np.allclose(loaded_waveform_container.wfs_lg,waveform_container.wfs_lg) assert loaded_waveform_container.run_number == waveform_container.run_number assert loaded_waveform_container.pixels_id.tolist() == waveform_container.pixels_id.tolist() assert loaded_waveform_container.nevents == waveform_container.nevents @@ -73,4 +76,7 @@ def test_sort_waveform_container(self): sorted_waveform_container = WaveformsMaker.sort(waveform_container) - assert sorted_waveform_container.event_id.tolist() == sorted(waveform_container.event_id.tolist()) \ No newline at end of file + assert sorted_waveform_container.event_id.tolist() == sorted(waveform_container.event_id.tolist()) + +if __name__ == "__main__" : + TestWaveformsContainer().test_create_waveform_container() \ No newline at end of file diff --git a/src/nectarchain/data/container/waveformsContainer.py b/src/nectarchain/data/container/waveformsContainer.py index 57563e80..18ae8465 100644 --- a/src/nectarchain/data/container/waveformsContainer.py +++ b/src/nectarchain/data/container/waveformsContainer.py @@ -158,7 +158,7 @@ def write(path : str, containers : WaveformsContainer, **kwargs) -> None: raise e @staticmethod - def load(path : str) -> WaveformsContainer: + def load(path : str, run_number : int, **kwargs) -> WaveformsContainer: '''Load a WaveformsContainer from a FITS file previously written with WaveformsContainerIO.write() method. Note: Two files are loaded—the FITS file representing the waveform data and an HDF5 file representing the subarray configuration. @@ -166,15 +166,22 @@ def load(path : str) -> WaveformsContainer: Args: path (str): The path to the FITS file containing the waveform data. - + **kwargs: Additional keyword arguments. + explicit_filename (str): If provided, the explicit filename to load. Returns: WaveformsContainer: A WaveformsContainer instance loaded from the specified file. Example: waveformsContainer = WaveformsContainerIO.load(path, run_number) - """ ''' + if kwargs.get("explicit_filename", False): + filename = kwargs.get("explicit_filename") + log.info(f"Loading {filename}") + else: + log.info(f"Loading in {path} run number {run_number}") + filename = Path(path) / f"waveforms_run{run_number}.fits" + log.info(f"loading from {path}") - with fits.open(Path(path)) as hdul : + with fits.open(filename) as hdul : containers = WaveformsContainer() containers.run_number = hdul[0].header['RUN'] @@ -184,7 +191,7 @@ def load(path : str) -> WaveformsContainer: containers.camera = hdul[0].header['CAMERA'] - containers.subarray = SubarrayDescription.from_hdf(Path(path.replace('waveforms_','subarray_').replace('fits','hdf5'))) + containers.subarray = SubarrayDescription.from_hdf(Path(filename._str.replace('waveforms_','subarray_').replace('fits','hdf5'))) containers.pixels_id = hdul[0].data diff --git a/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py index 376e1f68..3b3a0c52 100644 --- a/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py @@ -2,7 +2,7 @@ from nectarchain.makers.calibration.gain.parameters import Parameter,Parameters from nectarchain.makers.calibration.gain import FlatFieldSingleHHVSPEMaker,FlatFieldSingleHHVStdSPEMaker import astropy.units as u -from nectarchain.data.container import ChargeContainer +from nectarchain.data.container import ChargesContainer import numpy as np import pytest @@ -16,14 +16,14 @@ def create_fake_chargeContainer() : nevents = 40 npixels = 10 rng = np.random.default_rng() - charge_hg = rng.integers(low=0, high=1000, size= (nevents,npixels)) - charge_lg = rng.integers(low=0, high=1000, size= (nevents,npixels)) + charges_hg = rng.integers(low=0, high=1000, size= (nevents,npixels)) + charges_lg = rng.integers(low=0, high=1000, size= (nevents,npixels)) peak_hg = rng.integers(low=0, high=60, size= (nevents,npixels)) peak_lg = rng.integers(low=0, high=60, size= (nevents,npixels)) run_number = 1234 - return ChargeContainer( - charge_hg = charge_hg , - charge_lg = charge_lg, + return ChargesContainer( + charges_hg = charges_hg , + charges_lg = charges_lg, peak_hg = peak_hg, peak_lg = peak_lg, run_number = run_number, @@ -100,7 +100,7 @@ def test_create_instance_invalid_input(self): # Tests that calling create_from_chargeContainer method with valid input parameters is successful def test_create_from_ChargeContainer_valid_input(self): chargeContainer = create_fake_chargeContainer() - maker = FlatFieldSingleHHVSPEMaker.create_from_chargeContainer(chargeContainer) + maker = FlatFieldSingleHHVSPEMaker.create_from_chargesContainer(chargeContainer) assert isinstance(maker, FlatFieldSingleHHVSPEMaker) diff --git a/src/nectarchain/makers/tests/test_chargesMakers.py b/src/nectarchain/makers/tests/test_chargesMakers.py index a83b9e52..659413ea 100644 --- a/src/nectarchain/makers/tests/test_chargesMakers.py +++ b/src/nectarchain/makers/tests/test_chargesMakers.py @@ -117,7 +117,7 @@ def test_write_load_container(self) : max_events = TestChargesMaker.max_events) chargesContainer_list = chargesMaker.make() ChargesContainerIO.write("/tmp/test_charge_container/",chargesContainer_list[0],overwrite = True) - loaded_charge = ChargesContainerIO.load(f"/tmp/test_charge_container/",run_number = TestChargesMaker.run_number) + loaded_charge = ChargesContainerIO.load(f"/tmp/test_charge_container",run_number = TestChargesMaker.run_number) assert np.array_equal(chargesContainer_list[0].charges_hg,loaded_charge.charges_hg) if __name__ == '__main__' : diff --git a/src/nectarchain/makers/tests/test_core.py b/src/nectarchain/makers/tests/test_core.py index 242dabef..d1aa0e05 100644 --- a/src/nectarchain/makers/tests/test_core.py +++ b/src/nectarchain/makers/tests/test_core.py @@ -1,6 +1,5 @@ -from nectarchain.makers import ChargesMaker,WaveformsMaker +from nectarchain.makers import ChargesMaker,WaveformsMaker,ArrayDataMaker from nectarchain.data.container import ChargesContainer -from ..core import ArrayDataMaker import logging logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level = logging.DEBUG) log = logging.getLogger(__name__) diff --git a/src/nectarchain/makers/tests/test_waveformsMakers.py b/src/nectarchain/makers/tests/test_waveformsMakers.py index 06a222eb..9a983926 100644 --- a/src/nectarchain/makers/tests/test_waveformsMakers.py +++ b/src/nectarchain/makers/tests/test_waveformsMakers.py @@ -81,7 +81,7 @@ def test_write_load_container(self) : max_events = TestWaveformsMaker.max_events) waveformsContainer_list = waveformsMaker.make() WaveformsContainerIO.write("/tmp/test_wfs_container/",waveformsContainer_list[0],overwrite = True) - loaded_wfs = WaveformsContainerIO.load(f"/tmp/test_wfs_container/waveforms_run{TestWaveformsMaker.run_number}.fits") + loaded_wfs = WaveformsContainerIO.load(f"/tmp/test_wfs_container",TestWaveformsMaker.run_number) assert np.array_equal(waveformsContainer_list[0].wfs_hg,loaded_wfs.wfs_hg) def test_create_from_events_list(self) : From 9aa08a8cc3eb93f334351cbf0be88d0f7a5d0d75 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Sun, 17 Sep 2023 05:19:00 +0200 Subject: [PATCH 55/62] docstrings makers module --- .../calibration/gain/FlatFieldSPEMakers.py | 734 ++++++++++++------ .../calibration/gain/PhotoStatisticMakers.py | 454 ++++++++--- .../makers/calibration/gain/parameters.py | 8 +- src/nectarchain/makers/chargesMakers.py | 365 ++++++--- src/nectarchain/makers/core.py | 513 +++++++++--- src/nectarchain/makers/waveformsMakers.py | 136 +++- 6 files changed, 1693 insertions(+), 517 deletions(-) diff --git a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py index 18a670a2..4529838b 100644 --- a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py @@ -13,6 +13,8 @@ import yaml import time +from typing import Tuple + import matplotlib.pyplot as plt from matplotlib.patches import Rectangle @@ -91,161 +93,232 @@ class FlatFieldSPEMaker(GainMaker) : #constructors def __init__(self,*args,**kwargs) -> None: + """ + Initializes the FlatFieldSPEMaker class. + + Parameters: + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Returns: + None + """ super().__init__(*args,**kwargs) self.__parameters = Parameters() -#getters and setters @property - def npixels(self) : return len(self._pixels_id) + def npixels(self): + """ + Returns the number of pixels. + + Returns: + int: The number of pixels. + """ + return len(self._pixels_id) + @property - def parameters(self) : return copy.deepcopy(self.__parameters) + def parameters(self): + """ + Returns a deep copy of the internal parameters. + + Returns: + dict: A deep copy of the internal parameters. + """ + return copy.deepcopy(self.__parameters) + @property - def _parameters(self) : return self.__parameters + def _parameters(self): + """ + Returns the internal parameters. + + Returns: + dict: The internal parameters. + """ + return self.__parameters #methods - def read_param_from_yaml(self,parameters_file,only_update = False) -> None: - with open(f"{os.path.dirname(os.path.abspath(__file__))}/{parameters_file}") as parameters : - param = yaml.safe_load(parameters) - if only_update : - for i,name in enumerate(self.__parameters.parnames) : - dico = param.get(name,False) - if dico : + def read_param_from_yaml(self, parameters_file, only_update=False) -> None: + """ + Reads parameters from a YAML file and updates the internal parameters of the FlatFieldSPEMaker class. + + Args: + parameters_file (str): The name of the YAML file containing the parameters. + only_update (bool, optional): If True, only the parameters that exist in the YAML file will be updated. Default is False. + + Returns: + None + """ + with open(f"{os.path.dirname(os.path.abspath(__file__))}/{parameters_file}") as parameters: + param = yaml.safe_load(parameters) + if only_update: + for i, name in enumerate(self.__parameters.parnames): + dico = param.get(name, False) + if dico: self._parameters.parameters[i].value = dico.get('value') - self._parameters.parameters[i].min = dico.get("min",np.nan) - self._parameters.parameters[i].max = dico.get("max",np.nan) - else : - for name,dico in param.items() : + self._parameters.parameters[i].min = dico.get("min", np.nan) + self._parameters.parameters[i].max = dico.get("max", np.nan) + else: + for name, dico in param.items(): setattr(self, f"__{name}", Parameter( - name = name, - value = dico["value"], - min = dico.get("min",np.nan), - max = dico.get("max",np.nan), - unit = dico.get("unit",u.dimensionless_unscaled) - )) + name=name, + value=dico["value"], + min=dico.get("min", np.nan), + max=dico.get("max", np.nan), + unit=dico.get("unit", u.dimensionless_unscaled) + )) self._parameters.append(eval(f"self.__{name}")) @staticmethod - def _update_parameters(parameters,charge,counts,**kwargs) : - try : - coeff_ped,coeff_mean = __class__._get_mean_gaussian_fit(charge,counts,**kwargs) + def _update_parameters(parameters: Parameters, charge: np.ndarray, counts: np.ndarray, **kwargs) -> Parameters: + """ + Update the parameters of the FlatFieldSPEMaker class based on the input charge and counts data. + + Args: + parameters (Parameters): An instance of the Parameters class that holds the internal parameters of the FlatFieldSPEMaker class. + charge (np.ndarray): An array of charge values. + counts (np.ndarray): An array of corresponding counts values. + **kwargs: Additional keyword arguments. + + Returns: + Parameters: The updated parameters object with the pedestal and mean values and their corresponding limits. + """ + try: + coeff_ped, coeff_mean = __class__._get_mean_gaussian_fit(charge, counts, **kwargs) pedestal = parameters['pedestal'] pedestal.value = coeff_ped[1] pedestal.min = coeff_ped[1] - coeff_ped[2] pedestal.max = coeff_ped[1] + coeff_ped[2] - log.debug(f"pedestal updated : {pedestal}") + log.debug(f"pedestal updated: {pedestal}") pedestalWidth = parameters["pedestalWidth"] pedestalWidth.value = pedestal.max - pedestal.value pedestalWidth.max = 3 * pedestalWidth.value - log.debug(f"pedestalWidth updated : {pedestalWidth.value}") - - if (coeff_mean[1] - pedestal.value < 0) or ((coeff_mean[1] - coeff_mean[2]) - pedestal.max < 0) : raise MeanValueError("mean gaussian fit not good") + log.debug(f"pedestalWidth updated: {pedestalWidth.value}") + + if (coeff_mean[1] - pedestal.value < 0) or ((coeff_mean[1] - coeff_mean[2]) - pedestal.max < 0): + raise MeanValueError("mean gaussian fit not good") mean = parameters['mean'] mean.value = coeff_mean[1] - pedestal.value mean.min = (coeff_mean[1] - coeff_mean[2]) - pedestal.max mean.max = (coeff_mean[1] + coeff_mean[2]) - pedestal.min - log.debug(f"mean updated : {mean}") - except MeanValueError as e : - log.warning(e,exc_info=True) + log.debug(f"mean updated: {mean}") + except MeanValueError as e: + log.warning(e, exc_info=True) log.warning("mean parameters limits and starting value not changed") - except Exception as e : - log.warning(e, exc_info = True) + except Exception as e: + log.warning(e, exc_info=True) log.warning("pedestal and mean parameters limits and starting value not changed") - return parameters @staticmethod - def _get_mean_gaussian_fit(charge, counts ,extension = "",**kwargs): - #charge = charge_in.data[~histo_in.mask] - #histo = histo_in.data[~histo_in.mask] + def _get_mean_gaussian_fit(charge: np.ndarray, counts: np.ndarray, extension: str = "", **kwargs) -> Tuple[np.ndarray, np.ndarray]: + """ + Perform a Gaussian fit on the data to determine the pedestal and mean values. + + Args: + charge (np.ndarray): An array of charge values. + counts (np.ndarray): An array of corresponding counts. + extension (str, optional): An extension string. Defaults to "". + **kwargs: Additional keyword arguments. + + Returns: + Tuple[np.ndarray, np.ndarray]: A tuple of fit coefficients for the pedestal and mean. + + Example Usage: + flat_field_maker = FlatFieldSPEMaker() + charge = np.array([1, 2, 3, 4, 5]) + counts = np.array([10, 20, 30, 40, 50]) + coeff, coeff_mean = flat_field_maker._get_mean_gaussian_fit(charge, counts) + print(coeff) # Output: [norm,peak_value, peak_width] + print(coeff_mean) # Output: [norm,peak_value_mean, peak_width_mean] + """ windows_lenght = __class__._Windows_lenght order = __class__._Order histo_smoothed = savgol_filter(counts, windows_lenght, order) - peaks = find_peaks(histo_smoothed,10) + peaks = find_peaks(histo_smoothed, 10) peak_max = np.argmax(histo_smoothed[peaks[0]]) - peak_pos,peak_value = charge[peaks[0][peak_max]], counts[peaks[0][peak_max]] - coeff, _ = curve_fit(weight_gaussian, charge[:peaks[0][peak_max]], histo_smoothed[:peaks[0][peak_max]],p0 = [peak_value,peak_pos,1]) - if log.getEffectiveLevel() == logging.DEBUG and kwargs.get("display",False) : - log.debug('plotting figures with prefit parameters computation') - fig,ax = plt.subplots(1,1,figsize = (5,5)) - ax.errorbar(charge,counts,np.sqrt(counts),zorder=0,fmt=".",label = "data") - ax.plot(charge,histo_smoothed,label = f'smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})') - ax.plot(charge,weight_gaussian(charge,coeff[0],coeff[1],coeff[2]),label = 'gaussian fit of the pedestal, left tail only') - ax.set_xlim([peak_pos - 500,None]) - ax.vlines(coeff[1],0,peak_value,label = f'pedestal initial value = {coeff[1]:.0f}',color = 'red') - ax.add_patch(Rectangle((coeff[1]-coeff[2], 0), 2 * coeff[2], peak_value,fc=to_rgba('red', 0.5))) + peak_pos, peak_value = charge[peaks[0][peak_max]], counts[peaks[0][peak_max]] + coeff, _ = curve_fit(weight_gaussian, charge[:peaks[0][peak_max]], histo_smoothed[:peaks[0][peak_max]], p0=[peak_value, peak_pos, 1]) + if log.getEffectiveLevel() == logging.DEBUG and kwargs.get("display", False): + log.debug('plotting figures with prefit parameters computation') + fig, ax = plt.subplots(1, 1, figsize=(5, 5)) + ax.errorbar(charge, counts, np.sqrt(counts), zorder=0, fmt=".", label="data") + ax.plot(charge, histo_smoothed, label=f'smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})') + ax.plot(charge, weight_gaussian(charge, coeff[0], coeff[1], coeff[2]), label='gaussian fit of the pedestal, left tail only') + ax.set_xlim([peak_pos - 500, None]) + ax.vlines(coeff[1], 0, peak_value, label=f'pedestal initial value = {coeff[1]:.0f}', color='red') + ax.add_patch(Rectangle((coeff[1] - coeff[2], 0), 2 * coeff[2], peak_value, fc=to_rgba('red', 0.5))) ax.set_xlabel("Charge (ADC)", size=15) ax.set_ylabel("Events", size=15) ax.legend(fontsize=7) - os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/",exist_ok=True) + os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/", exist_ok=True) fig.savefig(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/initialization_pedestal_pixel{extension}_{os.getpid()}.pdf") fig.clf() plt.close(fig) - del fig,ax - #nosw find SPE peak excluding pedestal data - mask = charge > coeff[1]+3*coeff[2] + del fig, ax + mask = charge > coeff[1] + 3 * coeff[2] peaks_mean = find_peaks(histo_smoothed[mask]) - peak_max_mean = np.argmax(histo_smoothed[mask][peaks_mean[0]]) - peak_pos_mean,peak_value_mean = charge[mask][peaks_mean[0][peak_max_mean]], histo_smoothed[mask][peaks_mean[0][peak_max_mean]] - mask = (charge > ((coeff[1]+peak_pos_mean)/2)) * (charge < (peak_pos_mean + (peak_pos_mean-coeff[1])/2)) - coeff_mean, _ = curve_fit(weight_gaussian, charge[mask], histo_smoothed[mask],p0 = [peak_value_mean,peak_pos_mean,1]) - if log.getEffectiveLevel() == logging.DEBUG and kwargs.get("display",False) : - log.debug('plotting figures with prefit parameters computation') - fig,ax = plt.subplots(1,1,figsize = (5,5)) - ax.errorbar(charge,counts,np.sqrt(counts),zorder=0,fmt=".",label = "data") - ax.plot(charge,histo_smoothed,label = f'smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})') - ax.plot(charge,weight_gaussian(charge,coeff_mean[0],coeff_mean[1],coeff_mean[2]),label = 'gaussian fit of the SPE') - ax.vlines(coeff_mean[1],0,peak_value,label = f'mean initial value = {coeff_mean[1] - coeff[1]:.0f}',color = "red") - ax.add_patch(Rectangle((coeff_mean[1]-coeff_mean[2], 0), 2 * coeff_mean[2], peak_value_mean,fc=to_rgba('red', 0.5))) - ax.set_xlim([peak_pos - 500,None]) + peak_pos_mean, peak_value_mean = charge[mask][peaks_mean[0][peak_max_mean]], histo_smoothed[mask][peaks_mean[0][peak_max_mean]] + mask = (charge > ((coeff[1] + peak_pos_mean) / 2)) * (charge < (peak_pos_mean + (peak_pos_mean - coeff[1]) / 2)) + coeff_mean, _ = curve_fit(weight_gaussian, charge[mask], histo_smoothed[mask], p0=[peak_value_mean, peak_pos_mean, 1]) + if log.getEffectiveLevel() == logging.DEBUG and kwargs.get("display", False): + log.debug('plotting figures with prefit parameters computation') + fig, ax = plt.subplots(1, 1, figsize=(5, 5)) + ax.errorbar(charge, counts, np.sqrt(counts), zorder=0, fmt=".", label="data") + ax.plot(charge, histo_smoothed, label=f'smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})') + ax.plot(charge, weight_gaussian(charge, coeff_mean[0], coeff_mean[1], coeff_mean[2]), label='gaussian fit of the SPE') + ax.vlines(coeff_mean[1], 0, peak_value, label=f'mean initial value = {coeff_mean[1] - coeff[1]:.0f}', color="red") + ax.add_patch(Rectangle((coeff_mean[1] - coeff_mean[2], 0), 2 * coeff_mean[2], peak_value_mean, fc=to_rgba('red', 0.5))) + ax.set_xlim([peak_pos - 500, None]) ax.set_xlabel("Charge (ADC)", size=15) ax.set_ylabel("Events", size=15) ax.legend(fontsize=7) - os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/",exist_ok=True) + os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/", exist_ok=True) fig.savefig(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/initialization_mean_pixel{extension}_{os.getpid()}.pdf") fig.clf() plt.close(fig) - del fig,ax - + del fig, ax return coeff, coeff_mean - def _update_table_from_parameters(self) -> None: - for param in self._parameters.parameters : - if not(param.name in self._results.colnames) : - self._results.add_column(Column(data = np.empty((self.npixels),dtype = np.float64),name = param.name,unit = param.unit)) - self._results.add_column(Column(data = np.empty((self.npixels),dtype = np.float64),name = f"{param.name}_error",unit = param.unit)) + def _update_table_from_parameters(self) -> None: + """ + Update the result table based on the parameters of the FlatFieldSPEMaker class. + This method adds columns to the table for each parameter and its corresponding error. + """ + + for param in self._parameters.parameters: + if not (param.name in self._results.colnames): + self._results.add_column(Column(data=np.empty((self.npixels), dtype=np.float64), name=param.name, unit=param.unit)) + self._results.add_column(Column(data=np.empty((self.npixels), dtype=np.float64), name=f"{param.name}_error", unit=param.unit)) class FlatFieldSingleHHVSPEMaker(FlatFieldSPEMaker) : """ - class to perform fit of the SPE signal with all free parameters + This class represents a FlatFieldSingleHHVSPEMaker object. + Args: + charge (np.ma.masked_array or array-like): The charge data. + counts (np.ma.masked_array or array-like): The counts data. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. Attributes: - - __parameters_file: Private field that stores the name of the parameters file. - - __fit_array: Private field that stores the fit instances. - - _reduced_name: Protected field that stores the name of the reduced data. - - __nproc_default: Private field that stores the default number of processes for multiprocessing. - - __chunksize_default: Private field that stores the default chunk size for multiprocessing. - - Members: - - __charge: Private field that stores the charge data. - - __counts: Private field that stores the counts data. - - __pedestal: Private field that stores the pedestal parameter. - - _parameters: Protected field that stores the parameters. - - _results: Protected field that stores the fit results. - + __charge (np.ma.masked_array): The charge data as a masked array. + __counts (np.ma.masked_array): The counts data as a masked array. + __pedestal (Parameter): The pedestal value. + _parameters (list): List of parameters. + __parameters_file (str): The path to the parameters file. + _results (Table): Table of results. Methods: - - __init__(self, charge, counts, *args, **kwargs): Constructor method that initializes the instance with the charge and counts data. - - create_from_chargeContainer(cls, signal, **kwargs): Class method that creates an instance from a charge container. - - create_from_run_number(cls, run_number, **kwargs): Class method that creates an instance from a run number. - - make(self, pixels_id=None, multiproc=True, display=True, **kwargs): Method that performs the fit on the specified pixels and returns the fit results. - - display(self, pixels_id, **kwargs): Method that plots the fit for the specified pixels. + __init__: Initializes the FlatFieldSingleHHVSPEMaker object. + create_from_chargesContainer: Creates an instance of FlatFieldSingleHHVSPEMaker using charge and counts data from a ChargesContainer object. + create_from_run_number(cls, run_number, **kwargs): Class method that creates an instance from a run number. + make(self, pixels_id=None, multiproc=True, display=True, **kwargs): Method that performs the fit on the specified pixels and returns the fit results. + display(self, pixels_id, **kwargs): Method that plots the fit for the specified pixels. """ - __parameters_file = 'parameters_signal.yaml' __fit_array = None _reduced_name = "FlatFieldSingleSPE" @@ -254,79 +327,136 @@ class to perform fit of the SPE signal with all free parameters #constructors - def __init__(self,charge,counts,*args,**kwargs) -> None: - super().__init__(*args,**kwargs) - if isinstance(charge,np.ma.masked_array) : + def __init__(self, charge, counts, *args, **kwargs) -> None: + """ + Initializes the FlatFieldSingleHHVSPEMaker object. + Args: + charge (np.ma.masked_array or array-like): The charge data. + counts (np.ma.masked_array or array-like): The counts data. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + """ + super().__init__(*args, **kwargs) + if isinstance(charge, np.ma.masked_array): self.__charge = charge - else : + else: self.__charge = np.ma.asarray(charge) - if isinstance(counts,np.ma.masked_array) : + if isinstance(counts, np.ma.masked_array): self.__counts = counts - else : + else: self.__counts = np.ma.asarray(counts) - - self.__pedestal = Parameter(name = "pedestal", - value = (np.min(self.__charge) + np.sum(self.__charge * self.__counts)/(np.sum(self.__counts)))/2, - min = np.min(self.__charge), - max = np.sum(self.__charge*self.__counts)/np.sum(self.__counts), - unit = u.dimensionless_unscaled) - - + self.__pedestal = Parameter( + name="pedestal", + value=(np.min(self.__charge) + np.sum(self.__charge * self.__counts) / np.sum(self.__counts)) / 2, + min=np.min(self.__charge), + max=np.sum(self.__charge * self.__counts) / np.sum(self.__counts), + unit=u.dimensionless_unscaled + ) self._parameters.append(self.__pedestal) - - self.read_param_from_yaml(kwargs.get('parameters_file',self.__parameters_file)) - + self.read_param_from_yaml(kwargs.get('parameters_file', self.__parameters_file)) self._update_table_from_parameters() - - self._results.add_column(Column(np.zeros((self.npixels),dtype = np.float64),"likelihood",unit = u.dimensionless_unscaled)) - self._results.add_column(Column(np.zeros((self.npixels),dtype = np.float64),"pvalue",unit = u.dimensionless_unscaled)) - + self._results.add_column(Column(np.zeros((self.npixels), dtype=np.float64), "likelihood", + unit=u.dimensionless_unscaled)) + self._results.add_column(Column(np.zeros((self.npixels), dtype=np.float64), "pvalue", + unit=u.dimensionless_unscaled)) @classmethod - def create_from_chargesContainer(cls, signal : ChargesContainer,**kwargs) : - histo = ChargesMaker.histo_hg(signal,autoscale = True) - return cls(charge = histo[1],counts = histo[0],pixels_id = signal.pixels_id,**kwargs) - + def create_from_chargesContainer(cls, signal: ChargesContainer, **kwargs): + """ + Creates an instance of FlatFieldSingleHHVSPEMaker using charge and counts data from a ChargesContainer object. + Args: + signal (ChargesContainer): The ChargesContainer object. + **kwargs: Additional keyword arguments. + Returns: + FlatFieldSingleHHVSPEMaker: An instance of FlatFieldSingleHHVSPEMaker. + """ + histo = ChargesMaker.histo_hg(signal, autoscale=True) + return cls(charge=histo[1], counts=histo[0], pixels_id=signal.pixels_id, **kwargs) + @classmethod def create_from_run_number(cls, run_number : int, **kwargs) : raise NotImplementedError("Need to implement here the use of the WaveformsMaker and ChargesMaker to produce the chargesContainer to be pass into the __ini__") #getters and setters @property - def charge(self) : return copy.deepcopy(self.__charge) + def charge(self) : + """ + Returns a deep copy of the __charge attribute. + """ + return copy.deepcopy(self.__charge) + @property - def _charge(self) : return self.__charge + def _charge(self) : + """ + Returns the __charge attribute. + """ + return self.__charge @property - def counts(self) : return copy.deepcopy(self.__counts) + def counts(self) : + """ + Returns a deep copy of the __counts attribute. + """ + return copy.deepcopy(self.__counts) + @property - def _counts(self) : return self.__counts + def _counts(self) : + """ + Returns the __counts attribute. + """ + return self.__counts #methods - def _fill_results_table_from_dict(self,dico,pixels_id) -> None: - chi2_sig = signature(__class__.cost(self._charge,self._counts)) - for i in range(len(pixels_id)) : - values = dico[i].get(f"values_{i}",None) - errors = dico[i].get(f"errors_{i}",None) - if not((values is None) or (errors is None)) : + def _fill_results_table_from_dict(self, dico: dict, pixels_id: np.ndarray) -> None: + """ + Populates the results table with fit values and errors for each pixel based on the dictionary provided as input. + + Args: + dico (dict): A dictionary containing fit values and errors for each pixel. + pixels_id (np.ndarray): An array of pixel IDs. + + Returns: + None + """ + chi2_sig = signature(__class__.cost(self._charge, self._counts)) + for i in range(len(pixels_id)): + values = dico[i].get(f"values_{i}", None) + errors = dico[i].get(f"errors_{i}", None) + if not ((values is None) or (errors is None)): index = np.argmax(self._results["pixels_id"] == pixels_id[i]) - if len(values) != len(chi2_sig.parameters) : + if len(values) != len(chi2_sig.parameters): e = Exception("the size out the minuit output parameters values array does not fit the signature of the minimized cost function") - log.error(e,exc_info=True) + log.error(e, exc_info=True) raise e - for j,key in enumerate(chi2_sig.parameters) : + for j, key in enumerate(chi2_sig.parameters): self._results[key][index] = values[j] self._results[f"{key}_error"][index] = errors[j] - if key == 'mean' : + if key == 'mean': self._high_gain[index] = values[j] - self._results[f"high_gain_error"][index] = [errors[j],errors[j]] + self._results[f"high_gain_error"][index] = [errors[j], errors[j]] self._results[f"high_gain"][index] = values[j] self._results['is_valid'][index] = True self._results["likelihood"][index] = __class__.__fit_array[i].fcn(__class__.__fit_array[i].values) ndof = self._counts.data[index][~self._counts.mask[index]].shape[0] - __class__.__fit_array[i].nfit - self._results["pvalue"][index] = Statistics.chi2_pvalue(ndof,__class__.__fit_array[i].fcn(__class__.__fit_array[i].values)) + self._results["pvalue"][index] = Statistics.chi2_pvalue(ndof, __class__.__fit_array[i].fcn(__class__.__fit_array[i].values)) @staticmethod - def _NG_Likelihood_Chi2(pp,res,mu2,n,muped,sigped,lum,charge,counts,**kwargs) : + def _NG_Likelihood_Chi2(pp : float,res: float,mu2: float,n: float,muped: float,sigped: float,lum: float,charge : np.ndarray,counts:np.ndarray,**kwargs) : + """ + Calculates the chi-square value using the MPE2 function. + Parameters: + pp (float): The pp parameter. + res (float): The res parameter. + mu2 (float): The mu2 parameter. + n (float): The n parameter. + muped (float): The muped parameter. + sigped (float): The sigped parameter. + lum (float): The lum parameter. + charge (np.ndarray): An array of charge values. + counts (np.ndarray): An array of count values. + **kwargs: Additional keyword arguments. + Returns: + float: The chi-square value. + """ pdf = MPE2(charge,pp,res,mu2,n,muped,sigped,lum,**kwargs) #log.debug(f"pdf : {np.sum(pdf)}") Ntot = np.sum(counts) @@ -334,10 +464,18 @@ def _NG_Likelihood_Chi2(pp,res,mu2,n,muped,sigped,lum,charge,counts,**kwargs) : mask = counts > 0 Lik = np.sum(((pdf*Ntot-counts)[mask])**2/counts[mask]) #2 times faster return Lik - + @staticmethod - def cost(charge,counts) : - def Chi2(pedestal,pp,luminosity,resolution,mean,n,pedestalWidth) : + def cost(charge:np.ndarray,counts:np.ndarray) : + """ + Defines a function called Chi2 that calculates the chi-square value using the _NG_Likelihood_Chi2 method. + Parameters: + charge (np.ndarray): An array of charge values. + counts (np.ndarray): An array of count values. + Returns: + function: The Chi2 function. + """ + def Chi2(pedestal: float,pp: float,luminosity: float,resolution: float,mean: float,n: float,pedestalWidth: float) : #assert not(np.isnan(pp) or np.isnan(resolution) or np.isnan(mean) or np.isnan(n) or np.isnan(pedestal) or np.isnan(pedestalWidth) or np.isnan(luminosity)) for i in range(1000): if (gammainc(i+1,luminosity) < 1e-5): @@ -346,53 +484,93 @@ def Chi2(pedestal,pp,luminosity,resolution,mean,n,pedestalWidth) : kwargs = {"ntotalPE" : ntotalPE} return __class__._NG_Likelihood_Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity,charge,counts,**kwargs) return Chi2 - + #@njit(parallel=True,nopython = True) - def _make_fit_array_from_parameters(self, pixels_id = None, **kwargs) : - if pixels_id is None : + def _make_fit_array_from_parameters(self, pixels_id : np.ndarray = None, **kwargs) -> np.ndarray: + """ + Create an array of Minuit fit instances based on the parameters and data for each pixel. + + Args: + pixels_id (optional): An array of pixel IDs. If not provided, all pixels will be used. + + Returns: + np.ndarray: An array of Minuit fit instances, one for each pixel. + """ + if pixels_id is None: npix = self.npixels pixels_id = self.pixels_id - else : + else: npix = len(pixels_id) - fit_array = np.empty((npix),dtype = np.object_) + fit_array = np.empty((npix), dtype=np.object_) - for i,_id in enumerate(pixels_id) : - #for j in prange(len(pixels_id)) : - # _id = pixels_id[j] + for i, _id in enumerate(pixels_id): index = np.where(self.pixels_id == _id)[0][0] - parameters = __class__._update_parameters(self.parameters,self._charge[index].data[~self._charge[index].mask],self._counts[index].data[~self._charge[index].mask],pixel_id=_id,**kwargs) + parameters = __class__._update_parameters(self.parameters, self._charge[index].data[~self._charge[index].mask], self._counts[index].data[~self._charge[index].mask], pixel_id=_id, **kwargs) minuitParameters = UtilsMinuit.make_minuit_par_kwargs(parameters) - minuit_kwargs = {parname : minuitParameters['values'][parname] for parname in minuitParameters['values']} - log.info(f'creation of fit instance for pixel : {_id}') - fit_array[i] = Minuit(__class__.cost(self._charge[index].data[~self._charge[index].mask],self._counts[index].data[~self._charge[index].mask]),**minuit_kwargs) + minuit_kwargs = {parname: minuitParameters['values'][parname] for parname in minuitParameters['values']} + log.info(f'creation of fit instance for pixel: {_id}') + fit_array[i] = Minuit(__class__.cost(self._charge[index].data[~self._charge[index].mask], self._counts[index].data[~self._charge[index].mask]), **minuit_kwargs) log.debug('fit created') fit_array[i].errordef = Minuit.LIKELIHOOD fit_array[i].strategy = 0 fit_array[i].tol = 1e40 fit_array[i].print_level = 1 fit_array[i].throw_nan = True - UtilsMinuit.set_minuit_parameters_limits_and_errors(fit_array[i],minuitParameters) + UtilsMinuit.set_minuit_parameters_limits_and_errors(fit_array[i], minuitParameters) log.debug(fit_array[i].values) log.debug(fit_array[i].limits) log.debug(fit_array[i].fixed) + return fit_array @staticmethod - def run_fit(i) : + def run_fit(i: int) -> dict: + """ + Perform a fit on a specific pixel using the Minuit package. + + Args: + i (int): The index of the pixel to perform the fit on. + + Returns: + dict: A dictionary containing the fit values and errors for the specified pixel. + The keys are "values_i" and "errors_i", where "i" is the index of the pixel. + """ log.info("Starting") __class__.__fit_array[i].migrad() __class__.__fit_array[i].hesse() _values = np.array([params.value for params in __class__.__fit_array[i].params]) _errors = np.array([params.error for params in __class__.__fit_array[i].params]) log.info("Finished") - return {f"values_{i}" : _values, f"errors_{i}" : _errors} + return {f"values_{i}": _values, f"errors_{i}": _errors} def make(self, - pixels_id = None, - multiproc = True, - display = True, - **kwargs) : + pixels_id :np.ndarray= None, + multiproc : bool = True, + display : bool = True, + **kwargs) -> np.ndarray: + """ + Perform a fit on specified pixels and return the fit results. + + Args: + pixels_id (np.ndarray, optional): An array of pixel IDs to perform the fit on. If not provided, the fit will be performed on all pixels. Default is None. + multiproc (bool, optional): A boolean indicating whether to use multiprocessing for the fit. Default is True. + display (bool, optional): A boolean indicating whether to display the fit results. Default is True. + **kwargs (optional): Additional keyword arguments. + + Returns: + np.ndarray: An array of fit instances. + + Example Usage: + # Initialize the FlatFieldSingleHHVSPEMaker object + maker = FlatFieldSingleHHVSPEMaker(charge, counts) + + # Perform the fit on all pixels and display the fit results + results = maker.make() + + # Perform the fit on specific pixels and display the fit results + results = maker.make(pixels_id=[1, 2, 3]) + """ log.info("running maker") log.info('checking asked pixels id') if pixels_id is None : @@ -457,53 +635,82 @@ def make(self, return output - def plot_single(pixel_id,charge,counts,pp,resolution,gain,gain_error,n,pedestal,pedestalWidth,luminosity,likelihood) : - fig,ax = plt.subplots(1,1,figsize=(8, 8)) - ax.errorbar(charge,counts,np.sqrt(counts),zorder=0,fmt=".",label = "data") + def plot_single(pixel_id: int, charge: np.ndarray, counts: np.ndarray, pp: float, resolution: float, gain: float, gain_error: float, n: float, pedestal: float, pedestalWidth: float, luminosity: float, likelihood: float) -> tuple: + """ + Generate a plot of the data and a model fit for a specific pixel. + + Args: + pixel_id (int): The ID of the pixel for which the plot is generated. + charge (np.ndarray): An array of charge values. + counts (np.ndarray): An array of event counts corresponding to the charge values. + pp (float): The value of the `pp` parameter. + resolution (float): The value of the `resolution` parameter. + gain (float): The value of the `gain` parameter. + gain_error (float): The value of the `gain_error` parameter. + n (float): The value of the `n` parameter. + pedestal (float): The value of the `pedestal` parameter. + pedestalWidth (float): The value of the `pedestalWidth` parameter. + luminosity (float): The value of the `luminosity` parameter. + likelihood (float): The value of the `likelihood` parameter. + + Returns: + tuple: A tuple containing the generated plot figure and the axes of the plot. + """ + fig, ax = plt.subplots(1, 1, figsize=(8, 8)) + ax.errorbar(charge, counts, np.sqrt(counts), zorder=0, fmt=".", label="data") ax.plot(charge, - np.trapz(counts,charge)*MPE2( - charge, - pp, - resolution, - gain, - n, - pedestal, - pedestalWidth, - luminosity, - ), - zorder=1, - linewidth=2, - label = f"SPE model fit \n gain : {gain - gain_error:.2f} < {gain:.2f} < {gain + gain_error:.2f} ADC/pe,\n likelihood : {likelihood:.2f}") + np.trapz(counts, charge) * MPE2( + charge, + pp, + resolution, + gain, + n, + pedestal, + pedestalWidth, + luminosity, + ), + zorder=1, + linewidth=2, + label=f"SPE model fit \n gain : {gain - gain_error:.2f} < {gain:.2f} < {gain + gain_error:.2f} ADC/pe,\n likelihood : {likelihood:.2f}") ax.set_xlabel("Charge (ADC)", size=15) ax.set_ylabel("Events", size=15) ax.set_title(f"SPE fit pixel id : {pixel_id}") ax.set_xlim([pedestal - 6 * pedestalWidth, None]) ax.legend(fontsize=18) - return fig,ax - - def display(self,pixels_id,**kwargs) : - figpath = kwargs.get('figpath',f"/tmp/NectarGain_pid{os.getpid()}") - os.makedirs(figpath,exist_ok = True) - for _id in pixels_id : - index = np.argmax(self._results['pixels_id'] == _id) - fig,ax = __class__.plot_single( + return fig, ax + + + def display(self, pixels_id: np.ndarray, **kwargs) -> None: + """ + Display and save the plot for each specified pixel ID. + + Args: + pixels_id (np.ndarray): An array of pixel IDs. + **kwargs: Additional keyword arguments. + figpath (str): The path to save the generated plot figures. Defaults to "/tmp/NectarGain_pid{os.getpid()}". + """ + figpath = kwargs.get('figpath', f"/tmp/NectarGain_pid{os.getpid()}") + os.makedirs(figpath, exist_ok=True) + for _id in pixels_id: + index = np.argmax(self._results['pixels_id'] == _id) + fig, ax = __class__.plot_single( _id, self._charge[index], self._counts[index], - self._results['pp'][index].value, - self._results['resolution'][index].value, - self._results['high_gain'][index].value, + self._results['pp'][index].value, + self._results['resolution'][index].value, + self._results['high_gain'][index].value, self._results['high_gain_error'][index].value.mean(), - self._results['n'][index].value, - self._results['pedestal'][index].value, - self._results['pedestalWidth'][index].value, + self._results['n'][index].value, + self._results['pedestal'][index].value, + self._results['pedestalWidth'][index].value, self._results['luminosity'][index].value, self._results['likelihood'][index], ) fig.savefig(f"{figpath}/fit_SPE_pixel{_id}.pdf") fig.clf() plt.close(fig) - del fig,ax + del fig, ax @@ -512,62 +719,127 @@ class FlatFieldSingleHHVStdSPEMaker(FlatFieldSingleHHVSPEMaker): __parameters_file = 'parameters_signalStd.yaml' _reduced_name = "FlatFieldSingleStdSPE" -#constructors - def __init__(self,charge,counts,*args,**kwargs) -> None: - super().__init__(charge,counts,*args,**kwargs) + def __init__(self, charge : np.ndarray, counts: np.ndarray, *args, **kwargs) -> None: + """ + Initializes a new instance of the FlatFieldSingleHHVStdSPEMaker class. + + Args: + charge (np.ndarray): The charge data. + counts (np.ndarray): The counts data. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + """ + super().__init__(charge, counts, *args, **kwargs) self.__fix_parameters() -#methods def __fix_parameters(self) -> None: - """this method should be used to fix n and pp + """ + Fixes the values of the n and pp parameters by setting their frozen attribute to True. """ log.info("updating parameters by fixing pp and n") pp = self._parameters["pp"] pp.frozen = True n = self._parameters["n"] n.frozen = True - class FlatFieldSingleNominalSPEMaker(FlatFieldSingleHHVSPEMaker): - """class to perform fit of the SPE signal at nominal voltage from fitted data obtained with 1400V run - Thus, n, pp and res are fixed""" + """ + A class to perform a fit of the single photoelectron (SPE) signal at nominal voltage using fitted data obtained from a 1400V run. + Inherits from FlatFieldSingleHHVSPEMaker. + Fixes the parameters n, pp, and res. + Optionally fixes the luminosity parameter. + + Args: + charge (np.ndarray): The charge values. + counts (np.ndarray): The counts values. + nectarGainSPEresult (str): The path to the fitted data obtained from a 1400V run. + same_luminosity (bool, optional): Whether to fix the luminosity parameter. Defaults to False. + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + + Attributes: + __parameters_file (str): The path to the parameters file for the fit at nominal voltage. + _reduced_name (str): The name of the reduced data for the fit at nominal voltage. + __same_luminosity (bool): Whether the luminosity parameter should be fixed. + __nectarGainSPEresult (QTable): The fitted data obtained from a 1400V run, filtered for valid pixels. + + Example Usage: + # Create an instance of FlatFieldSingleNominalSPEMaker + maker = FlatFieldSingleNominalSPEMaker(charge, counts, nectarGainSPEresult='fit_result.txt', same_luminosity=True) + + # Perform the fit on the specified pixels and return the fit results + results = maker.make(pixels_id=[1, 2, 3]) + + # Plot the fit for the specified pixels + maker.display(pixels_id=[1, 2, 3]) + """ + __parameters_file = 'parameters_signal_fromHHVFit.yaml' _reduced_name = "FlatFieldSingleNominalSPE" -#constructors - def __init__(self, charge, counts, nectarGainSPEresult : str, same_luminosity : bool = False, *args, **kwargs) -> None: + def __init__(self, charge:np.ndarray, counts:np.ndarray, nectarGainSPEresult: str, same_luminosity: bool = False, *args, **kwargs) -> None: + """ + Initializes an instance of FlatFieldSingleNominalSPEMaker. + + Args: + charge (np.ndarray): The charge values. + counts (np.ndarray): The counts values. + nectarGainSPEresult (str): The path to the fitted data obtained from a 1400V run. + same_luminosity (bool, optional): Whether to fix the luminosity parameter. Defaults to False. + *args: Variable length argument list. + **kwargs: Arbitrary keyword arguments. + """ super().__init__(charge, counts, *args, **kwargs) self.__fix_parameters(same_luminosity) self.__same_luminosity = same_luminosity self.__nectarGainSPEresult = self._read_SPEresult(nectarGainSPEresult) - if len(self.__nectarGainSPEresult) == 0 : + if len(self.__nectarGainSPEresult) == 0: log.warning("The intersection between pixels id from the data and those valid from the SPE fit result is empty") -#getters and setters @property - def nectarGainSPEresult(self) : return copy.deepcopy(self.__nectarGainSPEresult) + def nectarGainSPEresult(self): + """ + QTable: The fitted data obtained from a 1400V run, filtered for valid pixels. + """ + return copy.deepcopy(self.__nectarGainSPEresult) @property - def same_luminosity(self) : return copy.deepcopy(self.__same_luminosity) + def same_luminosity(self): + """ + bool: Whether the luminosity parameter should be fixed. + """ + return copy.deepcopy(self.__same_luminosity) -#methods - def _read_SPEresult(self,nectarGainSPEresult : str) : - table = QTable.read(nectarGainSPEresult,format = "ascii.ecsv") + def _read_SPEresult(self, nectarGainSPEresult: str): + """ + Reads the fitted data obtained from a 1400V run and returns a filtered table of valid pixels. + + Args: + nectarGainSPEresult (str): The path to the fitted data obtained from a 1400V run. + + Returns: + QTable: The filtered table of valid pixels. + """ + table = QTable.read(nectarGainSPEresult, format="ascii.ecsv") table = table[table["is_valid"]] argsort = [] mask = [] - for _id in self._pixels_id : - if _id in table['pixels_id'] : - argsort.append(np.where(_id==table['pixels_id'])[0][0]) + for _id in self._pixels_id: + if _id in table['pixels_id']: + argsort.append(np.where(_id == table['pixels_id'])[0][0]) mask.append(True) - else : + else: mask.append(False) self._pixels_id = self._pixels_id[np.array(mask)] return table[np.array(argsort)] - def __fix_parameters(self, same_luminosity : bool) -> None: - """this method should be used to fix n, pp, res and possibly luminosity + def __fix_parameters(self, same_luminosity: bool) -> None: + """ + Fixes the parameters n, pp, res, and possibly luminosity. + + Args: + same_luminosity (bool): Whether to fix the luminosity parameter. """ log.info("updating parameters by fixing pp, n and res") pp = self._parameters["pp"] @@ -576,28 +848,52 @@ def __fix_parameters(self, same_luminosity : bool) -> None: n.frozen = True resolution = self._parameters["resolution"] resolution.frozen = True - if same_luminosity : + if same_luminosity: log.info("fixing luminosity") luminosity = self._parameters["luminosity"] luminosity.frozen = True - def _make_fit_array_from_parameters(self, pixels_id = None, **kwargs) : - return super()._make_fit_array_from_parameters(pixels_id = pixels_id, nectarGainSPEresult = self.__nectarGainSPEresult, **kwargs) + def _make_fit_array_from_parameters(self, pixels_id=None, **kwargs): + """ + Generates the fit array from the fixed parameters and the fitted data obtained from a 1400V run. + + Args: + pixels_id (array-like, optional): The pixels to generate the fit array for. Defaults to None. + **kwargs: Arbitrary keyword arguments. + + Returns: + array-like: The fit array. + """ + return super()._make_fit_array_from_parameters(pixels_id=pixels_id, nectarGainSPEresult=self.__nectarGainSPEresult, **kwargs) @staticmethod - def _update_parameters(parameters,charge,counts,pixel_id,nectarGainSPEresult,**kwargs) : - param = super()._update_parameters(parameters,charge,counts,**kwargs) + def _update_parameters(parameters:Parameters, charge:np.ndarray, counts:np.ndarray, pixel_id, nectarGainSPEresult:QTable, **kwargs): + """ + Updates the parameters with the fixed values from the fitted data obtained from a 1400V run. + + Args: + parameters (Parameters): The parameters to update. + charge (np.ndarray): The charge values. + counts (np.ndarray): The counts values. + pixel_id (int): The pixel ID. + nectarGainSPEresult (QTable): The fitted data obtained from a 1400V run. + **kwargs: Arbitrary keyword arguments. + + Returns: + dict: The updated parameters. + """ + param = super()._update_parameters(parameters, charge, counts, **kwargs) luminosity = param["luminosity"] resolution = param["resolution"] pp = param["pp"] n = param["n"] - + index = np.where(pixel_id == nectarGainSPEresult["pixels_id"])[0][0] resolution.value = nectarGainSPEresult[index]["resolution"].value pp.value = nectarGainSPEresult[index]["pp"].value n.value = nectarGainSPEresult[index]["n"].value - if luminosity.frozen : + if luminosity.frozen: luminosity.value = nectarGainSPEresult[index]["luminosity"].value return param \ No newline at end of file diff --git a/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py b/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py index 2b453b42..9ec2aba9 100644 --- a/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py +++ b/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py @@ -27,24 +27,89 @@ __all__ = ["PhotoStatisticMaker"] + class PhotoStatisticMaker(GainMaker): + """ + The `PhotoStatisticMaker` class is a subclass of `GainMaker` and is used to calculate photo statistics for a given set of charge data. It provides methods to create an instance from charge containers or run numbers, as well as methods to calculate various statistics such as gain and standard deviation. + + Example Usage: + # Create an instance of PhotoStatisticMaker using charge containers + FFcharge = ChargesContainer(...) + Pedcharge = ChargesContainer(...) + coefCharge_FF_Ped = 0.5 + SPE_result = "path/to/SPE_results" + photo_stat = PhotoStatisticMaker.create_from_chargeContainer(FFcharge, Pedcharge, coefCharge_FF_Ped, SPE_result) + + # Calculate and retrieve the gain values + gain_hg = photo_stat.gainHG + gain_lg = photo_stat.gainLG + + # Plot the correlation between photo statistic gain and SPE gain + photo_stat_gain = np.array(...) + SPE_gain = np.array(...) + fig = PhotoStatisticMaker.plot_correlation(photo_stat_gain, SPE_gain) + + Methods: + - `__init__(self, FFcharge_hg, FFcharge_lg, Pedcharge_hg, Pedcharge_lg, coefCharge_FF_Ped, SPE_resolution, *args, **kwargs)`: Constructor method to initialize the `PhotoStatisticMaker` instance with charge data and other parameters. + - `create_from_chargeContainer(cls, FFcharge, Pedcharge, coefCharge_FF_Ped, SPE_result, **kwargs)`: Class method to create an instance of `PhotoStatisticMaker` from charge containers. + - `create_from_run_numbers(cls, FFrun, Pedrun, SPE_result, **kwargs)`: Class method to create an instance of `PhotoStatisticMaker` from run numbers. + - `__readSPE(SPEresults) -> tuple`: Static method to read SPE resolution from a file and return the resolution and pixel IDs. + - `__get_charges_FF_Ped_reshaped(FFcharge, Pedcharge, SPE_resolution, SPE_pixels_id) -> dict`: Static method to reshape the charge data based on the intersection of pixel IDs and return a dictionary of reshaped data. + - `__readFF(FFRun, **kwargs) -> dict`: Static method to read FF data from a file and return the FF charge data and coefficient. + - `__readPed(PedRun, **kwargs) -> dict`: Static method to read Ped data from a file and return the Ped charge data. + - `__check_shape(self) -> None`: Method to check the shape of the charge data arrays. + - `make(self, **kwargs) -> None`: Method to run the photo statistic method and store the results. + - `plot_correlation(photoStat_gain, SPE_gain) -> fig`: Static method to plot the correlation between photo statistic gain and SPE gain. + + Fields: + - `SPE_resolution`: Property to get the SPE resolution. + - `sigmaPedHG`: Property to get the standard deviation of Pedcharge_hg. + - `sigmaChargeHG`: Property to get the standard deviation of FFcharge_hg - meanPedHG. + - `meanPedHG`: Property to get the mean of Pedcharge_hg. + - `meanChargeHG`: Property to get the mean of FFcharge_hg - meanPedHG. + - `BHG`: Property to calculate the BHG value. + - `gainHG`: Property to calculate the gain for high gain. + - `sigmaPedLG`: Property to get the standard deviation of Pedcharge_lg. + - `sigmaChargeLG`: Property to get the standard deviation of FFcharge_lg - meanPedLG. + - `meanPedLG`: Property to get the mean of Pedcharge_lg. + - `meanChargeLG`: Property to get the mean of FFcharge_lg - meanPedLG. + - `BLG`: Property to calculate the BLG value. + - `gainLG`: Property to calculate the gain for low gain. + """ _reduced_name = "PhotoStatistic" #constructors def __init__(self, - FFcharge_hg, - FFcharge_lg, - Pedcharge_hg, - Pedcharge_lg, - coefCharge_FF_Ped, + FFcharge_hg : np.ndarray, + FFcharge_lg : np.ndarray, + Pedcharge_hg: np.ndarray, + Pedcharge_lg: np.ndarray, + coefCharge_FF_Ped : float, SPE_resolution, *args, **kwargs ) -> None: + """ + Initializes the instance of the PhotoStatisticMaker class with charge data and other parameters. + + Args: + FFcharge_hg (np.ndarray): Array of charge data for high gain in the FF (Flat Field) image. + FFcharge_lg (np.ndarray): Array of charge data for low gain in the FF image. + Pedcharge_hg (np.ndarray): Array of charge data for high gain in the Ped (Pedestal) image. + Pedcharge_lg (np.ndarray): Array of charge data for low gain in the Ped image. + coefCharge_FF_Ped (float): Coefficient to convert FF charge to Ped charge. + SPE_resolution: Array-like of single photoelectron (SPE) resolutions for each pixel, or single value to use the same for each pixel. + + Raises: + TypeError: If SPE_resolution is not provided in a valid format. + + Returns: + None + """ super().__init__(*args,**kwargs) self.__coefCharge_FF_Ped = coefCharge_FF_Ped - + self.__FFcharge_hg = FFcharge_hg self.__FFcharge_lg = FFcharge_lg @@ -68,11 +133,24 @@ def __init__(self, def create_from_chargeContainer(cls, FFcharge : ChargesContainer, Pedcharge : ChargesContainer, - coefCharge_FF_Ped, - SPE_resolution, + coefCharge_FF_Ped : float, + SPE_result, **kwargs) : - if isinstance(SPE_resolution , str) or isinstance(SPE_resolution , Path) : - SPE_resolution,SPE_pixels_id = __class__.__readSPE(SPE_resolution) + """ + Create an instance of the PhotoStatisticMaker class from Pedestal and Flatfield runs stored in ChargesContainer. + + Args: + FFcharge (ChargesContainer): Array of charge data for the FF image. + Pedcharge (ChargesContainer): Array of charge data for the Ped image. + coefCharge_FF_Ped (float): Coefficient to convert FF charge to Ped charge. + SPE_result (str or Path): Path to the SPE result file (optional). + **kwargs: Additional keyword arguments for initializing the PhotoStatisticMaker instance. + + Returns: + PhotoStatisticMaker: An instance of the PhotoStatisticMaker class created from the ChargesContainer instances. + """ + if isinstance(SPE_result , str) or isinstance(SPE_result , Path) : + SPE_resolution,SPE_pixels_id = __class__.__readSPE(SPE_result) else : SPE_pixels_id = None @@ -85,23 +163,57 @@ def create_from_chargeContainer(cls, return cls(coefCharge_FF_Ped = coefCharge_FF_Ped, **kwargs) @classmethod - def create_from_run_numbers(cls, FFrun : int, Pedrun : int, SPE_resolution : str, **kwargs) : + def create_from_run_numbers(cls, FFrun: int, Pedrun: int, SPE_result: str, **kwargs): + """ + Create an instance of the PhotoStatisticMaker class by reading the FF (Flat Field) and Ped (Pedestal) charge data from run numbers. + + Args: + FFrun (int): The run number for the FF charge data. + Pedrun (int): The run number for the Ped charge data. + SPE_result (str): The path to the SPE result file. + **kwargs: Additional keyword arguments. + + Returns: + PhotoStatisticMaker: An instance of the PhotoStatisticMaker class created from the FF and Ped charge data and the SPE result file. + """ FFkwargs = __class__.__readFF(FFrun, **kwargs) Pedkwargs = __class__.__readPed(Pedrun, **kwargs) kwargs.update(FFkwargs) kwargs.update(Pedkwargs) - return cls.create_from_chargeContainer(SPE_resolution = SPE_resolution, **kwargs) + return cls.create_from_chargeContainer(SPE_result=SPE_result, **kwargs) #methods @staticmethod - def __readSPE(SPEresults) : + def __readSPE(SPEresults) -> tuple: + """ + Reads the SPE resolution from a file and returns the resolution values and corresponding pixel IDs. + + Args: + SPEresults (str): The file path to the SPE results file. + + Returns: + tuple: A tuple containing the SPE resolution values and corresponding pixel IDs. + """ log.info(f'reading SPE resolution from {SPEresults}') table = QTable.read(SPEresults) table.sort('pixels_id') return table['resolution'][table['is_valid']].value,table['pixels_id'][table['is_valid']].value @staticmethod - def __get_charges_FF_Ped_reshaped( FFcharge : ChargesContainer, Pedcharge : ChargesContainer, SPE_resolution, SPE_pixels_id) : + def __get_charges_FF_Ped_reshaped( FFcharge : ChargesContainer, Pedcharge : ChargesContainer, SPE_resolution : np.ndarray, SPE_pixels_id: np.ndarray)-> dict : + """ + Reshapes the FF (Flat Field) and Ped (Pedestal) charges based on the intersection of pixel IDs between the two charges. + Selects the charges for the high-gain and low-gain channels and returns them along with the common pixel IDs. + + Args: + FFcharge (ChargesContainer): The charges container for the Flat Field data. + Pedcharge (ChargesContainer): The charges container for the Pedestal data. + SPE_resolution (np.ndarray): An array containing the SPE resolutions. + SPE_pixels_id (np.ndarray): An array containing the pixel IDs for the SPE data. + + Returns: + dict: A dictionary containing the reshaped data, including the common pixel IDs, SPE resolution (if provided), and selected charges for the high-gain and low-gain channels. + """ log.info("reshape of SPE, Ped and FF data with intersection of pixel ids") out = {} @@ -121,129 +233,279 @@ def __get_charges_FF_Ped_reshaped( FFcharge : ChargesContainer, Pedcharge : Char return out @staticmethod - def __readFF(FFRun,**kwargs) : + def __readFF(FFRun: int, **kwargs) -> dict: + """ + Reads FF charge data from a FITS file. + Args: + - FFRun (int): The run number for the FF data. + - kwargs (optional): Additional keyword arguments. + Returns: + - dict: A dictionary containing the FF charge data (`FFcharge`) and the coefficient for the FF charge (`coefCharge_FF_Ped`). + """ log.info('reading FF data') - method = kwargs.get('method','FullWaveformSum') - FFchargeExtractorWindowLength = kwargs.get('FFchargeExtractorWindowLength',None) - if method != 'FullWaveformSum' : - if FFchargeExtractorWindowLength is None : + method = kwargs.get('method', 'FullWaveformSum') + FFchargeExtractorWindowLength = kwargs.get('FFchargeExtractorWindowLength', None) + if method != 'FullWaveformSum': + if FFchargeExtractorWindowLength is None: e = Exception(f"we have to specify FFchargeExtractorWindowLength argument if charge extractor method is not FullwaveformSum") - log.error(e,exc_info=True) + log.error(e, exc_info=True) raise e - else : + else: coefCharge_FF_Ped = FFchargeExtractorWindowLength / constants.N_SAMPLES - else : + else: coefCharge_FF_Ped = 1 - if isinstance(FFRun,int) : - try : - FFcharge = ChargesContainerIO.load(f"{os.environ['NECTARCAMDATA']}/charges/{method}",FFRun) + if isinstance(FFRun, int): + try: + FFcharge = ChargesContainerIO.load(f"{os.environ['NECTARCAMDATA']}/charges/{method}", FFRun) log.info(f'charges have ever been computed for FF run {FFRun}') - except Exception as e : + except Exception as e: log.error("charge have not been yet computed") raise e - else : - e = TypeError("FFRun must be int or ChargeContainer") - log.error(e,exc_info = True) + else: + e = TypeError("FFRun must be int") + log.error(e, exc_info=True) raise e - return {"FFcharge" : FFcharge, "coefCharge_FF_Ped" : coefCharge_FF_Ped} - + return {"FFcharge": FFcharge, "coefCharge_FF_Ped": coefCharge_FF_Ped} @staticmethod - def __readPed(PedRun,**kwargs) : + def __readPed(PedRun: int, **kwargs) -> dict: + """ + Reads Ped charge data from a FITS file. + Args: + - PedRun (int): The run number for the Ped data. + - kwargs (optional): Additional keyword arguments. + Returns: + - dict: A dictionary containing the Ped charge data (`Pedcharge`). + """ log.info('reading Ped data') - method = 'FullWaveformSum'#kwargs.get('method','std') - if isinstance(PedRun,int) : - try : - Pedcharge = ChargesContainerIO.load(f"{os.environ['NECTARCAMDATA']}/charges/{method}",PedRun) + method = 'FullWaveformSum' # kwargs.get('method','std') + if isinstance(PedRun, int): + try: + Pedcharge = ChargesContainerIO.load(f"{os.environ['NECTARCAMDATA']}/charges/{method}", PedRun) log.info(f'charges have ever been computed for Ped run {PedRun}') - except Exception as e : + except Exception as e: log.error("charge have not been yet computed") raise e - else : - e = TypeError("PedRun must be int or ChargeContainer") - log.error(e,exc_info = True) + else: + e = TypeError("PedRun must be int") + log.error(e, exc_info=True) raise e - return {"Pedcharge" : Pedcharge} + return {"Pedcharge": Pedcharge} def __check_shape(self) -> None: + """ + Checks the shape of certain attributes and raises an exception if the shape is not as expected. + """ try : self.__FFcharge_hg[0] * self.__FFcharge_lg[0] * self.__Pedcharge_hg[0] * self.__Pedcharge_lg[0] * self.__SPE_resolution * self._pixels_id except Exception as e : log.error(e,exc_info = True) raise e - def make(self,**kwargs)-> None: + def make(self, **kwargs) -> None: + """ + Runs the photo statistic method and assigns values to the high_gain and low_gain keys in the _results dictionary. + + Args: + **kwargs: Additional keyword arguments (not used in this method). + + Returns: + None + """ log.info('running photo statistic method') self._results["high_gain"] = self.gainHG self._results["low_gain"] = self.gainLG - #self._results["is_valid"] = self._SPEvalid - - - def plot_correlation(photoStat_gain,SPE_gain) : - mask = (photoStat_gain>20) * (SPE_gain>0) * (photoStat_gain<80) - a, b, r, p_value, std_err = linregress(photoStat_gain[mask], SPE_gain[mask],'greater') - x = np.linspace(photoStat_gain[mask].min(),photoStat_gain[mask].max(),1000) - y = lambda x: a * x + b - with quantity_support() : - fig,ax = plt.subplots(1,1,figsize=(8, 6)) - ax.scatter(photoStat_gain[mask],SPE_gain[mask],marker =".") - ax.plot(x,y(x),color = 'red', label = f"linear fit,\n a = {a:.2e},\n b = {b:.2e},\n r = {r:.2e},\n p_value = {p_value:.2e},\n std_err = {std_err:.2e}") - ax.plot(x,x,color = 'black',label = "y = x") + # self._results["is_valid"] = self._SPEvalid + + + def plot_correlation(photoStat_gain: np.ndarray, SPE_gain: np.ndarray) -> plt.Figure: + """ + Plot the correlation between the photo statistic gain and the single photoelectron (SPE) gain. + + Args: + photoStat_gain (np.ndarray): Array of photo statistic gain values. + SPE_gain (np.ndarray): Array of SPE gain values. + + Returns: + fig (plt.Figure): The figure object containing the scatter plot and the linear fit line. + """ + + # Create a mask to filter the data points based on certain criteria + mask = (photoStat_gain > 20) * (SPE_gain > 0) * (photoStat_gain < 80) + + # Perform a linear regression analysis on the filtered data points + a, b, r, p_value, std_err = linregress(photoStat_gain[mask], SPE_gain[mask], 'greater') + + # Generate a range of x-values for the linear fit line + x = np.linspace(photoStat_gain[mask].min(), photoStat_gain[mask].max(), 1000) + + # Define a lambda function for the linear fit line + y = lambda x: a * x + b + + with quantity_support(): + # Create a scatter plot of the filtered data points + fig, ax = plt.subplots(1, 1, figsize=(8, 6)) + ax.scatter(photoStat_gain[mask], SPE_gain[mask], marker=".") + + # Plot the linear fit line using the x-values and the lambda function + ax.plot(x, y(x), color='red', + label=f"linear fit,\n a = {a:.2e},\n b = {b:.2e},\n r = {r:.2e},\n p_value = {p_value:.2e},\n std_err = {std_err:.2e}") + + # Plot the line y = x + ax.plot(x, x, color='black', label="y = x") + ax.set_xlabel("Gain Photo stat (ADC)", size=15) ax.set_ylabel("Gain SPE fit (ADC)", size=15) - #ax.set_xlim(xmin = 0) - #ax.set_ylim(ymin = 0) ax.legend(fontsize=15) + return fig -#getters and setters - @property - def SPE_resolution(self) : return copy.deepcopy(self.__SPE_resolution) +@property +def SPE_resolution(self) -> float: + """ + Returns a deep copy of the SPE resolution. - @property - def sigmaPedHG(self) : return np.std(self.__Pedcharge_hg ,axis = 0) * np.sqrt(self.__coefCharge_FF_Ped) + Returns: + float: The SPE resolution. + """ + return copy.deepcopy(self.__SPE_resolution) - @property - def sigmaChargeHG(self) : return np.std(self.__FFcharge_hg - self.meanPedHG, axis = 0) - @property - def meanPedHG(self) : return np.mean(self.__Pedcharge_hg ,axis = 0) * self.__coefCharge_FF_Ped +@property +def sigmaPedHG(self) -> float: + """ + Calculates and returns the standard deviation of Pedcharge_hg multiplied by the square root of coefCharge_FF_Ped. - @property - def meanChargeHG(self) : return np.mean(self.__FFcharge_hg - self.meanPedHG, axis = 0) + Returns: + float: The standard deviation of Pedcharge_hg. + """ + return np.std(self.__Pedcharge_hg, axis=0) * np.sqrt(self.__coefCharge_FF_Ped) - @property - def BHG(self) : - min_events = np.min((self.__FFcharge_hg.shape[0],self.__Pedcharge_hg.shape[0])) - upper = (np.power(self.__FFcharge_hg.mean(axis = 1)[:min_events] - self.__Pedcharge_hg.mean(axis = 1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeHG.mean(),2)).mean(axis = 0) - lower = np.power(self.meanChargeHG.mean(),2)#np.power(self.meanChargeHG,2)#np.power(self.meanChargeHG.mean(),2) - return np.sqrt(upper/lower) - @property - def gainHG(self) : - return ((np.power(self.sigmaChargeHG,2) - np.power(self.sigmaPedHG,2) - np.power(self.BHG * self.meanChargeHG,2)) - /(self.meanChargeHG * (1 + np.power(self.SPE_resolution,2)))) - +@property +def sigmaChargeHG(self) -> float: + """ + Calculates and returns the standard deviation of FFcharge_hg minus meanPedHG. + + Returns: + float: The standard deviation of FFcharge_hg minus meanPedHG. + """ + return np.std(self.__FFcharge_hg - self.meanPedHG, axis=0) + + +@property +def meanPedHG(self) -> float: + """ + Calculates and returns the mean of Pedcharge_hg multiplied by coefCharge_FF_Ped. + + Returns: + float: The mean of Pedcharge_hg. + """ + return np.mean(self.__Pedcharge_hg, axis=0) * self.__coefCharge_FF_Ped + + +@property +def meanChargeHG(self) -> float: + """ + Calculates and returns the mean of FFcharge_hg minus meanPedHG. + + Returns: + float: The mean of FFcharge_hg minus meanPedHG. + """ + return np.mean(self.__FFcharge_hg - self.meanPedHG, axis=0) + + +@property +def BHG(self) -> float: + """ + Calculates and returns the BHG value. + + Returns: + float: The BHG value. + """ + min_events = np.min((self.__FFcharge_hg.shape[0], self.__Pedcharge_hg.shape[0])) + upper = (np.power(self.__FFcharge_hg.mean(axis=1)[:min_events] - self.__Pedcharge_hg.mean(axis=1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeHG.mean(), 2)).mean(axis=0) + lower = np.power(self.meanChargeHG.mean(), 2) + return np.sqrt(upper / lower) + + +@property +def gainHG(self) -> float: + """ + Calculates and returns the gain for high gain charge data. + + Returns: + float: The gain for high gain charge data. + """ + return ((np.power(self.sigmaChargeHG, 2) - np.power(self.sigmaPedHG, 2) - np.power(self.BHG * self.meanChargeHG, 2)) + / (self.meanChargeHG * (1 + np.power(self.SPE_resolution, 2)))) + + +@property +def sigmaPedLG(self) -> float: + """ + Calculates and returns the standard deviation of Pedcharge_lg multiplied by the square root of coefCharge_FF_Ped. + + Returns: + float: The standard deviation of Pedcharge_lg. + """ + return np.std(self.__Pedcharge_lg, axis=0) * np.sqrt(self.__coefCharge_FF_Ped) + + +@property +def sigmaChargeLG(self) -> float: + """ + Calculates and returns the standard deviation of FFcharge_lg minus meanPedLG. + + Returns: + float: The standard deviation of FFcharge_lg minus meanPedLG. + """ + return np.std(self.__FFcharge_lg - self.meanPedLG, axis=0) + + +@property +def meanPedLG(self) -> float: + """ + Calculates and returns the mean of Pedcharge_lg multiplied by coefCharge_FF_Ped. + + Returns: + float: The mean of Pedcharge_lg. + """ + return np.mean(self.__Pedcharge_lg, axis=0) * self.__coefCharge_FF_Ped + + +@property +def meanChargeLG(self) -> float: + """ + Calculates and returns the mean of FFcharge_lg minus meanPedLG. + + Returns: + float: The mean of FFcharge_lg minus meanPedLG. + """ + return np.mean(self.__FFcharge_lg - self.meanPedLG, axis=0) - @property - def sigmaPedLG(self) : return np.std(self.__Pedcharge_lg ,axis = 0) * np.sqrt(self.__coefCharge_FF_Ped) - @property - def sigmaChargeLG(self) : return np.std(self.__FFcharge_lg - self.meanPedLG,axis = 0) +@property +def BLG(self) -> float: + """ + Calculates and returns the BLG value. - @property - def meanPedLG(self) : return np.mean(self.__Pedcharge_lg,axis = 0) * self.__coefCharge_FF_Ped + Returns: + float: The BLG value. + """ + min_events = np.min((self.__FFcharge_lg.shape[0], self.__Pedcharge_lg.shape[0])) + upper = (np.power(self.__FFcharge_lg.mean(axis=1)[:min_events] - self.__Pedcharge_lg.mean(axis=1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeLG.mean(), 2)).mean(axis=0) + lower = np.power(self.meanChargeLG.mean(), 2) + return np.sqrt(upper / lower) - @property - def meanChargeLG(self) : return np.mean(self.__FFcharge_lg - self.meanPedLG,axis = 0) - @property - def BLG(self) : - min_events = np.min((self.__FFcharge_lg.shape[0],self.__Pedcharge_lg.shape[0])) - upper = (np.power(self.__FFcharge_lg.mean(axis = 1)[:min_events] - self.__Pedcharge_lg.mean(axis = 1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeLG.mean(),2)).mean(axis = 0) - lower = np.power(self.meanChargeLG.mean(),2) #np.power(self.meanChargeLG,2) #np.power(self.meanChargeLG.mean(),2) - return np.sqrt(upper/lower) +@property +def gainLG(self) -> float: + """ + Calculates and returns the gain for low gain charge data. - @property - def gainLG(self) : return ((np.power(self.sigmaChargeLG,2) - np.power(self.sigmaPedLG,2) - np.power(self.BLG * self.meanChargeLG,2)) - /(self.meanChargeLG * (1 + np.power(self.SPE_resolution,2)))) + Returns: + float: The gain for low gain charge data. + """ + return ((np.power(self.sigmaChargeLG, 2) - np.power(self.sigmaPedLG, 2) - np.power(self.BLG * self.meanChargeLG, 2)) + / (self.meanChargeLG * (1 + np.power(self.SPE_resolution, 2)))) diff --git a/src/nectarchain/makers/calibration/gain/parameters.py b/src/nectarchain/makers/calibration/gain/parameters.py index 11c6e8d9..96d6ddba 100644 --- a/src/nectarchain/makers/calibration/gain/parameters.py +++ b/src/nectarchain/makers/calibration/gain/parameters.py @@ -11,7 +11,7 @@ class Parameter() : def __init__(self, - name, + name:str, value, min = np.nan, max = np.nan, @@ -38,7 +38,7 @@ def __str__(self): @property def name(self) : return self.__name @name.setter - def name(self,value) : self.__name = value + def name(self,value: str) : self.__name = value @property def value(self) : return self.__value @@ -58,7 +58,7 @@ def max(self,value) : self.__max = value @property def unit(self) : return self.__unit @unit.setter - def unit(self,value) : self.__unit = value + def unit(self,value:u.Unit) : self.__unit = value @property def error(self) : return self.__error @@ -78,7 +78,7 @@ def __init__(self,parameters_liste : list = []) -> None: def append(self,parameter : Parameter) -> None: self.__parameters.append(parameter) - def __getitem__(self,key) : + def __getitem__(self,key:str) : for parameter in self.__parameters : if parameter.name == key : return parameter diff --git a/src/nectarchain/makers/chargesMakers.py b/src/nectarchain/makers/chargesMakers.py index afd8d325..91ebef31 100644 --- a/src/nectarchain/makers/chargesMakers.py +++ b/src/nectarchain/makers/chargesMakers.py @@ -9,6 +9,8 @@ import time from ctapipe_io_nectarcam import constants +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer +from ctapipe.instrument.subarray import SubarrayDescription from ctapipe.containers import EventType from ctapipe.image.extractor import (FullWaveformSum, FixedWindowSum, @@ -96,9 +98,61 @@ def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): pass + class ChargesMaker(ArrayDataMaker) : + """ + The `ChargesMaker` class is a subclass of `ArrayDataMaker` and is responsible for computing charges and peaks from waveforms. It provides methods for initializing the class, making charges and peaks for events, creating output containers, sorting charges containers, selecting charges for specific pixels, computing histograms of charges, and more. + + Example Usage: + # Create an instance of ChargesMaker + charges_maker = ChargesMaker(run_number=1234, max_events=1000, run_file='data.fits') + + # Initialize the charges maker for a specific trigger type + charges_maker._init_trigger_type(trigger_type='TypeA') + + # Make charges and peaks for events + charges_maker.make(n_events=100, trigger_type=['TypeA', 'TypeB'], method='FullWaveformSum') + + # Get the charges and peaks for a specific trigger type + charges_hg = charges_maker.charges_hg(trigger='TypeA') + charges_lg = charges_maker.charges_lg(trigger='TypeA') + peaks_hg = charges_maker.peak_hg(trigger='TypeA') + peaks_lg = charges_maker.peak_lg(trigger='TypeA') + + # Create a charges container from waveforms + waveforms_container = WaveformsContainer() + charges_container = ChargesMaker.create_from_waveforms(waveforms_container, method='FullWaveformSum') + + # Compute histograms of charges + histo_hg = ChargesMaker.histo_hg(charges_container, n_bins=1000, autoscale=True) + histo_lg = ChargesMaker.histo_lg(charges_container, n_bins=1000, autoscale=True) + + Methods: + - __init__(self, run_number: int, max_events: int = None, run_file=None, *args, **kwargs): Constructor method to initialize the class. + - _init_trigger_type(self, trigger_type : EventType, **kwargs): Method to initialize the charges for a specific trigger type. + - make(self, n_events=np.inf, trigger_type: list = None, restart_from_beginning=False, method: str = "FullWaveformSum", *args, **kwargs): Method to make charges and peaks for events. + - _make_event(self, event : NectarCAMDataContainer, trigger: EventType, method: str = "FullWaveformSum", *args, **kwargs): Method to make charges and peaks for a single event. + - _make_output_container(self, trigger_type : EventType, method: str, *args, **kwargs): Method to create output containers for charges and peaks. + - sort(chargesContainer: ChargesContainer, method='event_id'): Static method to sort charges containers based on a specified method. + - select_charges_hg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): Static method to select charges for specific pixels in the high-gain channel. + - select_charges_lg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): Static method to select charges for specific pixels in the low-gain channel. + - charges_hg(self, trigger : EventType): Method to get the charges in the high-gain channel for a specific trigger type. + - charges_lg(self, trigger : EventType): Method to get the charges in the low-gain channel for a specific trigger type. + - peak_hg(self, trigger : EventType): Method to get the peaks in the high-gain channel for a specific trigger type. + - peak_lg(self, trigger : EventType): Method to get the peaks in the low-gain channel for a specific trigger type. + - create_from_waveforms(waveformsContainer: WaveformsContainer, method: str = "FullWaveformSum", **kwargs) -> ChargesContainer: Static method to create a charges container from waveforms. + - compute_charge(waveformContainer: WaveformsContainer, channel: int, method: str = "FullWaveformSum", **kwargs): Static method to compute charges and peaks from waveforms. + - histo_hg(chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True) -> ma.masked_array: Static method to compute a histogram of charges in the high-gain channel. + - histo_lg(chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True) -> ma.masked_array: Static method to compute a histogram of charges in the low-gain channel. + + Fields: + - __charges_hg: Dictionary to store the charges in the high-gain channel for each trigger type. + - __charges_lg: Dictionary to store the charges in the low-gain channel for each trigger type. + - __peak_hg: Dictionary to store the peaks in the high-gain channel for each trigger type. + - __peak_lg: Dictionary to store the peaks in the low-gain channel for each trigger type. + """ #constructors - def __init__(self,run_number : int,max_events : int = None,run_file = None,*args,**kwargs): + def __init__(self,run_number : int,max_events : int = None,run_file:str = None,*args,**kwargs): """construtor Args: @@ -113,8 +167,18 @@ def __init__(self,run_number : int,max_events : int = None,run_file = None,*args self.__peak_hg = {} self.__peak_lg = {} - def _init_trigger_type(self,trigger_type,**kwargs) : - super()._init_trigger_type(trigger_type,**kwargs) + def _init_trigger_type(self, trigger_type: EventType, **kwargs): + """ + Initializes the ChargesMaker based on the trigger type. + + Args: + trigger_type (EventType): The type of trigger. + **kwargs: Additional keyword arguments. + + Returns: + None + """ + super()._init_trigger_type(trigger_type, **kwargs) name = __class__._get_name_trigger(trigger_type) log.info(f"initialization of the ChargesMaker following trigger type : {name}") self.__charges_hg[f"{name}"] = [] @@ -122,27 +186,48 @@ def _init_trigger_type(self,trigger_type,**kwargs) : self.__peak_hg[f"{name}"] = [] self.__peak_lg[f"{name}"] = [] - - def make(self, - n_events = np.inf, - trigger_type : list = None, - restart_from_begining = False, - method: str = "FullWaveformSum", - *args,**kwargs): - kwargs["method"]=method - return super().make(n_events=n_events, - trigger_type=trigger_type, - restart_from_begining=restart_from_begining, - *args,**kwargs) - - def _make_event(self, - event, - trigger : EventType, - method: str = "FullWaveformSum", - *args, - **kwargs - ) : + def make(self, n_events=np.inf, trigger_type: list = None, restart_from_beginning: bool = False, method: str = "FullWaveformSum", *args, **kwargs): + """ + Makes charges based on the specified arguments. + + Args: + n_events (int): The number of events to process (default is infinity). + trigger_type (list): The type of trigger (default is None). + restart_from_beginning (bool): Whether to restart the processing from the beginning (default is False). + method (str): The method to use for making charges (default is "FullWaveformSum"). + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + + Returns: + The result of the parent class's make method. + """ + kwargs["method"] = method + return super().make(n_events=n_events, trigger_type=trigger_type, restart_from_beginning=restart_from_beginning, *args, **kwargs) + def _make_event(self, + event : NectarCAMDataContainer, + trigger : EventType, + method: str = "FullWaveformSum", + *args, + **kwargs + ) : + """ + Make charges and peaks for a single event. + + Args: + event (NectarCAMDataContainer): The event container that contains the data for a single event. + trigger (EventType): The type of trigger for the event. + method (str, optional): The method to use for computing charges and peaks. Defaults to "FullWaveformSum". + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + + Returns: + None + + Summary: + This method extracts waveforms from the event, computes broken pixels, and then uses an image extractor to calculate charges and peaks for the high-gain and low-gain channels. The results are stored in dictionaries for each trigger type. + ``` + """ wfs_hg_tmp,wfs_lg_tmp = super()._make_event(event = event, trigger = trigger, return_wfs = True, @@ -165,23 +250,39 @@ def _make_event(self, self.__peak_lg[f"{name}"].append(__image[1].tolist()) @staticmethod - def _get_imageExtractor(method,subarray,**kwargs) : + def _get_imageExtractor(method : str,subarray : SubarrayDescription,**kwargs) : + """ + Create an instance of a charge extraction method based on the provided method name and subarray description. + Args: + method (str): The name of the charge extraction method. + subarray (SubarrayDescription): The description of the subarray. + **kwargs (dict): Additional keyword arguments for the charge extraction method. + Returns: + imageExtractor: An instance of the charge extraction method specified by `method` with the provided subarray description and keyword arguments. + """ if not(method in list_ctapipe_charge_extractor or method in list_nectarchain_charge_extractor) : raise ArgumentError(f"method must be in {list_ctapipe_charge_extractor}") - extractor_kwargs = {} for key in eval(method).class_own_traits().keys() : if key in kwargs.keys() : extractor_kwargs[key] = kwargs[key] - if "apply_integration_correction" in eval(method).class_own_traits().keys() : #to change the default behavior of ctapipe extractor extractor_kwargs["apply_integration_correction"] = kwargs.get("apply_integration_correction",False) - log.debug(f"Extracting charges with method {method} and extractor_kwargs {extractor_kwargs}") imageExtractor = eval(method)(subarray,**extractor_kwargs) return imageExtractor - def _make_output_container(self,trigger_type,method : str,*args,**kwargs) : + def _make_output_container(self,trigger_type : EventType, method : str,*args,**kwargs) : + """ + Create an output container for the specified trigger type and method. + Args: + trigger_type (EventType): The type of trigger. + method (str): The name of the charge extraction method. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + Returns: + list: A list of ChargesContainer objects. + """ output = [] for trigger in trigger_type : chargesContainer = ChargesContainer( @@ -210,7 +311,18 @@ def _make_output_container(self,trigger_type,method : str,*args,**kwargs) : return output @staticmethod - def sort(chargesContainer :ChargesContainer, method = 'event_id') : + def sort(chargesContainer :ChargesContainer, method :str = 'event_id') : + """ + Sorts the charges in a ChargesContainer object based on the specified method. + Args: + chargesContainer (ChargesContainer): The ChargesContainer object to be sorted. + method (str, optional): The sorting method. Defaults to 'event_id'. + Returns: + ChargesContainer: A new ChargesContainer object with the charges sorted based on the specified method. + + Raises: + ArgumentError: If the specified method is not valid. + """ output = ChargesContainer( run_number = chargesContainer.run_number, npixels = chargesContainer.npixels, @@ -218,7 +330,6 @@ def sort(chargesContainer :ChargesContainer, method = 'event_id') : pixels_id = chargesContainer.pixels_id, nevents = chargesContainer.nevents, method = chargesContainer.method - ) if method == 'event_id' : index = np.argsort(chargesContainer.event_id) @@ -228,37 +339,86 @@ def sort(chargesContainer :ChargesContainer, method = 'event_id') : else : raise ArgumentError(f"{method} is not a valid method for sorting") return output - @staticmethod def select_charges_hg(chargesContainer :ChargesContainer,pixel_id : np.ndarray) : + """ + Selects the charges from the ChargesContainer object for the given pixel_id and returns the result transposed. + Args: + chargesContainer (ChargesContainer): The ChargesContainer object. + pixel_id (np.ndarray): An array of pixel IDs. + Returns: + np.ndarray: The selected charges from the ChargesContainer object for the given pixel_id, transposed. + """ res = __class__.select_container_array_field(container = chargesContainer,pixel_id = pixel_id,field = 'charges_hg') res = res.transpose(1,0) return res - - @staticmethod def select_charges_lg(chargesContainer : ChargesContainer,pixel_id : np.ndarray) : + """ + Selects the charges from the ChargesContainer object for the given pixel_id and returns the result transposed. + Args: + chargesContainer (ChargesContainer): The ChargesContainer object. + pixel_id (np.ndarray): An array of pixel IDs. + Returns: + np.ndarray: The selected charges from the ChargesContainer object for the given pixel_id, transposed. + """ res = __class__.select_container_array_field(container = chargesContainer,pixel_id = pixel_id,field = 'charges_lg') res = res.transpose(1,0) return res - - - def charges_hg(self,trigger) : return np.array(self.__charges_hg[__class__._get_name_trigger(trigger)],dtype = np.uint16) - def charges_lg(self,trigger) : return np.array(self.__charges_lg[__class__._get_name_trigger(trigger)],dtype = np.uint16) - - def peak_hg(self,trigger) : return np.array(self.__peak_hg[__class__._get_name_trigger(trigger)],dtype = np.uint16) - def peak_lg(self,trigger) : return np.array(self.__peak_lg[__class__._get_name_trigger(trigger)],dtype = np.uint16) + def charges_hg(self,trigger : EventType) : + """ + Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. + Args: + trigger (EventType): The specific trigger type. + Returns: + np.ndarray: The charges for the specific trigger type. + """ + return np.array(self.__charges_hg[__class__._get_name_trigger(trigger)],dtype = np.uint16) + def charges_lg(self,trigger : EventType) : + """ + Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. + Args: + trigger (EventType): The specific trigger type. + Returns: + np.ndarray: The charges for the specific trigger type. + """ + return np.array(self.__charges_lg[__class__._get_name_trigger(trigger)],dtype = np.uint16) + def peak_hg(self,trigger : EventType) : + """ + Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. + Args: + trigger (EventType): The specific trigger type. + Returns: + np.ndarray: The peak charges for the specific trigger type. + """ + return np.array(self.__peak_hg[__class__._get_name_trigger(trigger)],dtype = np.uint16) + def peak_lg(self,trigger : EventType) : + """ + Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. + Args: + trigger (EventType): The specific trigger type. + Returns: + np.ndarray: The peak charges for the specific trigger type. + """ + return np.array(self.__peak_lg[__class__._get_name_trigger(trigger)],dtype = np.uint16) @staticmethod def create_from_waveforms(waveformsContainer: WaveformsContainer, method: str = "FullWaveformSum", **kwargs) -> ChargesContainer: + """ + Create a ChargesContainer object from waveforms using the specified charge extraction method. + Args: + waveformsContainer (WaveformsContainer): The waveforms container object. + method (str, optional): The charge extraction method to use (default is "FullWaveformSum"). + **kwargs: Additional keyword arguments to pass to the charge extraction method. + Returns: + ChargesContainer: The charges container object containing the computed charges and peak times. + """ chargesContainer = ChargesContainer() - for field in waveformsContainer.keys() : if not(field in ["subarray","nsamples","wfs_hg","wfs_lg"]) : chargesContainer[field] = waveformsContainer[field] - log.info(f"computing hg charge with {method} method") charges_hg, peak_hg = __class__.compute_charge(waveformsContainer, constants.HIGH_GAIN, method, **kwargs) charges_hg = np.array(charges_hg, dtype=np.uint16) @@ -270,31 +430,27 @@ def create_from_waveforms(waveformsContainer: WaveformsContainer, method: str = chargesContainer.peak_hg = peak_hg chargesContainer.peak_lg = peak_lg chargesContainer.method = method - return chargesContainer - + @staticmethod def compute_charge(waveformContainer : WaveformsContainer,channel : int,method : str = "FullWaveformSum" ,**kwargs) : - """compute charge from waveforms - + """ + Compute charge from waveforms. Args: - waveformContainer (WaveformsContainer): the waveforms - channel (int): channel you want to compute charges - method (str, optional): ctapipe Image Extractor method method. Defaults to "FullWaveformSum". - + waveformContainer (WaveformsContainer): The waveforms container object. + channel (int): The channel to compute charges for. + method (str, optional): The charge extraction method to use (default is "FullWaveformSum"). + **kwargs: Additional keyword arguments to pass to the charge extraction method. Raises: - ArgumentError: extraction method unknown - ArgumentError: channel unknown - + ArgumentError: If the extraction method is unknown. + ArgumentError: If the channel is unknown. Returns: - output of the extractor called on waveforms + tuple: A tuple containing the computed charges and peak times. """ - #import is here for fix issue with pytest (TypeError : inference is not possible with python <3.9 (Numba conflict bc there is no inference...)) from .extractor.utils import CtapipeExtractor - + imageExtractor = __class__._get_imageExtractor(method = method,subarray = waveformContainer.subarray,**kwargs) - if channel == constants.HIGH_GAIN: out = np.array([CtapipeExtractor.get_image_peak_time(imageExtractor(waveformContainer.wfs_hg[i],__class__.TEL_ID,channel,waveformContainer.broken_pixels_hg)) for i in range(len(waveformContainer.wfs_hg))]).transpose(1,0,2) return out[0],out[1] @@ -303,55 +459,70 @@ def compute_charge(waveformContainer : WaveformsContainer,channel : int,method : return out[0],out[1] else : raise ArgumentError(f"channel must be {constants.LOW_GAIN} or {constants.HIGH_GAIN}") - + @staticmethod - def histo_hg(chargesContainer : ChargesContainer,n_bins : int = 1000,autoscale : bool = True) -> ma.masked_array: - return __class__._histo(chargesContainer = chargesContainer, - field = 'charges_hg', - n_bins = n_bins, - autoscale = autoscale) + def histo_hg(chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True) -> ma.masked_array: + """ + Computes histogram of high gain charges from a ChargesContainer object. + + Args: + chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. + n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. + autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. + + Returns: + ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. + """ + return __class__._histo(chargesContainer=chargesContainer, field='charges_hg', n_bins=n_bins, autoscale=autoscale) @staticmethod - def histo_lg(chargesContainer : ChargesContainer,n_bins : int = 1000,autoscale : bool = True) -> ma.masked_array: - return __class__._histo(chargesContainer = chargesContainer, - field = 'charges_lg', - n_bins = n_bins, - autoscale = autoscale) + def histo_lg(chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True) -> ma.masked_array: + """ + Computes histogram of low gain charges from a ChargesContainer object. + + Args: + chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. + n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. + autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. + + Returns: + ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. + """ + return __class__._histo(chargesContainer=chargesContainer, field='charges_lg', n_bins=n_bins, autoscale=autoscale) @staticmethod - def _histo(chargesContainer : ChargesContainer,field : str, n_bins : int = 1000,autoscale : bool = True) -> ma.masked_array: - """method to compute histogram of HG channel - Numba is used to compute histograms in vectorized way - + def _histo(chargesContainer: ChargesContainer, field: str, n_bins: int = 1000, autoscale: bool = True) -> ma.masked_array: + """ + Computes histogram of charges for a given field from a ChargesContainer object. + Numba is used to compute histograms in a vectorized way. + Args: - n_bins (int, optional): number of bins in charge (ADC counts). Defaults to 1000. - autoscale (bool, optional): auto detect number of bins by pixels (bin witdh = 1 ADC). Defaults to True. - + chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. + field (str): The field name for which the histogram is computed. + n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. + autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. + Returns: - np.ndarray: masked array of charge histograms (histo,charge) + ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. """ - mask_broken_pix = np.array((chargesContainer[field] == chargesContainer[field].mean(axis = 0)).mean(axis=0),dtype = bool) + mask_broken_pix = np.array((chargesContainer[field] == chargesContainer[field].mean(axis=0)).mean(axis=0), dtype=bool) log.debug(f"there are {mask_broken_pix.sum()} broken pixels (charge stays at same level for each events)") - - if autoscale : - all_range = np.arange(np.uint16(np.min(chargesContainer[field].T[~mask_broken_pix].T)) - 0.5,np.uint16(np.max(chargesContainer[field].T[~mask_broken_pix].T)) + 1.5,1) - #hist_ma = ma.masked_array(np.zeros((self[field].shape[1],all_range.shape[0]),dtype = np.uint16), mask=np.zeros((self[field].shape[1],all_range.shape[0]),dtype = bool)) - charge_ma = ma.masked_array((all_range.reshape(all_range.shape[0],1) @ np.ones((1,chargesContainer[field].shape[1]))).T, mask=np.zeros((chargesContainer[field].shape[1],all_range.shape[0]),dtype = bool)) - + + if autoscale: + all_range = np.arange(np.uint16(np.min(chargesContainer[field].T[~mask_broken_pix].T)) - 0.5, np.uint16(np.max(chargesContainer[field].T[~mask_broken_pix].T)) + 1.5, 1) + charge_ma = ma.masked_array((all_range.reshape(all_range.shape[0], 1) @ np.ones((1, chargesContainer[field].shape[1]))).T, mask=np.zeros((chargesContainer[field].shape[1], all_range.shape[0]), dtype=bool)) broxen_pixels_mask = np.array([mask_broken_pix for i in range(charge_ma.shape[1])]).T - #hist_ma.mask = new_data_mask.T start = time.time() - _mask, hist_ma_data = make_histo(chargesContainer[field].T, all_range, mask_broken_pix)#, charge_ma.data, charge_ma.mask, hist_ma.data, hist_ma.mask) - charge_ma.mask = np.logical_or(_mask,broxen_pixels_mask) - hist_ma = ma.masked_array(hist_ma_data,mask = charge_ma.mask) + _mask, hist_ma_data = make_histo(chargesContainer[field].T, all_range, mask_broken_pix) + charge_ma.mask = np.logical_or(_mask, broxen_pixels_mask) + hist_ma = ma.masked_array(hist_ma_data, mask=charge_ma.mask) log.debug(f"histogram hg computation time : {time.time() - start} sec") - - return ma.masked_array((hist_ma,charge_ma)) - - else : - hist = np.array([np.histogram(chargesContainer[field].T[i],bins=n_bins)[0] for i in range(chargesContainer[field].shape[1])]) - charge = np.array([np.histogram(chargesContainer[field].T[i],bins=n_bins)[1] for i in range(chargesContainer[field].shape[1])]) - charge_edges = np.array([np.mean(charge.T[i:i+2],axis = 0) for i in range(charge.shape[1]-1)]).T - - return np.array((hist,charge_edges)) - + + return ma.masked_array((hist_ma, charge_ma)) + + else: + hist = np.array([np.histogram(chargesContainer[field].T[i], bins=n_bins)[0] for i in range(chargesContainer[field].shape[1])]) + charge = np.array([np.histogram(chargesContainer[field].T[i], bins=n_bins)[1] for i in range(chargesContainer[field].shape[1])]) + charge_edges = np.array([np.mean(charge.T[i:i+2], axis=0) for i in range(charge.shape[1]-1)]).T + + return np.array((hist, charge_edges)) \ No newline at end of file diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 308122c1..e945cb8a 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -12,6 +12,7 @@ from ctapipe.containers import EventType from ctapipe.instrument import CameraGeometry from ctapipe_io_nectarcam import constants +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer @@ -20,6 +21,10 @@ __all__ = ["ArrayDataMaker"] +"""The code snippet is a part of a class hierarchy for data processing. +It includes the `BaseMaker` abstract class, the `EventsLoopMaker` and `ArrayDataMaker` subclasses. +These classes are used to perform computations on data from a specific run.""" + class BaseMaker(ABC): """Mother class for all the makers, the role of makers is to do computation on the data. """ @@ -31,7 +36,7 @@ def make(self, *args, **kwargs): """ pass @staticmethod - def load_run(run_number : int,max_events : int = None, run_file = None) : + def load_run(run_number : int,max_events : int = None, run_file = None) -> NectarCAMEventSource: """Static method to load from $NECTARCAMDATA directory data for specified run with max_events Args:self.__run_number = run_number @@ -41,6 +46,7 @@ def load_run(run_number : int,max_events : int = None, run_file = None) : Returns: List[ctapipe_io_nectarcam.NectarCAMEventSource]: List of EventSource for each run files """ + # Load the data from the run file. if run_file is None : generic_filename,_ = DataManagement.findrun(run_number) log.info(f"{str(generic_filename)} will be loaded") @@ -51,78 +57,190 @@ def load_run(run_number : int,max_events : int = None, run_file = None) : return eventsource -class LoopEventsMaker(BaseMaker): - def __init__(self,run_number : int,max_events : int = None,run_file = None,*args,**kwargs): - """construtor +class EventsLoopMaker(BaseMaker): + """ + A class for data processing and computation on events from a specific run. + + Args: + run_number (int): The ID of the run to be processed. + max_events (int, optional): The maximum number of events to be loaded. Defaults to None. + run_file (optional): The specific run file to be loaded. + + Example Usage: + maker = EventsLoopMaker(run_number=1234, max_events=1000) + maker.make(n_events=500) + """ + + def __init__(self, run_number: int, max_events: int = None, run_file=None, *args, **kwargs): + """ + Constructor method that initializes the EventsLoopMaker object. Args: - run_number (int): id of the run to be loaded - maxevents (int, optional): max of events to be loaded. Defaults to 0, to load everythings. - nevents (int, optional) : number of events in run if known (parameter used to save computing time) - run_file (optional) : if provided, will load this run file + run_number (int): The ID of the run to be loaded. + max_events (int, optional): The maximum number of events to be loaded. Defaults to None. + run_file (optional): The specific run file to be loaded. """ - super().__init__(*args,**kwargs) + super().__init__(*args, **kwargs) self.__run_number = run_number self.__run_file = run_file self.__max_events = max_events - self.__reader = __class__.load_run(run_number,max_events,run_file = run_file) + self.__reader = __class__.load_run(run_number, max_events, run_file=run_file) - #from reader members + # from reader members self.__npixels = self.__reader.camera_config.num_pixels self.__pixels_id = self.__reader.camera_config.expected_pixels_id - + log.info(f"N pixels : {self.npixels}") - def make(self,n_events = np.inf, restart_from_begining = False,*args,**kwargs) : - if restart_from_begining : - log.debug('restart from begining : creation of the EventSource reader') - self.__reader = __class__.load_run(self.__run_number,self.__max_events,run_file = self.__run_file) + def make(self, n_events=np.inf, restart_from_beginning : bool =False, *args, **kwargs): + """ + Method to iterate over the events and perform computations on each event. + + Args: + n_events (int, optional): The number of events to process. Defaults to np.inf. + restart_from_beginning (bool, optional): Whether to restart from the beginning of the run. Defaults to False. + """ + if restart_from_beginning: + log.debug('restart from beginning : creation of the EventSource reader') + self.__reader = __class__.load_run(self.__run_number, self.__max_events, run_file=self.__run_file) n_traited_events = 0 - for i,event in enumerate(self.__reader): - if i%100 == 0: + for i, event in enumerate(self.__reader): + if i % 100 == 0: log.info(f"reading event number {i}") - self._make_event(event,*args,**kwargs) + self._make_event(event, *args, **kwargs) n_traited_events += 1 - if n_traited_events >= n_events : + if n_traited_events >= n_events: break - @abstractmethod - def _make_event(self, event : EventType) : pass + def _make_event(self, event: NectarCAMDataContainer): + """ + Abstract method that needs to be implemented by subclasses. + This method is called for each event in the run to perform computations on the event. + + Args: + event (NectarCAMDataContainer): The event to perform computations on. + """ + pass - @property - def _run_file(self) : return self.__run_file @property - def _max_events(self) : return self.__max_events + def _run_file(self): + """ + Getter method for the _run_file attribute. + """ + return self.__run_file + @property - def _reader(self) : return self.__reader + def _max_events(self): + """ + Getter method for the _max_events attribute. + """ + return self.__max_events + + @property + def _reader(self): + """ + Getter method for the _reader attribute. + """ + return self.__reader + @_reader.setter - def _reader(self,value) : - if isinstance(value,NectarCAMEventSource) : + def _reader(self, value): + """ + Setter method to set a new NectarCAMEventSource to the _reader attribute. + + Args: + value: a NectarCAMEventSource instance. + """ + if isinstance(value, NectarCAMEventSource): self.__reader = value - else : + else: raise TypeError("The reader must be a NectarCAMEventSource") + @property - def _npixels(self) : return self.__npixels + def _npixels(self): + """ + Getter method for the _npixels attribute. + """ + return self.__npixels + @property - def _pixels_id(self) : return self.__pixels_id + def _pixels_id(self): + """ + Getter method for the _pixels_id attribute. + """ + return self.__pixels_id + @property - def _run_number(self) : return self.__run_number + def _run_number(self): + """ + Getter method for the _run_number attribute. + """ + return self.__run_number + @property - def reader(self) : return copy.deepcopy(self.__reader) + def reader(self): + """ + Getter method for the reader attribute. + """ + return copy.deepcopy(self.__reader) + @property - def npixels(self) : return copy.deepcopy(self.__npixels) + def npixels(self): + """ + Getter method for the npixels attribute. + """ + return copy.deepcopy(self.__npixels) + @property - def pixels_id(self) : return copy.deepcopy(self.__pixels_id) + def pixels_id(self): + """ + Getter method for the pixels_id attribute. + """ + return copy.deepcopy(self.__pixels_id) + @property - def run_number(self) : return copy.deepcopy(self.__run_number) + def run_number(self): + """ + Getter method for the run_number attribute. + """ + return copy.deepcopy(self.__run_number) -class ArrayDataMaker(LoopEventsMaker) : +class ArrayDataMaker(EventsLoopMaker) : + """ + Class used to loop over the events of a run and to extract informations that are stored in arrays. + Example Usage: + - Create an instance of the ArrayDataMaker class + maker = ArrayDataMaker(run_number=1234, max_events=1000) + + - Perform data processing on the specified run + maker.make(n_events=500, trigger_type=[EventType.SKY]) + + - Access the computed data + ucts_timestamp = maker.ucts_timestamp(EventType.SKY) + event_type = maker.event_type(EventType.SKY) + + Inputs: + - run_number (int): The ID of the run to be processed. + - max_events (int, optional): The maximum number of events to be loaded. Defaults to None, which loads all events. + - run_file (optional): The specific run file to be loaded. + + Flow: + 1. The ArrayDataMaker class is initialized with the run number, maximum events, or a run file. + 2. The make method is called to perform data processing on the specified run. + 3. The _make_event method is called for each event in the run to extract and store relevant data. + 4. The computed data is stored in instance variables. + 5. The computed data can be accessed using getter methods. + + Outputs: + - Computed data such as UCTS timestamps, event types, and event IDs can be accessed using getter methods provided by the ArrayDataMaker class. + """ + TEL_ID = 0 CAMERA_NAME = "NectarCam-003" CAMERA = CameraGeometry.from_name(CAMERA_NAME) @@ -148,7 +266,16 @@ def __init__(self,run_number : int,max_events : int = None,run_file = None,*args self.__broken_pixels_hg = {} self.__broken_pixels_lg = {} - def _init_trigger_type(self,trigger,**kwargs) : + def _init_trigger_type(self, trigger : EventType, **kwargs): + """ + Initializes empty lists for different trigger types in the ArrayDataMaker class. + + Args: + trigger (EventType): The trigger type for which the lists are being initialized. + + Returns: + None. The method only initializes the empty lists for the trigger type. + """ name = __class__._get_name_trigger(trigger) self.__ucts_timestamp[f"{name}"] = [] self.__ucts_busy_counter[f"{name}"] = [] @@ -159,57 +286,103 @@ def _init_trigger_type(self,trigger,**kwargs) : self.__broken_pixels_hg[f"{name}"] = [] self.__broken_pixels_lg[f"{name}"] = [] - @staticmethod - def _compute_broken_pixels(wfs_hg,wfs_lg,**kwargs) : - log.warning("computation of broken pixels is not yet implemented") - return np.zeros((wfs_hg.shape[:-1]),dtype = bool),np.zeros((wfs_hg.shape[:-1]),dtype = bool) @staticmethod - def _compute_broken_pixels_event(event : EventType,pixels_id,**kwargs) : + def _compute_broken_pixels(wfs_hg, wfs_lg, **kwargs): + """ + Computes broken pixels for high and low gain waveforms. + Args: + wfs_hg (ndarray): High gain waveforms. + wfs_lg (ndarray): Low gain waveforms. + **kwargs: Additional keyword arguments. + Returns: + tuple: Two arrays of zeros with the same shape as `wfs_hg` (or `wfs_lg`) but without the last dimension. + """ log.warning("computation of broken pixels is not yet implemented") - return np.zeros((len(pixels_id)),dtype = bool),np.zeros((len(pixels_id)),dtype = bool) + return np.zeros((wfs_hg.shape[:-1]), dtype=bool), np.zeros((wfs_hg.shape[:-1]), dtype=bool) @staticmethod - def _get_name_trigger(trigger : EventType) : - if trigger is None : + def _compute_broken_pixels_event(event: NectarCAMDataContainer, pixels_id : np.ndarray, **kwargs): + """ + Computes broken pixels for a specific event and pixel IDs. + Args: + event (NectarCAMDataContainer): An event. + pixels_id (list or np.ndarray): IDs of pixels. + **kwargs: Additional keyword arguments. + Returns: + tuple: Two arrays of zeros with the length of `pixels_id`. + """ + log.warning("computation of broken pixels is not yet implemented") + return np.zeros((len(pixels_id)), dtype=bool), np.zeros((len(pixels_id)), dtype=bool) + + @staticmethod + def _get_name_trigger(trigger: EventType): + """ + Gets the name of a trigger event. + Args: + trigger (EventType): A trigger event. + Returns: + str: The name of the trigger event. + """ + if trigger is None: name = "None" - else : + else: name = trigger.name return name + - def make(self,n_events = np.inf, trigger_type : list = None, restart_from_begining = False,*args,**kwargs) : - """mathod to extract data from the EventSource + def make(self, n_events=np.inf, trigger_type: list = None, restart_from_begining : bool=False, *args, **kwargs): + """ + Method to extract data from the EventSource. Args: - trigger_type (list[EventType], optional): only events with the asked trigger type will be use. Defaults to None. - compute_trigger_patern (bool, optional): To recompute on our side the trigger patern. Defaults to False. + n_events (int, optional): The maximum number of events to process. Default is np.inf. + trigger_type (list[EventType], optional): Only events with the specified trigger types will be used. Default is None. + restart_from_begining (bool, optional): Whether to restart the event source reader. Default is False. + *args: Additional arguments that can be passed to the method. + **kwargs: Additional keyword arguments that can be passed to the method. + + Returns: + The output container created by the _make_output_container method. """ - if ~np.isfinite(n_events) : + if ~np.isfinite(n_events): log.warning('no needed events number specified, it may cause a memory error') - if isinstance(trigger_type,EventType) or trigger_type is None : + if isinstance(trigger_type, EventType) or trigger_type is None: trigger_type = [trigger_type] - for _trigger_type in trigger_type : - self._init_trigger_type(_trigger_type) + for _trigger_type in trigger_type: + self._init_trigger_type(_trigger_type) - if restart_from_begining : + if restart_from_begining: log.debug('restart from begining : creation of the EventSource reader') - self._reader = __class__.load_run(self._run_number,self._max_events,run_file = self._run_file) + self._reader = __class__.load_run(self._run_number, self._max_events, run_file=self._run_file) n_traited_events = 0 - for i,event in enumerate(self._reader): - if i%100 == 0: + for i, event in enumerate(self._reader): + if i % 100 == 0: log.info(f"reading event number {i}") - for trigger in trigger_type : - if (trigger is None) or (trigger == event.trigger.event_type) : - self._make_event(event,trigger,*args,**kwargs) + for trigger in trigger_type: + if (trigger is None) or (trigger == event.trigger.event_type): + self._make_event(event, trigger, *args, **kwargs) n_traited_events += 1 - if n_traited_events >= n_events : + if n_traited_events >= n_events: break - return self._make_output_container(trigger_type,*args,**kwargs) + return self._make_output_container(trigger_type, *args, **kwargs) + + + def _make_event(self, event : NectarCAMDataContainer, trigger : EventType, *args, **kwargs): + """ + Method to extract data from the event. + Args: + event (NectarCAMDataContainer): The event object. + trigger (EventType): The trigger type. + *args: Additional arguments that can be passed to the method. + **kwargs: Additional keyword arguments that can be passed to the method. - def _make_event(self,event,trigger,*args,**kwargs) : + Returns: + If the return_wfs keyword argument is True, the method returns the high and low gain waveforms from the event. + """ name = __class__._get_name_trigger(trigger) self.__event_id[f'{name}'].append(np.uint16(event.index.event_id)) self.__ucts_timestamp[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_timestamp) @@ -217,21 +390,33 @@ def _make_event(self,event,trigger,*args,**kwargs) : self.__ucts_busy_counter[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_busy_counter) self.__ucts_event_counter[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_event_counter) self.__trig_patter_all[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.trigger_pattern.T) - - if kwargs.get("return_wfs",False) : - get_wfs_hg=event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id] - get_wfs_lg=event.r0.tel[0].waveform[constants.LOW_GAIN][self.pixels_id] - return get_wfs_hg,get_wfs_lg + + if kwargs.get("return_wfs", False): + get_wfs_hg = event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id] + get_wfs_lg = event.r0.tel[0].waveform[constants.LOW_GAIN][self.pixels_id] + return get_wfs_hg, get_wfs_lg @abstractmethod def _make_output_container(self) : pass @staticmethod - def select_container_array_field(container :ArrayDataContainer,pixel_id : np.ndarray,field : str) : - mask_contain_pixels_id = np.array([pixel in container.pixels_id for pixel in pixel_id],dtype = bool) - for pixel in pixel_id[~mask_contain_pixels_id] : log.warning(f"You asked for pixel_id {pixel} but it is not present in this container, skip this one") - res = np.array([np.take(container[field],np.where(container.pixels_id == pixel)[0][0],axis = 1) for pixel in pixel_id[mask_contain_pixels_id]]) + def select_container_array_field(container: ArrayDataContainer, pixel_id: np.ndarray, field: str) -> np.ndarray: + """ + Selects specific fields from an ArrayDataContainer object based on a given list of pixel IDs. + + Args: + container (ArrayDataContainer): An object of type ArrayDataContainer that contains the data. + pixel_id (ndarray): An array of pixel IDs for which the data needs to be selected. + field (str): The name of the field to be selected from the container. + + Returns: + ndarray: An array containing the selected data for the given pixel IDs. + """ + mask_contain_pixels_id = np.array([pixel in container.pixels_id for pixel in pixel_id], dtype=bool) + for pixel in pixel_id[~mask_contain_pixels_id]: + log.warning(f"You asked for pixel_id {pixel} but it is not present in this container, skip this one") + res = np.array([np.take(container[field], np.where(container.pixels_id == pixel)[0][0], axis=1) for pixel in pixel_id[mask_contain_pixels_id]]) ####could be nice to return np.ma.masked_array(data = res, mask = container.broken_pixels_hg.transpose(res.shape[1],res.shape[0],res.shape[2])) return res @@ -268,32 +453,182 @@ def merge(container_a : ArrayDataContainer,container_b : ArrayDataContainer) -> @property - def nsamples(self) : return copy.deepcopy(self.__nsamples) + def nsamples(self) : + """ + Returns a deep copy of the nsamples attribute. + + Returns: + np.ndarray: A deep copy of the nsamples attribute. + """ + return copy.deepcopy(self.__nsamples) + @property - def _nsamples(self) : return self.__nsamples - def nevents(self,trigger) : return len(self.__event_id[__class__._get_name_trigger(trigger)]) + def _nsamples(self) : + """ + Returns the nsamples attribute. + + Returns: + np.ndarray: The nsamples attribute. + """ + return self.__nsamples + + def nevents(self,trigger : EventType) : + """ + Returns the number of events for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the number of events is requested. + + Returns: + int: The number of events for the specified trigger type. + """ + return len(self.__event_id[__class__._get_name_trigger(trigger)]) + @property - def _broken_pixels_hg(self) : return self.__broken_pixels_hg - def broken_pixels_hg(self,trigger) : return np.array(self.__broken_pixels_hg[__class__._get_name_trigger(trigger)],dtype = bool) + def _broken_pixels_hg(self) : + """ + Returns the broken_pixels_hg attribute. + + Returns: + np.ndarray: The broken_pixels_hg attribute. + """ + return self.__broken_pixels_hg + + def broken_pixels_hg(self,trigger : EventType) : + """ + Returns an array of broken pixels for high gain for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the broken pixels for high gain are requested. + + Returns: + np.ndarray: An array of broken pixels for high gain for the specified trigger type. + """ + return np.array(self.__broken_pixels_hg[__class__._get_name_trigger(trigger)],dtype = bool) + @property - def _broken_pixels_lg(self) : return self.__broken_pixels_lg - def broken_pixels_lg(self,trigger) : return np.array(self.__broken_pixels_lg[__class__._get_name_trigger(trigger)],dtype = bool) - def ucts_timestamp(self,trigger) : return np.array(self.__ucts_timestamp[__class__._get_name_trigger(trigger)],dtype = np.uint64) - def ucts_busy_counter(self,trigger) : return np.array(self.__ucts_busy_counter[__class__._get_name_trigger(trigger)],dtype = np.uint32) - def ucts_event_counter(self,trigger) : return np.array(self.__ucts_event_counter[__class__._get_name_trigger(trigger)],dtype = np.uint32) - def event_type(self,trigger) : return np.array(self.__event_type[__class__._get_name_trigger(trigger)],dtype = np.uint8) - def event_id(self,trigger) : return np.array(self.__event_id[__class__._get_name_trigger(trigger)],dtype = np.uint32) - def multiplicity(self,trigger) : + def _broken_pixels_lg(self) : + """ + Returns the broken_pixels_lg attribute. + + Returns: + np.ndarray: The broken_pixels_lg attribute. + """ + return self.__broken_pixels_lg + + def broken_pixels_lg(self,trigger : EventType) : + """ + Returns an array of broken pixels for low gain for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the broken pixels for low gain are requested. + + Returns: + np.ndarray: An array of broken pixels for low gain for the specified trigger type. + """ + return np.array(self.__broken_pixels_lg[__class__._get_name_trigger(trigger)],dtype = bool) + + def ucts_timestamp(self,trigger : EventType) : + """ + Returns an array of UCTS timestamps for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the UCTS timestamps are requested. + + Returns: + np.ndarray: An array of UCTS timestamps for the specified trigger type. + """ + return np.array(self.__ucts_timestamp[__class__._get_name_trigger(trigger)],dtype = np.uint64) + + def ucts_busy_counter(self,trigger : EventType) : + """ + Returns an array of UCTS busy counters for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the UCTS busy counters are requested. + + Returns: + np.ndarray: An array of UCTS busy counters for the specified trigger type. + """ + return np.array(self.__ucts_busy_counter[__class__._get_name_trigger(trigger)],dtype = np.uint32) + + def ucts_event_counter(self,trigger : EventType) : + """ + Returns an array of UCTS event counters for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the UCTS event counters are requested. + + Returns: + np.ndarray: An array of UCTS event counters for the specified trigger type. + """ + return np.array(self.__ucts_event_counter[__class__._get_name_trigger(trigger)],dtype = np.uint32) + + def event_type(self,trigger : EventType) : + """ + Returns an array of event types for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the event types are requested. + + Returns: + np.ndarray: An array of event types for the specified trigger type. + """ + return np.array(self.__event_type[__class__._get_name_trigger(trigger)],dtype = np.uint8) + + def event_id(self,trigger : EventType) : + """ + Returns an array of event IDs for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the event IDs are requested. + + Returns: + np.ndarray: An array of event IDs for the specified trigger type. + """ + return np.array(self.__event_id[__class__._get_name_trigger(trigger)],dtype = np.uint32) + + def multiplicity(self,trigger : EventType) : + """ + Returns an array of multiplicities for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the multiplicities are requested. + + Returns: + np.ndarray: An array of multiplicities for the specified trigger type. + """ tmp = self.trig_pattern(trigger) if len(tmp) == 0 : return np.array([]) else : return np.uint16(np.count_nonzero(tmp,axis = 1)) - def trig_pattern(self,trigger) : + + def trig_pattern(self,trigger : EventType) : + """ + Returns an array of trigger patterns for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the trigger patterns are requested. + + Returns: + np.ndarray: An array of trigger patterns for the specified trigger type. + """ tmp = self.trig_pattern_all(trigger) if len(tmp) == 0 : return np.array([]) else : return tmp.any(axis = 2) - def trig_pattern_all(self,trigger) : return np.array(self.__trig_patter_all[f"{__class__._get_name_trigger(trigger)}"],dtype = bool) + + def trig_pattern_all(self,trigger : EventType) : + """ + Returns an array of trigger patterns for all events for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the trigger patterns for all events are requested. + + Returns: + np.ndarray: An array of trigger patterns for all events for the specified trigger type. + """ + return np.array(self.__trig_patter_all[f"{__class__._get_name_trigger(trigger)}"],dtype = bool) diff --git a/src/nectarchain/makers/waveformsMakers.py b/src/nectarchain/makers/waveformsMakers.py index ecc01603..f9ff48e4 100644 --- a/src/nectarchain/makers/waveformsMakers.py +++ b/src/nectarchain/makers/waveformsMakers.py @@ -11,6 +11,8 @@ from ctapipe.instrument import SubarrayDescription from ctapipe_io_nectarcam import constants from ctapipe.containers import EventType +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer + from ..data.container import WaveformsContainer from .core import ArrayDataMaker @@ -47,6 +49,19 @@ def create_from_events_list(events_list : list, subarray : SubarrayDescription, pixels_id : int, ) : + """Create a container for the extracted waveforms from a list of events. + + Args: + events_list (list[NectarCAMDataContainer]): A list of events to extract waveforms from. + run_number (int): The ID of the run to be loaded. + npixels (int): The number of pixels in the waveforms. + nsamples (int): The number of samples in the waveforms. + subarray (SubarrayDescription): The subarray description instance. + pixels_id (int): The ID of the pixels to extract waveforms from. + + Returns: + WaveformsContainer: A container object that contains the extracted waveforms and other relevant information. + """ container = WaveformsContainer( run_number = run_number, npixels = npixels, @@ -93,7 +108,13 @@ def create_from_events_list(events_list : list, return container - def _init_trigger_type(self,trigger_type,**kwargs) : + def _init_trigger_type(self,trigger_type : EventType,**kwargs) : + """Initialize the waveformsMaker following the trigger type. + + Args: + trigger_type: The type of trigger. + + """ super()._init_trigger_type(trigger_type,**kwargs) name = __class__._get_name_trigger(trigger_type) log.info(f"initialization of the waveformsMaker following trigger type : {name}") @@ -102,11 +123,18 @@ def _init_trigger_type(self,trigger_type,**kwargs) : def _make_event(self, - event, + event : NectarCAMDataContainer, trigger : EventType, *args, **kwargs ) : + """Process an event and extract waveforms. + + Args: + event (NectarCAMDataContainer): The event to process and extract waveforms from. + trigger (EventType): The type of trigger for the event. + + """ wfs_hg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) wfs_lg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) @@ -123,7 +151,15 @@ def _make_event(self, self._broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) self._broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) - def _make_output_container(self,trigger_type,*args,**kwargs) : + def _make_output_container(self,trigger_type : EventType,*args,**kwargs) : + """Make the output container for the selected trigger types. + + Args: + trigger_type (EventType): The selected trigger types. + + Returns: + list[WaveformsContainer]: A list of output containers for the selected trigger types. + """ output = [] for trigger in trigger_type : waveformsContainer = WaveformsContainer( @@ -151,7 +187,16 @@ def _make_output_container(self,trigger_type,*args,**kwargs) : return output @staticmethod - def sort(waveformsContainer :WaveformsContainer, method = 'event_id') : + def sort(waveformsContainer :WaveformsContainer, method : str = 'event_id') : + """Sort the waveformsContainer based on a specified method. + + Args: + waveformsContainer (WaveformsContainer): The waveformsContainer to be sorted. + method (str, optional): The sorting method. Defaults to 'event_id'. + + Returns: + WaveformsContainer: The sorted waveformsContainer. + """ output = WaveformsContainer( run_number = waveformsContainer.run_number, npixels = waveformsContainer.npixels, @@ -171,13 +216,31 @@ def sort(waveformsContainer :WaveformsContainer, method = 'event_id') : return output @staticmethod - def select_waveforms_hg(waveformsContainer:WaveformsContainer,pixel_id : np.ndarray) : + def select_waveforms_hg(waveformsContainer:WaveformsContainer,pixel_id : np.ndarray) : + """Select HIGH GAIN waveforms from the container. + + Args: + waveformsContainer (WaveformsContainer): The container object that contains the waveforms. + pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms from the container. + + Returns: + np.ndarray: An array of selected waveforms from the container. + """ res = __class__.select_container_array_field(container = waveformsContainer,pixel_id = pixel_id,field = 'wfs_lg') res = res.transpose(1,0,2) return res @staticmethod def select_waveforms_lg(waveformsContainer:WaveformsContainer,pixel_id : np.ndarray) : + """Select LOW GAIN waveforms from the container. + + Args: + waveformsContainer (WaveformsContainer): The container object that contains the waveforms. + pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms from the container. + + Returns: + np.ndarray: An array of selected waveforms from the container. + """ res = __class__.select_container_array_field(container = waveformsContainer,pixel_id = pixel_id,field = 'wfs_hg') res = res.transpose(1,0,2) return res @@ -185,13 +248,62 @@ def select_waveforms_lg(waveformsContainer:WaveformsContainer,pixel_id : np.ndar @property - def _geometry(self) : return self.__geometry + def _geometry(self): + """ + Returns the private __geometry attribute of the WaveformsMaker class. + + :return: The value of the private __geometry attribute. + """ + return self.__geometry + @property - def _subarray(self) : return self.__subarray + def _subarray(self): + """ + Returns the private __subarray attribute of the WaveformsMaker class. + + :return: The value of the private __subarray attribute. + """ + return self.__subarray @property - def geometry(self) : return copy.deepcopy(self.__geometry) + def geometry(self): + """ + Returns a deep copy of the geometry attribute. + + Returns: + A deep copy of the geometry attribute. + """ + return copy.deepcopy(self.__geometry) + @property - def subarray(self) : return copy.deepcopy(self.__subarray) - def wfs_hg(self,trigger) : return np.array(self.__wfs_hg[__class__._get_name_trigger(trigger)],dtype = np.uint16) - def wfs_lg(self,trigger) : return np.array(self.__wfs_lg[__class__._get_name_trigger(trigger)],dtype = np.uint16) - \ No newline at end of file + def subarray(self): + """ + Returns a deep copy of the subarray attribute. + + Returns: + A deep copy of the subarray attribute. + """ + return copy.deepcopy(self.__subarray) + + def wfs_hg(self, trigger: EventType): + """ + Returns the waveform data for the specified trigger type. + + Args: + trigger (EventType): The type of trigger for which the waveform data is requested. + + Returns: + An array of waveform data for the specified trigger type. + """ + return np.array(self.__wfs_hg[__class__._get_name_trigger(trigger)], dtype=np.uint16) + + def wfs_lg(self, trigger: EventType): + """ + Returns the waveform data for the specified trigger type in the low gain channel. + + Args: + trigger (EventType): The type of trigger for which the waveform data is requested. + + Returns: + An array of waveform data for the specified trigger type in the low gain channel. + """ + return np.array(self.__wfs_lg[__class__._get_name_trigger(trigger)], dtype=np.uint16) \ No newline at end of file From 0023b45188ba86b5cf1af142f48bc7574a0118b4 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Sun, 17 Sep 2023 05:28:08 +0200 Subject: [PATCH 56/62] follow pre-commit hook --- src/nectarchain/__init__.py | 2 +- src/nectarchain/data/__init__.py | 2 +- src/nectarchain/data/container/__init__.py | 4 +- .../data/container/chargesContainer.py | 180 +++-- src/nectarchain/data/container/core.py | 67 +- .../data/container/tests/test_charge.py | 185 +++--- .../data/container/tests/test_waveforms.py | 105 +-- .../data/container/waveformsContainer.py | 223 ++++--- src/nectarchain/data/management.py | 147 ++-- src/nectarchain/display/__init__.py | 2 +- src/nectarchain/display/display.py | 63 +- src/nectarchain/makers/__init__.py | 2 +- .../makers/calibration/__init__.py | 4 +- src/nectarchain/makers/calibration/core.py | 42 +- .../makers/calibration/flatfieldMakers.py | 20 +- .../calibration/gain/FlatFieldSPEMakers.py | 625 ++++++++++++------ .../calibration/gain/PhotoStatisticMakers.py | 263 +++++--- .../calibration/gain/WhiteTargetSPEMakers.py | 20 +- .../makers/calibration/gain/__init__.py | 5 +- .../makers/calibration/gain/gainMakers.py | 46 +- .../makers/calibration/gain/parameters.py | 139 ++-- .../gain/tests/test_FlatFieldSPEMakers.py | 126 ++-- .../calibration/gain/tests/test_gainMakers.py | 22 +- .../makers/calibration/gain/utils/__init__.py | 4 +- .../makers/calibration/gain/utils/error.py | 37 +- .../makers/calibration/gain/utils/utils.py | 214 +++--- .../makers/calibration/pedestalMakers.py | 19 +- .../makers/calibration/tests/test_core.py | 35 +- src/nectarchain/makers/chargesMakers.py | 575 ++++++++++------ src/nectarchain/makers/core.py | 348 ++++++---- src/nectarchain/makers/extractor/__init__.py | 2 +- .../makers/extractor/charge_extractor.py | 116 ++-- .../makers/extractor/tests/test_utils.py | 12 +- src/nectarchain/makers/extractor/utils.py | 10 +- .../makers/tests/test_chargesMakers.py | 237 ++++--- src/nectarchain/makers/tests/test_core.py | 49 +- .../makers/tests/test_waveformsMakers.py | 197 ++++-- src/nectarchain/makers/waveformsMakers.py | 263 ++++---- src/nectarchain/tests/test_version.py | 2 +- .../ggrolleron/gain_PhotoStat_computation.py | 195 +++--- .../gain_SPEfit_combined_computation.py | 257 +++---- .../ggrolleron/gain_SPEfit_computation.py | 221 ++++--- .../ggrolleron/load_wfs_compute_charge.py | 544 ++++++++------- .../user_scripts/ggrolleron/test.py | 127 ++-- 44 files changed, 3460 insertions(+), 2298 deletions(-) diff --git a/src/nectarchain/__init__.py b/src/nectarchain/__init__.py index 0d42b290..2d735710 100644 --- a/src/nectarchain/__init__.py +++ b/src/nectarchain/__init__.py @@ -1,3 +1,3 @@ from .version import __version__ -__all__ = ['__version__'] \ No newline at end of file +__all__ = ["__version__"] diff --git a/src/nectarchain/data/__init__.py b/src/nectarchain/data/__init__.py index 4db71b78..1506a100 100644 --- a/src/nectarchain/data/__init__.py +++ b/src/nectarchain/data/__init__.py @@ -1,2 +1,2 @@ +from .container import * from .management import * -from .container import * \ No newline at end of file diff --git a/src/nectarchain/data/container/__init__.py b/src/nectarchain/data/container/__init__.py index 9df719d0..87fa166c 100644 --- a/src/nectarchain/data/container/__init__.py +++ b/src/nectarchain/data/container/__init__.py @@ -1,3 +1,3 @@ -from .waveformsContainer import * from .chargesContainer import * -from .core import * \ No newline at end of file +from .core import * +from .waveformsContainer import * diff --git a/src/nectarchain/data/container/chargesContainer.py b/src/nectarchain/data/container/chargesContainer.py index 90bac147..1d985850 100644 --- a/src/nectarchain/data/container/chargesContainer.py +++ b/src/nectarchain/data/container/chargesContainer.py @@ -1,17 +1,20 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers -import numpy as np -from ctapipe.containers import Field -from abc import ABC import os +from abc import ABC from pathlib import Path + +import numpy as np from astropy.io import fits +from ctapipe.containers import Field from .core import ArrayDataContainer + class ChargesContainer(ArrayDataContainer): """ A container that holds information about charges from a specific run. @@ -24,28 +27,14 @@ class ChargesContainer(ArrayDataContainer): method (str): The charge extraction method used. """ - charges_hg = Field( - type=np.ndarray, - description='The high gain charges' - ) - charges_lg = Field( - type=np.ndarray, - description='The low gain charges' - ) - peak_hg = Field( - type=np.ndarray, - description='The high gain peak time' - ) - peak_lg = Field( - type=np.ndarray, - description='The low gain peak time' - ) - method = Field( - type=str, - description='The charge extraction method used' - ) - -class ChargesContainerIO(ABC) : + charges_hg = Field(type=np.ndarray, description="The high gain charges") + charges_lg = Field(type=np.ndarray, description="The low gain charges") + peak_hg = Field(type=np.ndarray, description="The high gain peak time") + peak_lg = Field(type=np.ndarray, description="The low gain peak time") + method = Field(type=str, description="The charge extraction method used") + + +class ChargesContainerIO(ABC): """ The `ChargesContainerIO` class provides methods for writing and loading `ChargesContainer` instances to/from FITS files. @@ -68,8 +57,9 @@ class ChargesContainerIO(ABC) : Fields: The `ChargesContainerIO` class does not have any fields. """ + @staticmethod - def write(path : Path, container : ChargesContainer,**kwargs) -> None: + def write(path: Path, container: ChargesContainer, **kwargs) -> None: """Write a ChargesContainer instance to a FITS file. Args: path (str): The directory where the FITS file will be saved. @@ -87,51 +77,97 @@ def write(path : Path, container : ChargesContainer,**kwargs) -> None: chargesContainer = ChargesContainer() ChargesContainerIO.write(path, chargesContainer, suffix="v1", overwrite=True) """ - suffix = kwargs.get("suffix","") - if suffix != "" : suffix = f"_{suffix}" + suffix = kwargs.get("suffix", "") + if suffix != "": + suffix = f"_{suffix}" log.info(f"saving in {path}") - os.makedirs(path,exist_ok = True) + os.makedirs(path, exist_ok=True) hdr = fits.Header() - hdr['RUN'] = container.run_number - hdr['NEVENTS'] = container.nevents - hdr['NPIXELS'] = container.npixels - hdr['METHOD'] = container.method - hdr['CAMERA'] = container.camera - hdr['COMMENT'] = f"The charge containeur for run {container.run_number} with {container.method} method : primary is the pixels id, then you can find HG charge, LG charge, HG peak and LG peak, 2 last HDU are composed of event properties and trigger patern" - - primary_hdu = fits.PrimaryHDU(container.pixels_id,header=hdr) - charge_hg_hdu = fits.ImageHDU(container.charges_hg,name = "HG charge") - charge_lg_hdu = fits.ImageHDU(container.charges_lg,name = "LG charge") - peak_hg_hdu = fits.ImageHDU(container.peak_hg, name = 'HG peak time') - peak_lg_hdu = fits.ImageHDU(container.peak_lg, name = 'LG peak time') - - col1 = fits.Column(array = container.broken_pixels_hg, name = "HG broken pixels", format = f'{container.broken_pixels_hg.shape[1]}L') - col2 = fits.Column(array = container.broken_pixels_lg, name = "LG broken pixels", format = f'{container.broken_pixels_lg.shape[1]}L') + hdr["RUN"] = container.run_number + hdr["NEVENTS"] = container.nevents + hdr["NPIXELS"] = container.npixels + hdr["METHOD"] = container.method + hdr["CAMERA"] = container.camera + hdr[ + "COMMENT" + ] = f"The charge containeur for run {container.run_number} with {container.method} method : primary is the pixels id, then you can find HG charge, LG charge, HG peak and LG peak, 2 last HDU are composed of event properties and trigger patern" + + primary_hdu = fits.PrimaryHDU(container.pixels_id, header=hdr) + charge_hg_hdu = fits.ImageHDU(container.charges_hg, name="HG charge") + charge_lg_hdu = fits.ImageHDU(container.charges_lg, name="LG charge") + peak_hg_hdu = fits.ImageHDU(container.peak_hg, name="HG peak time") + peak_lg_hdu = fits.ImageHDU(container.peak_lg, name="LG peak time") + + col1 = fits.Column( + array=container.broken_pixels_hg, + name="HG broken pixels", + format=f"{container.broken_pixels_hg.shape[1]}L", + ) + col2 = fits.Column( + array=container.broken_pixels_lg, + name="LG broken pixels", + format=f"{container.broken_pixels_lg.shape[1]}L", + ) coldefs = fits.ColDefs([col1, col2]) - broken_pixels = fits.BinTableHDU.from_columns(coldefs,name = 'trigger patern') - - col1 = fits.Column(array = container.event_id, name = "event_id", format = '1I') - col2 = fits.Column(array = container.event_type, name = "event_type", format = '1I') - col3 = fits.Column(array = container.ucts_timestamp, name = "ucts_timestamp", format = '1K') - col4 = fits.Column(array = container.ucts_busy_counter, name = "ucts_busy_counter", format = '1I') - col5 = fits.Column(array = container.ucts_event_counter, name = "ucts_event_counter", format = '1I') - col6 = fits.Column(array = container.multiplicity, name = "multiplicity", format = '1I') + broken_pixels = fits.BinTableHDU.from_columns(coldefs, name="trigger patern") + + col1 = fits.Column(array=container.event_id, name="event_id", format="1I") + col2 = fits.Column(array=container.event_type, name="event_type", format="1I") + col3 = fits.Column( + array=container.ucts_timestamp, name="ucts_timestamp", format="1K" + ) + col4 = fits.Column( + array=container.ucts_busy_counter, name="ucts_busy_counter", format="1I" + ) + col5 = fits.Column( + array=container.ucts_event_counter, name="ucts_event_counter", format="1I" + ) + col6 = fits.Column( + array=container.multiplicity, name="multiplicity", format="1I" + ) coldefs = fits.ColDefs([col1, col2, col3, col4, col5, col6]) - event_properties = fits.BinTableHDU.from_columns(coldefs, name = 'event properties') - - col1 = fits.Column(array = container.trig_pattern_all, name = "trig_pattern_all", format = f'{4 * container.trig_pattern_all.shape[1]}L',dim = f'({ container.trig_pattern_all.shape[1]},4)') - col2 = fits.Column(array = container.trig_pattern, name = "trig_pattern", format = f'{container.trig_pattern_all.shape[1]}L') + event_properties = fits.BinTableHDU.from_columns( + coldefs, name="event properties" + ) + + col1 = fits.Column( + array=container.trig_pattern_all, + name="trig_pattern_all", + format=f"{4 * container.trig_pattern_all.shape[1]}L", + dim=f"({ container.trig_pattern_all.shape[1]},4)", + ) + col2 = fits.Column( + array=container.trig_pattern, + name="trig_pattern", + format=f"{container.trig_pattern_all.shape[1]}L", + ) coldefs = fits.ColDefs([col1, col2]) - trigger_patern = fits.BinTableHDU.from_columns(coldefs, name = 'trigger patern') - - hdul = fits.HDUList([primary_hdu, charge_hg_hdu, charge_lg_hdu,peak_hg_hdu,peak_lg_hdu,broken_pixels,event_properties,trigger_patern]) - try : - hdul.writeto(Path(path)/f"charge_run{container.run_number}{suffix}.fits",overwrite=kwargs.get('overwrite',False)) - log.info(f"charge saved in {Path(path)}/charge_run{container.run_number}{suffix}.fits") - except OSError as e : + trigger_patern = fits.BinTableHDU.from_columns(coldefs, name="trigger patern") + + hdul = fits.HDUList( + [ + primary_hdu, + charge_hg_hdu, + charge_lg_hdu, + peak_hg_hdu, + peak_lg_hdu, + broken_pixels, + event_properties, + trigger_patern, + ] + ) + try: + hdul.writeto( + Path(path) / f"charge_run{container.run_number}{suffix}.fits", + overwrite=kwargs.get("overwrite", False), + ) + log.info( + f"charge saved in {Path(path)}/charge_run{container.run_number}{suffix}.fits" + ) + except OSError as e: log.warning(e) - except Exception as e : - log.error(e,exc_info = True) + except Exception as e: + log.error(e, exc_info=True) raise e @staticmethod @@ -158,11 +194,11 @@ def load(path: str, run_number: int, **kwargs) -> ChargesContainer: with fits.open(filename) as hdul: container = ChargesContainer() - container.run_number = hdul[0].header['RUN'] - container.nevents = hdul[0].header['NEVENTS'] - container.npixels = hdul[0].header['NPIXELS'] - container.method = hdul[0].header['METHOD'] - container.camera = hdul[0].header['CAMERA'] + container.run_number = hdul[0].header["RUN"] + container.nevents = hdul[0].header["NEVENTS"] + container.npixels = hdul[0].header["NPIXELS"] + container.method = hdul[0].header["METHOD"] + container.camera = hdul[0].header["CAMERA"] container.pixels_id = hdul[0].data container.charges_hg = hdul[1].data container.charges_lg = hdul[2].data diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index d54140c4..d034246e 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -1,13 +1,15 @@ -import sys import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +import sys + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers -from ctapipe.containers import Container,Field import numpy as np +from ctapipe.containers import Container, Field + +__all__ = ["ArrayDataContainer"] -__all__= ["ArrayDataContainer"] class ArrayDataContainer(Container): """ @@ -43,51 +45,18 @@ class ArrayDataContainer(Container): type=int, description="number of effective pixels", ) - pixels_id = Field( - type=np.ndarray, - description="pixel ids" - ) - broken_pixels_hg = Field( - type=np.ndarray, - description="high gain broken pixels" - ) - broken_pixels_lg = Field( - type=np.ndarray, - description="low gain broken pixels" - ) + pixels_id = Field(type=np.ndarray, description="pixel ids") + broken_pixels_hg = Field(type=np.ndarray, description="high gain broken pixels") + broken_pixels_lg = Field(type=np.ndarray, description="low gain broken pixels") camera = Field( type=str, description="camera name", ) - ucts_timestamp = Field( - type=np.ndarray, - description="events ucts timestamp" - ) - ucts_busy_counter = Field( - type=np.ndarray, - description="ucts busy counter" - ) - ucts_event_counter = Field( - type=np.ndarray, - description="ucts event counter" - ) - event_type = Field( - type=np.ndarray, - description="trigger event type" - ) - event_id = Field( - type=np.ndarray, - description="event ids" - ) - trig_pattern_all = Field( - type=np.ndarray, - description="trigger pattern" - ) - trig_pattern = Field( - type=np.ndarray, - description="reduced trigger pattern" - ) - multiplicity = Field( - type=np.ndarray, - description="events multiplicity" - ) \ No newline at end of file + ucts_timestamp = Field(type=np.ndarray, description="events ucts timestamp") + ucts_busy_counter = Field(type=np.ndarray, description="ucts busy counter") + ucts_event_counter = Field(type=np.ndarray, description="ucts event counter") + event_type = Field(type=np.ndarray, description="trigger event type") + event_id = Field(type=np.ndarray, description="event ids") + trig_pattern_all = Field(type=np.ndarray, description="trigger pattern") + trig_pattern = Field(type=np.ndarray, description="reduced trigger pattern") + multiplicity = Field(type=np.ndarray, description="events multiplicity") diff --git a/src/nectarchain/data/container/tests/test_charge.py b/src/nectarchain/data/container/tests/test_charge.py index da559baa..270d5056 100644 --- a/src/nectarchain/data/container/tests/test_charge.py +++ b/src/nectarchain/data/container/tests/test_charge.py @@ -1,65 +1,70 @@ -from nectarchain.data.container import ChargesContainer,ChargesContainerIO -from nectarchain.makers import ChargesMaker import glob + import numpy as np -def create_fake_chargeContainer() : +from nectarchain.data.container import ChargesContainer, ChargesContainerIO +from nectarchain.makers import ChargesMaker + + +def create_fake_chargeContainer(): nevents = TestChargesContainer.nevents npixels = TestChargesContainer.npixels rng = np.random.default_rng() return ChargesContainer( - pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]), - nevents =nevents, - npixels =npixels, - charges_hg = rng.integers(low=0, high=1000, size= (nevents,npixels)), - charges_lg = rng.integers(low=0, high=1000, size= (nevents,npixels)), - peak_hg = rng.integers(low=0, high=60, size= (nevents,npixels)), - peak_lg = rng.integers(low=0, high=60, size= (nevents,npixels)), - run_number = TestChargesContainer.run_number, - camera = 'TEST', - broken_pixels_hg = rng.integers(low=0, high=1, size= (nevents,npixels)), - broken_pixels_lg = rng.integers(low=0, high=1, size= (nevents,npixels)), - ucts_timestamp =rng.integers(low=0, high=100, size= (nevents)), - ucts_busy_counter =rng.integers(low=0, high=100, size= (nevents)), - ucts_event_counter =rng.integers(low=0, high=100, size= (nevents)), - event_type =rng.integers(low=0, high=1, size= (nevents)), - event_id =rng.integers(low=0, high=1000, size= (nevents)), - trig_pattern_all = rng.integers(low=0, high=1, size= (nevents,npixels,4)), - trig_pattern = rng.integers(low=0, high=1, size= (nevents,npixels)), - multiplicity =rng.integers(low=0, high=1, size= (nevents)) + pixels_id=np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10]), + nevents=nevents, + npixels=npixels, + charges_hg=rng.integers(low=0, high=1000, size=(nevents, npixels)), + charges_lg=rng.integers(low=0, high=1000, size=(nevents, npixels)), + peak_hg=rng.integers(low=0, high=60, size=(nevents, npixels)), + peak_lg=rng.integers(low=0, high=60, size=(nevents, npixels)), + run_number=TestChargesContainer.run_number, + camera="TEST", + broken_pixels_hg=rng.integers(low=0, high=1, size=(nevents, npixels)), + broken_pixels_lg=rng.integers(low=0, high=1, size=(nevents, npixels)), + ucts_timestamp=rng.integers(low=0, high=100, size=(nevents)), + ucts_busy_counter=rng.integers(low=0, high=100, size=(nevents)), + ucts_event_counter=rng.integers(low=0, high=100, size=(nevents)), + event_type=rng.integers(low=0, high=1, size=(nevents)), + event_id=rng.integers(low=0, high=1000, size=(nevents)), + trig_pattern_all=rng.integers(low=0, high=1, size=(nevents, npixels, 4)), + trig_pattern=rng.integers(low=0, high=1, size=(nevents, npixels)), + multiplicity=rng.integers(low=0, high=1, size=(nevents)), ) + class TestChargesContainer: run_number = 1234 nevents = 140 npixels = 10 + # Tests that a ChargeContainer object can be created with valid input parameters. def test_create_charge_container(self): - pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) + pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10]) nevents = TestChargesContainer.nevents npixels = TestChargesContainer.npixels run_number = TestChargesContainer.run_number - charges_hg = np.random.randn(nevents,npixels) - charges_lg = np.random.randn(nevents,npixels) - peak_hg = np.random.randn(nevents,npixels) - peak_lg = np.random.randn(nevents,npixels) - method = 'FullWaveformSum' + charges_hg = np.random.randn(nevents, npixels) + charges_lg = np.random.randn(nevents, npixels) + peak_hg = np.random.randn(nevents, npixels) + peak_lg = np.random.randn(nevents, npixels) + method = "FullWaveformSum" charge_container = ChargesContainer( - charges_hg = charges_hg , - charges_lg = charges_lg, - peak_hg = peak_hg, - peak_lg = peak_lg, - run_number = run_number, - pixels_id = pixels_id, - nevents = nevents, - npixels = npixels, - method = method + charges_hg=charges_hg, + charges_lg=charges_lg, + peak_hg=peak_hg, + peak_lg=peak_lg, + run_number=run_number, + pixels_id=pixels_id, + nevents=nevents, + npixels=npixels, + method=method, ) - - assert np.allclose(charge_container.charges_hg,charges_hg) - assert np.allclose(charge_container.charges_lg,charges_lg) - assert np.allclose(charge_container.peak_hg,peak_hg) - assert np.allclose(charge_container.peak_lg,peak_lg) + + assert np.allclose(charge_container.charges_hg, charges_hg) + assert np.allclose(charge_container.charges_lg, charges_lg) + assert np.allclose(charge_container.peak_hg, peak_hg) + assert np.allclose(charge_container.peak_lg, peak_lg) assert charge_container.run_number == run_number assert charge_container.pixels_id.tolist() == pixels_id.tolist() assert charge_container.nevents == nevents @@ -67,7 +72,7 @@ def test_create_charge_container(self): assert charge_container.method == method # Tests that the from_waveforms method can be called with a valid waveformContainer and method parameter. - #def test_from_waveforms_valid_input(self): + # def test_from_waveforms_valid_input(self): # waveform_container = WaveformsContainer(...) # method = 'FullWaveformSum' # @@ -76,67 +81,85 @@ def test_create_charge_container(self): # assert isinstance(charge_container, ChargeContainer) # Tests that the ChargeContainer object can be written to a file and the file is created. - def test_write_charge_container(self, tmp_path = "/tmp"): + def test_write_charge_container(self, tmp_path="/tmp"): charge_container = create_fake_chargeContainer() tmp_path += f"/{np.random.randn(1)[0]}" - - ChargesContainerIO.write(tmp_path,charge_container) - - assert len(glob.glob(f"{tmp_path}/charge_run{TestChargesContainer.run_number}.fits")) == 1 + + ChargesContainerIO.write(tmp_path, charge_container) + + assert ( + len( + glob.glob( + f"{tmp_path}/charge_run{TestChargesContainer.run_number}.fits" + ) + ) + == 1 + ) # Tests that a ChargeContainer object can be loaded from a file and the object is correctly initialized. - def test_load_charge_container(self, tmp_path = "/tmp"): + def test_load_charge_container(self, tmp_path="/tmp"): charge_container = create_fake_chargeContainer() tmp_path += f"/{np.random.randn(1)[0]}" - - ChargesContainerIO.write(tmp_path,charge_container) - - loaded_charge_container = ChargesContainerIO.load(tmp_path,TestChargesContainer.run_number ) - + + ChargesContainerIO.write(tmp_path, charge_container) + + loaded_charge_container = ChargesContainerIO.load( + tmp_path, TestChargesContainer.run_number + ) + assert isinstance(loaded_charge_container, ChargesContainer) - assert np.allclose(loaded_charge_container.charges_hg,charge_container.charges_hg) - assert np.allclose(loaded_charge_container.charges_lg,charge_container.charges_lg) - assert np.allclose(loaded_charge_container.peak_hg,charge_container.peak_hg) - assert np.allclose(loaded_charge_container.peak_lg,charge_container.peak_lg) + assert np.allclose( + loaded_charge_container.charges_hg, charge_container.charges_hg + ) + assert np.allclose( + loaded_charge_container.charges_lg, charge_container.charges_lg + ) + assert np.allclose(loaded_charge_container.peak_hg, charge_container.peak_hg) + assert np.allclose(loaded_charge_container.peak_lg, charge_container.peak_lg) assert loaded_charge_container.run_number == charge_container.run_number - assert loaded_charge_container.pixels_id.tolist() == charge_container.pixels_id.tolist() + assert ( + loaded_charge_container.pixels_id.tolist() + == charge_container.pixels_id.tolist() + ) assert loaded_charge_container.nevents == charge_container.nevents assert loaded_charge_container.npixels == charge_container.npixels assert loaded_charge_container.method == charge_container.method # Tests that the ChargeContainer object can be sorted by event_id and the object is sorted accordingly. - def test_sort_charge_container(self): + def test_sort_charge_container(self): charge_container = create_fake_chargeContainer() - + sorted_charge_container = ChargesMaker.sort(charge_container) - - assert sorted_charge_container.event_id.tolist() == sorted(charge_container.event_id.tolist()) + + assert sorted_charge_container.event_id.tolist() == sorted( + charge_container.event_id.tolist() + ) # Tests that the run_number, pixels_id, npixels, nevents, method, multiplicity, and trig_pattern properties of the ChargeContainer object can be accessed and the values are correct. def test_access_properties(self): - pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) + pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10]) nevents = 40 npixels = 10 - charges_hg = np.random.randn(nevents,npixels) - charges_lg = np.random.randn(nevents,npixels) - peak_hg = np.random.randn(nevents,npixels) - peak_lg = np.random.randn(nevents,npixels) + charges_hg = np.random.randn(nevents, npixels) + charges_lg = np.random.randn(nevents, npixels) + peak_hg = np.random.randn(nevents, npixels) + peak_lg = np.random.randn(nevents, npixels) run_number = 1234 - method = 'FullWaveformSum' - charge_container = ChargesContainer( - charges_hg = charges_hg , - charges_lg = charges_lg, - peak_hg = peak_hg, - peak_lg = peak_lg, - run_number = run_number, - pixels_id = pixels_id, - nevents = nevents, - npixels = npixels, - method = method + method = "FullWaveformSum" + charge_container = ChargesContainer( + charges_hg=charges_hg, + charges_lg=charges_lg, + peak_hg=peak_hg, + peak_lg=peak_lg, + run_number=run_number, + pixels_id=pixels_id, + nevents=nevents, + npixels=npixels, + method=method, ) - + assert charge_container.run_number == run_number assert charge_container.pixels_id.tolist() == pixels_id.tolist() assert charge_container.npixels == npixels assert charge_container.nevents == nevents - assert charge_container.method == method \ No newline at end of file + assert charge_container.method == method diff --git a/src/nectarchain/data/container/tests/test_waveforms.py b/src/nectarchain/data/container/tests/test_waveforms.py index a74668c6..be38c45f 100644 --- a/src/nectarchain/data/container/tests/test_waveforms.py +++ b/src/nectarchain/data/container/tests/test_waveforms.py @@ -1,82 +1,97 @@ -from nectarchain.data.container import WaveformsContainer,WaveformsContainerIO -from nectarchain.makers import WaveformsMaker -from ctapipe.instrument import SubarrayDescription import glob + import numpy as np +from ctapipe.instrument import SubarrayDescription -def create_fake_waveformsContainer() : +from nectarchain.data.container import WaveformsContainer, WaveformsContainerIO +from nectarchain.makers import WaveformsMaker + + +def create_fake_waveformsContainer(): nevents = TestWaveformsContainer.nevents npixels = TestWaveformsContainer.npixels nsamples = TestWaveformsContainer.nsamples rng = np.random.default_rng() - faked_subarray = SubarrayDescription(name = 'TEST') + faked_subarray = SubarrayDescription(name="TEST") return WaveformsContainer( - pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]), - nevents =nevents, - npixels =npixels, - wfs_hg = rng.integers(low=0, high=1000, size= (nevents,npixels,nsamples)), - wfs_lg = rng.integers(low=0, high=1000, size= (nevents,npixels,nsamples)), - run_number = TestWaveformsContainer.run_number, - camera = 'TEST', - subarray = faked_subarray, - broken_pixels_hg = rng.integers(low=0, high=1, size= (nevents,npixels)), - broken_pixels_lg = rng.integers(low=0, high=1, size= (nevents,npixels)), - ucts_timestamp =rng.integers(low=0, high=100, size= (nevents)), - ucts_busy_counter =rng.integers(low=0, high=100, size= (nevents)), - ucts_event_counter =rng.integers(low=0, high=100, size= (nevents)), - event_type =rng.integers(low=0, high=1, size= (nevents)), - event_id =rng.integers(low=0, high=1000, size= (nevents)), - trig_pattern_all = rng.integers(low=0, high=1, size= (nevents,npixels,4)), - trig_pattern = rng.integers(low=0, high=1, size= (nevents,npixels)), - multiplicity =rng.integers(low=0, high=1, size= (nevents)) + pixels_id=np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10]), + nevents=nevents, + npixels=npixels, + wfs_hg=rng.integers(low=0, high=1000, size=(nevents, npixels, nsamples)), + wfs_lg=rng.integers(low=0, high=1000, size=(nevents, npixels, nsamples)), + run_number=TestWaveformsContainer.run_number, + camera="TEST", + subarray=faked_subarray, + broken_pixels_hg=rng.integers(low=0, high=1, size=(nevents, npixels)), + broken_pixels_lg=rng.integers(low=0, high=1, size=(nevents, npixels)), + ucts_timestamp=rng.integers(low=0, high=100, size=(nevents)), + ucts_busy_counter=rng.integers(low=0, high=100, size=(nevents)), + ucts_event_counter=rng.integers(low=0, high=100, size=(nevents)), + event_type=rng.integers(low=0, high=1, size=(nevents)), + event_id=rng.integers(low=0, high=1000, size=(nevents)), + trig_pattern_all=rng.integers(low=0, high=1, size=(nevents, npixels, 4)), + trig_pattern=rng.integers(low=0, high=1, size=(nevents, npixels)), + multiplicity=rng.integers(low=0, high=1, size=(nevents)), ) + class TestWaveformsContainer: run_number = 1234 nevents = 140 npixels = 10 nsamples = 5 + # Tests that a ChargeContainer object can be created with valid input parameters. def test_create_waveform_container(self): waveform_container = create_fake_waveformsContainer() - assert isinstance(waveform_container,WaveformsContainer) - + assert isinstance(waveform_container, WaveformsContainer) # Tests that the ChargeContainer object can be written to a file and the file is created. - def test_write_waveform_container(self, tmp_path = "/tmp"): + def test_write_waveform_container(self, tmp_path="/tmp"): waveform_container = create_fake_waveformsContainer() tmp_path += f"/{np.random.randn(1)[0]}" - - WaveformsContainerIO.write(tmp_path,waveform_container) - - assert len(glob.glob(f"{tmp_path}/*_run{TestWaveformsContainer.run_number}.fits")) == 1 + + WaveformsContainerIO.write(tmp_path, waveform_container) + + assert ( + len(glob.glob(f"{tmp_path}/*_run{TestWaveformsContainer.run_number}.fits")) + == 1 + ) # Tests that a ChargeContainer object can be loaded from a file and the object is correctly initialized. - def test_load_waveform_container(self, tmp_path = "/tmp"): + def test_load_waveform_container(self, tmp_path="/tmp"): waveform_container = create_fake_waveformsContainer() tmp_path += f"/{np.random.randn(1)[0]}" - - WaveformsContainerIO.write(tmp_path,waveform_container) - - loaded_waveform_container = WaveformsContainerIO.load(tmp_path,TestWaveformsContainer.run_number) - + + WaveformsContainerIO.write(tmp_path, waveform_container) + + loaded_waveform_container = WaveformsContainerIO.load( + tmp_path, TestWaveformsContainer.run_number + ) + assert isinstance(loaded_waveform_container, WaveformsContainer) - assert np.allclose(loaded_waveform_container.wfs_hg,waveform_container.wfs_hg) - assert np.allclose(loaded_waveform_container.wfs_lg,waveform_container.wfs_lg) + assert np.allclose(loaded_waveform_container.wfs_hg, waveform_container.wfs_hg) + assert np.allclose(loaded_waveform_container.wfs_lg, waveform_container.wfs_lg) assert loaded_waveform_container.run_number == waveform_container.run_number - assert loaded_waveform_container.pixels_id.tolist() == waveform_container.pixels_id.tolist() + assert ( + loaded_waveform_container.pixels_id.tolist() + == waveform_container.pixels_id.tolist() + ) assert loaded_waveform_container.nevents == waveform_container.nevents assert loaded_waveform_container.npixels == waveform_container.npixels assert loaded_waveform_container.nsamples == waveform_container.nsamples # Tests that the ChargeContainer object can be sorted by event_id and the object is sorted accordingly. - def test_sort_waveform_container(self): + def test_sort_waveform_container(self): waveform_container = create_fake_waveformsContainer() - + sorted_waveform_container = WaveformsMaker.sort(waveform_container) - - assert sorted_waveform_container.event_id.tolist() == sorted(waveform_container.event_id.tolist()) -if __name__ == "__main__" : - TestWaveformsContainer().test_create_waveform_container() \ No newline at end of file + assert sorted_waveform_container.event_id.tolist() == sorted( + waveform_container.event_id.tolist() + ) + + +if __name__ == "__main__": + TestWaveformsContainer().test_create_waveform_container() diff --git a/src/nectarchain/data/container/waveformsContainer.py b/src/nectarchain/data/container/waveformsContainer.py index 18ae8465..13037ea3 100644 --- a/src/nectarchain/data/container/waveformsContainer.py +++ b/src/nectarchain/data/container/waveformsContainer.py @@ -1,27 +1,26 @@ -import sys import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +import sys + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers import os - -from ctapipe.instrument.subarray import SubarrayDescription -import numpy as np - -from pathlib import Path - +from abc import ABC from enum import Enum +from pathlib import Path -from tqdm import tqdm -from ctapipe.containers import Field -from astropy.io import fits -from astropy.table import QTable,Column,Table import astropy.units as u -from abc import ABC +import numpy as np +from astropy.io import fits +from astropy.table import Column, QTable, Table +from ctapipe.containers import Field +from ctapipe.instrument.subarray import SubarrayDescription +from tqdm import tqdm from .core import ArrayDataContainer + class WaveformsContainer(ArrayDataContainer): """ A container that holds information about waveforms from a specific run. @@ -37,22 +36,12 @@ class WaveformsContainer(ArrayDataContainer): type=int, description="number of samples in the waveforms", ) - subarray = Field( - type=SubarrayDescription, - description="The subarray description" - ) - wfs_hg = Field( - type=np.ndarray, - description="high gain waveforms" - ) - wfs_lg = Field( - type=np.ndarray, - description="low gain waveforms" - ) + subarray = Field(type=SubarrayDescription, description="The subarray description") + wfs_hg = Field(type=np.ndarray, description="high gain waveforms") + wfs_lg = Field(type=np.ndarray, description="low gain waveforms") - -class WaveformsContainerIO(ABC) : +class WaveformsContainerIO(ABC): """ The `WaveformsContainerIO` class provides methods for writing and loading `WaveformsContainer` instances to/from FITS files. It also includes a method for writing the subarray configuration to an HDF5 file. @@ -79,8 +68,8 @@ class WaveformsContainerIO(ABC) : """ @staticmethod - def write(path : str, containers : WaveformsContainer, **kwargs) -> None: - '''Write the WaveformsContainer data to an output FITS file. + def write(path: str, containers: WaveformsContainer, **kwargs) -> None: + """Write the WaveformsContainer data to an output FITS file. This method creates two files: one FITS file representing the waveform data and one HDF5 file representing the subarray configuration. @@ -100,68 +89,112 @@ def write(path : str, containers : WaveformsContainer, **kwargs) -> None: Example: waveformsContainer = WaveformsContainer() WaveformsContainerIO.write(path, waveformsContainer, suffix="v1", overwrite=True) - ''' - suffix = kwargs.get("suffix","") - if suffix != "" : suffix = f"_{suffix}" + """ + suffix = kwargs.get("suffix", "") + if suffix != "": + suffix = f"_{suffix}" log.info(f"saving in {path}") - os.makedirs(path,exist_ok = True) + os.makedirs(path, exist_ok=True) hdr = fits.Header() - hdr['RUN'] = containers.run_number - hdr['NEVENTS'] = containers.nevents - hdr['NPIXELS'] = containers.npixels - hdr['NSAMPLES'] = containers.nsamples - hdr['SUBARRAY'] = containers.subarray.name - hdr['CAMERA'] = containers.camera - - - containers.subarray.to_hdf(f"{Path(path)}/subarray_run{containers.run_number}{suffix}.hdf5",overwrite=kwargs.get('overwrite',False)) - - - - hdr['COMMENT'] = f"The waveforms containeur for run {containers.run_number} : primary is the pixels id, 2nd HDU : high gain waveforms, 3rd HDU : low gain waveforms, 4th HDU : event properties and 5th HDU trigger paterns." - - primary_hdu = fits.PrimaryHDU(containers.pixels_id,header=hdr) - - wfs_hg_hdu = fits.ImageHDU(containers.wfs_hg,name = "HG Waveforms") - wfs_lg_hdu = fits.ImageHDU(containers.wfs_lg,name = "LG Waveforms") - - col1 = fits.Column(array = containers.broken_pixels_hg, name = "HG broken pixels", format = f'{containers.broken_pixels_hg.shape[1]}L') - col2 = fits.Column(array = containers.broken_pixels_lg, name = "LG broken pixels", format = f'{containers.broken_pixels_lg.shape[1]}L') + hdr["RUN"] = containers.run_number + hdr["NEVENTS"] = containers.nevents + hdr["NPIXELS"] = containers.npixels + hdr["NSAMPLES"] = containers.nsamples + hdr["SUBARRAY"] = containers.subarray.name + hdr["CAMERA"] = containers.camera + + containers.subarray.to_hdf( + f"{Path(path)}/subarray_run{containers.run_number}{suffix}.hdf5", + overwrite=kwargs.get("overwrite", False), + ) + + hdr[ + "COMMENT" + ] = f"The waveforms containeur for run {containers.run_number} : primary is the pixels id, 2nd HDU : high gain waveforms, 3rd HDU : low gain waveforms, 4th HDU : event properties and 5th HDU trigger paterns." + + primary_hdu = fits.PrimaryHDU(containers.pixels_id, header=hdr) + + wfs_hg_hdu = fits.ImageHDU(containers.wfs_hg, name="HG Waveforms") + wfs_lg_hdu = fits.ImageHDU(containers.wfs_lg, name="LG Waveforms") + + col1 = fits.Column( + array=containers.broken_pixels_hg, + name="HG broken pixels", + format=f"{containers.broken_pixels_hg.shape[1]}L", + ) + col2 = fits.Column( + array=containers.broken_pixels_lg, + name="LG broken pixels", + format=f"{containers.broken_pixels_lg.shape[1]}L", + ) coldefs = fits.ColDefs([col1, col2]) - broken_pixels = fits.BinTableHDU.from_columns(coldefs,name = 'trigger patern') - - col1 = fits.Column(array = containers.event_id, name = "event_id", format = '1J') - col2 = fits.Column(array = containers.event_type, name = "event_type", format = '1I') - col3 = fits.Column(array = containers.ucts_timestamp, name = "ucts_timestamp", format = '1K') - col4 = fits.Column(array = containers.ucts_busy_counter, name = "ucts_busy_counter", format = '1J') - col5 = fits.Column(array = containers.ucts_event_counter, name = "ucts_event_counter", format = '1J') - col6 = fits.Column(array = containers.multiplicity, name = "multiplicity", format = '1I') + broken_pixels = fits.BinTableHDU.from_columns(coldefs, name="trigger patern") + + col1 = fits.Column(array=containers.event_id, name="event_id", format="1J") + col2 = fits.Column(array=containers.event_type, name="event_type", format="1I") + col3 = fits.Column( + array=containers.ucts_timestamp, name="ucts_timestamp", format="1K" + ) + col4 = fits.Column( + array=containers.ucts_busy_counter, name="ucts_busy_counter", format="1J" + ) + col5 = fits.Column( + array=containers.ucts_event_counter, name="ucts_event_counter", format="1J" + ) + col6 = fits.Column( + array=containers.multiplicity, name="multiplicity", format="1I" + ) coldefs = fits.ColDefs([col1, col2, col3, col4, col5, col6]) - event_properties = fits.BinTableHDU.from_columns(coldefs,name = 'event properties') - - col1 = fits.Column(array = containers.trig_pattern_all, name = "trig_pattern_all", format = f'{4 * containers.trig_pattern_all.shape[1]}L',dim = f'({containers.trig_pattern_all.shape[1]},4)') - col2 = fits.Column(array = containers.trig_pattern, name = "trig_pattern", format = f'{containers.trig_pattern_all.shape[1]}L') + event_properties = fits.BinTableHDU.from_columns( + coldefs, name="event properties" + ) + + col1 = fits.Column( + array=containers.trig_pattern_all, + name="trig_pattern_all", + format=f"{4 * containers.trig_pattern_all.shape[1]}L", + dim=f"({containers.trig_pattern_all.shape[1]},4)", + ) + col2 = fits.Column( + array=containers.trig_pattern, + name="trig_pattern", + format=f"{containers.trig_pattern_all.shape[1]}L", + ) coldefs = fits.ColDefs([col1, col2]) - trigger_patern = fits.BinTableHDU.from_columns(coldefs,name = 'trigger patern') - - hdul = fits.HDUList([primary_hdu, wfs_hg_hdu, wfs_lg_hdu,broken_pixels,event_properties,trigger_patern]) - try : - hdul.writeto(Path(path)/f"waveforms_run{containers.run_number}{suffix}.fits",overwrite=kwargs.get('overwrite',False)) - log.info(f"run saved in {Path(path)}/waveforms_run{containers.run_number}{suffix}.fits") - except OSError as e : + trigger_patern = fits.BinTableHDU.from_columns(coldefs, name="trigger patern") + + hdul = fits.HDUList( + [ + primary_hdu, + wfs_hg_hdu, + wfs_lg_hdu, + broken_pixels, + event_properties, + trigger_patern, + ] + ) + try: + hdul.writeto( + Path(path) / f"waveforms_run{containers.run_number}{suffix}.fits", + overwrite=kwargs.get("overwrite", False), + ) + log.info( + f"run saved in {Path(path)}/waveforms_run{containers.run_number}{suffix}.fits" + ) + except OSError as e: log.warning(e) - except Exception as e : - log.error(e,exc_info = True) + except Exception as e: + log.error(e, exc_info=True) raise e - + @staticmethod - def load(path : str, run_number : int, **kwargs) -> WaveformsContainer: - '''Load a WaveformsContainer from a FITS file previously written with WaveformsContainerIO.write() method. + def load(path: str, run_number: int, **kwargs) -> WaveformsContainer: + """Load a WaveformsContainer from a FITS file previously written with WaveformsContainerIO.write() method. - Note: Two files are loaded—the FITS file representing the waveform data and an HDF5 file representing the subarray configuration. + Note: Two files are loaded—the FITS file representing the waveform data and an HDF5 file representing the subarray configuration. The HDF5 file should be located next to the FITS file. Args: @@ -172,7 +205,7 @@ def load(path : str, run_number : int, **kwargs) -> WaveformsContainer: WaveformsContainer: A WaveformsContainer instance loaded from the specified file. Example: waveformsContainer = WaveformsContainerIO.load(path, run_number) - ''' + """ if kwargs.get("explicit_filename", False): filename = kwargs.get("explicit_filename") log.info(f"Loading {filename}") @@ -181,29 +214,31 @@ def load(path : str, run_number : int, **kwargs) -> WaveformsContainer: filename = Path(path) / f"waveforms_run{run_number}.fits" log.info(f"loading from {path}") - with fits.open(filename) as hdul : + with fits.open(filename) as hdul: containers = WaveformsContainer() - containers.run_number = hdul[0].header['RUN'] - containers.nevents = hdul[0].header['NEVENTS'] - containers.npixels = hdul[0].header['NPIXELS'] - containers.nsamples = hdul[0].header['NSAMPLES'] - containers.camera = hdul[0].header['CAMERA'] - - - containers.subarray = SubarrayDescription.from_hdf(Path(filename._str.replace('waveforms_','subarray_').replace('fits','hdf5'))) + containers.run_number = hdul[0].header["RUN"] + containers.nevents = hdul[0].header["NEVENTS"] + containers.npixels = hdul[0].header["NPIXELS"] + containers.nsamples = hdul[0].header["NSAMPLES"] + containers.camera = hdul[0].header["CAMERA"] + containers.subarray = SubarrayDescription.from_hdf( + Path( + filename._str.replace("waveforms_", "subarray_").replace( + "fits", "hdf5" + ) + ) + ) containers.pixels_id = hdul[0].data containers.wfs_hg = hdul[1].data containers.wfs_lg = hdul[2].data - + broken_pixels = hdul[3].data containers.broken_pixels_hg = broken_pixels["HG broken pixels"] containers.broken_pixels_lg = broken_pixels["LG broken pixels"] - - table_prop = hdul[4].data containers.event_id = table_prop["event_id"] containers.event_type = table_prop["event_type"] @@ -216,4 +251,4 @@ def load(path : str, run_number : int, **kwargs) -> WaveformsContainer: containers.trig_pattern_all = table_trigger["trig_pattern_all"] containers.trig_pattern = table_trigger["trig_pattern"] - return containers \ No newline at end of file + return containers diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py index ed548466..6ac3f6f0 100644 --- a/src/nectarchain/data/management.py +++ b/src/nectarchain/data/management.py @@ -1,23 +1,25 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers -import os import glob +import os +from pathlib import Path +from typing import List, Tuple + +import browser_cookie3 import mechanize import requests -import browser_cookie3 - from DIRAC.Interfaces.API.Dirac import Dirac -from pathlib import Path -from typing import List,Tuple -__all__ = ['DataManagement'] +__all__ = ["DataManagement"] -class DataManagement() : + +class DataManagement: @staticmethod - def findrun(run_number : int,search_on_GRID = True) -> Tuple[Path,List[Path]]: + def findrun(run_number: int, search_on_GRID=True) -> Tuple[Path, List[Path]]: """method to find in NECTARCAMDATA the list of *.fits.fz files associated to run_number Args: @@ -26,37 +28,41 @@ def findrun(run_number : int,search_on_GRID = True) -> Tuple[Path,List[Path]]: Returns: (PosixPath,list): the path list of *fits.fz files """ - basepath=os.environ['NECTARCAMDATA'] - list = glob.glob(basepath+'**/*'+str(run_number)+'*.fits.fz',recursive=True) + basepath = os.environ["NECTARCAMDATA"] + list = glob.glob( + basepath + "**/*" + str(run_number) + "*.fits.fz", recursive=True + ) list_path = [Path(chemin) for chemin in list] - if len(list_path) == 0 : + if len(list_path) == 0: e = FileNotFoundError(f"run {run_number} is not present in {basepath}") - if search_on_GRID : - log.warning(e,exc_info=True) - log.info('will search files on GRID and fetch them') + if search_on_GRID: + log.warning(e, exc_info=True) + log.info("will search files on GRID and fetch them") lfns = DataManagement.get_GRID_location(run_number) DataManagement.getRunFromDIRAC(lfns) - list = glob.glob(basepath+'**/*'+str(run_number)+'*.fits.fz',recursive=True) + list = glob.glob( + basepath + "**/*" + str(run_number) + "*.fits.fz", recursive=True + ) list_path = [Path(chemin) for chemin in list] - else : - log.error(e,exc_info=True) + else: + log.error(e, exc_info=True) raise e - - name = list_path[0].name.split(".") name[2] = "*" - name = Path(str(list_path[0].parent))/(f"{name[0]}.{name[1]}.{name[2]}.{name[3]}.{name[4]}") + name = Path(str(list_path[0].parent)) / ( + f"{name[0]}.{name[1]}.{name[2]}.{name[3]}.{name[4]}" + ) log.info(f"Found {len(list_path)} files matching {name}") - #to sort list path - _sorted = sorted([[file,int(file.suffixes[1][1:])] for file in list_path]) + # to sort list path + _sorted = sorted([[file, int(file.suffixes[1][1:])] for file in list_path]) list_path = [_sorted[i][0] for i in range(len(_sorted))] - return name,list_path + return name, list_path @staticmethod - def getRunFromDIRAC(lfns : list): + def getRunFromDIRAC(lfns: list): """method do get run files from GRID-EGI from input lfns Args: @@ -64,13 +70,18 @@ def getRunFromDIRAC(lfns : list): """ dirac = Dirac() - for lfn in lfns : - if not(os.path.exists(f'{os.environ["NECTARCAMDATA"]}/{os.path.basename(lfn)}')): - dirac.getFile(lfn=lfn,destDir=os.environ["NECTARCAMDATA"],printOutput=True) - + for lfn in lfns: + if not ( + os.path.exists(f'{os.environ["NECTARCAMDATA"]}/{os.path.basename(lfn)}') + ): + dirac.getFile( + lfn=lfn, destDir=os.environ["NECTARCAMDATA"], printOutput=True + ) @staticmethod - def get_GRID_location(run_number : int,output_lfns = True, username = None,password = None) : + def get_GRID_location( + run_number: int, output_lfns=True, username=None, password=None + ): """method to get run location on GRID from Elog (work in progress!) Args: @@ -87,60 +98,66 @@ def get_GRID_location(run_number : int,output_lfns = True, username = None,passw url_run = f"http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?mode=full&reverse=0&reverse=1&npp=20&subtext=%23{run_number}" - if not(username is None or password is None) : - log.debug('log to Elog with username and password') - #log to Elog + if not (username is None or password is None): + log.debug("log to Elog with username and password") + # log to Elog br = mechanize.Browser() br.open(url) - form = br.select_form('form1') - for i in range(4) : + form = br.select_form("form1") + for i in range(4): log.debug(br.form.find_control(nr=i).name) - br.form['uname'] = username - br.form['upassword'] = password + br.form["uname"] = username + br.form["upassword"] = password br.method = "POST" req = br.submit() - #html_page = req.get_data() - cookies = br._ua_handlers['_cookies'].cookiejar - #get data - req = requests.get(f'http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?jcmd=&mode=Raw&attach=1&printable=1&reverse=0&reverse=1&npp=20&ma=&da=&ya=&ha=&na=&ca=&last=&mb=&db=&yb=&hb=&nb=&cb=&Author=&Setup=&Category=&Keyword=&Subject=%23{run_number}&ModuleCount=&subtext=',cookies = cookies) - - else : - #try to acces data by getting cookies from firefox and Chrome - log.debug('try to get data with cookies from Firefox abnd Chrome') + # html_page = req.get_data() + cookies = br._ua_handlers["_cookies"].cookiejar + # get data + req = requests.get( + f"http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?jcmd=&mode=Raw&attach=1&printable=1&reverse=0&reverse=1&npp=20&ma=&da=&ya=&ha=&na=&ca=&last=&mb=&db=&yb=&hb=&nb=&cb=&Author=&Setup=&Category=&Keyword=&Subject=%23{run_number}&ModuleCount=&subtext=", + cookies=cookies, + ) + + else: + # try to acces data by getting cookies from firefox and Chrome + log.debug("try to get data with cookies from Firefox abnd Chrome") cookies = browser_cookie3.load() - req = requests.get(f'http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?jcmd=&mode=Raw&attach=1&printable=1&reverse=0&reverse=1&npp=20&ma=&da=&ya=&ha=&na=&ca=&last=&mb=&db=&yb=&hb=&nb=&cb=&Author=&Setup=&Category=&Keyword=&Subject=%23{run_number}&ModuleCount=&subtext=',cookies = cookies) - - #if "ELOG Login" in req.text : + req = requests.get( + f"http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?jcmd=&mode=Raw&attach=1&printable=1&reverse=0&reverse=1&npp=20&ma=&da=&ya=&ha=&na=&ca=&last=&mb=&db=&yb=&hb=&nb=&cb=&Author=&Setup=&Category=&Keyword=&Subject=%23{run_number}&ModuleCount=&subtext=", + cookies=cookies, + ) + + # if "ELOG Login" in req.text : - lines = req.text.split('\r\n') + lines = req.text.split("\r\n") url_data = None - for i,line in enumerate(lines) : - if '

' in line : - url_data = line.split("

")[0].split('FC:')[1] + for i, line in enumerate(lines): + if "

" in line: + url_data = line.split("

")[0].split("FC:")[1] log.debug(f"url_data found {url_data}") break - - if i == len(lines)-1 : - e=Exception("lfns not found on GRID") - log.error(e,exc_info=True) + + if i == len(lines) - 1: + e = Exception("lfns not found on GRID") + log.error(e, exc_info=True) log.debug(lines) raise e - if output_lfns : + if output_lfns: lfns = [] - try : - #Dirac + try: + # Dirac dirac = Dirac() loc = f"/vo.cta.in2p3.fr/nectarcam/{url_data.split('/')[-2]}/{url_data.split('/')[-1]}" log.debug(f"searching in Dirac filecatalog at {loc}") res = dirac.listCatalogDirectory(loc, printOutput=True) - for key in res['Value']['Successful'][loc]['Files'].keys(): - if str(run_number) in key and "fits.fz" in key : + for key in res["Value"]["Successful"][loc]["Files"].keys(): + if str(run_number) in key and "fits.fz" in key: lfns.append(key) - except Exception as e : - log.error(e,exc_info = True) + except Exception as e: + log.error(e, exc_info=True) return lfns - else : + else: return url_data diff --git a/src/nectarchain/display/__init__.py b/src/nectarchain/display/__init__.py index 0b14e0cf..4941e7d3 100644 --- a/src/nectarchain/display/__init__.py +++ b/src/nectarchain/display/__init__.py @@ -1 +1 @@ -from .display import * \ No newline at end of file +from .display import * diff --git a/src/nectarchain/display/display.py b/src/nectarchain/display/display.py index 673efe1e..5bb2e414 100644 --- a/src/nectarchain/display/display.py +++ b/src/nectarchain/display/display.py @@ -1,42 +1,37 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers -from argparse import ArgumentError -import numpy as np -from matplotlib import pyplot as plt import copy -import os import glob -from pathlib import Path - +import os +import sys +from abc import ABC +from argparse import ArgumentError from enum import Enum +from pathlib import Path -from tqdm import tqdm - -from astropy.io import fits -from astropy.table import QTable,Column,Table import astropy.units as u - +import numpy as np +from astropy.io import fits +from astropy.table import Column, QTable, Table +from ctapipe.containers import EventType +from ctapipe.coordinates import CameraFrame, EngineeringCameraFrame +from ctapipe.instrument import CameraGeometry, SubarrayDescription, TelescopeDescription from ctapipe.visualization import CameraDisplay -from ctapipe.coordinates import CameraFrame,EngineeringCameraFrame -from ctapipe.instrument import CameraGeometry,SubarrayDescription,TelescopeDescription - from ctapipe_io_nectarcam import NectarCAMEventSource +from matplotlib import pyplot as plt +from tqdm import tqdm from ..data import DataManagement +from ..data.container import ArrayDataContainer, ChargesContainer, WaveformsContainer -import sys - -from ctapipe.containers import EventType -from ..data.container import WaveformsContainer,ChargesContainer,ArrayDataContainer - -from abc import ABC -class ContainerDisplay(ABC) : +class ContainerDisplay(ABC): @staticmethod - def display(container : ArrayDataContainer,evt,geometry, cmap = 'gnuplot2') : + def display(container: ArrayDataContainer, evt, geometry, cmap="gnuplot2"): """plot camera display for HIGH GAIN channel Args: @@ -46,18 +41,18 @@ def display(container : ArrayDataContainer,evt,geometry, cmap = 'gnuplot2') : Returns: CameraDisplay: thoe cameraDisplay plot """ - if isinstance(container,ChargesContainer) : + if isinstance(container, ChargesContainer): image = container.charges_hg - elif isinstance(container,WaveformsContainer) : + elif isinstance(container, WaveformsContainer): image = container.wfs_hg.sum(axis=2) - else : + else: log.warning("container can't be displayed") disp = CameraDisplay(geometry=geometry, image=image[evt], cmap=cmap) disp.add_colorbar() return disp @staticmethod - def plot_waveform(waveformsContainer : WaveformsContainer,evt,**kwargs) : + def plot_waveform(waveformsContainer: WaveformsContainer, evt, **kwargs): """plot the waveform of the evt in the HIGH GAIN channel Args: @@ -66,10 +61,10 @@ def plot_waveform(waveformsContainer : WaveformsContainer,evt,**kwargs) : Returns: tuple: the figure and axes """ - if 'figure' in kwargs.keys() and 'ax' in kwargs.keys() : - fig = kwargs.get('figure') - ax = kwargs.get('ax') - else : - fig,ax = plt.subplots(1,1) + if "figure" in kwargs.keys() and "ax" in kwargs.keys(): + fig = kwargs.get("figure") + ax = kwargs.get("ax") + else: + fig, ax = plt.subplots(1, 1) ax.plot(waveformsContainer.wfs_hg[evt].T) - return fig,ax + return fig, ax diff --git a/src/nectarchain/makers/__init__.py b/src/nectarchain/makers/__init__.py index 1cac4aa2..fa4ef370 100644 --- a/src/nectarchain/makers/__init__.py +++ b/src/nectarchain/makers/__init__.py @@ -1,3 +1,3 @@ +from .chargesMakers import * from .core import * from .waveformsMakers import * -from .chargesMakers import * diff --git a/src/nectarchain/makers/calibration/__init__.py b/src/nectarchain/makers/calibration/__init__.py index 2baa1183..0ded83f5 100644 --- a/src/nectarchain/makers/calibration/__init__.py +++ b/src/nectarchain/makers/calibration/__init__.py @@ -1,3 +1,3 @@ -from .pedestalMakers import * +from .flatfieldMakers import * from .gain import * -from .flatfieldMakers import * \ No newline at end of file +from .pedestalMakers import * diff --git a/src/nectarchain/makers/calibration/core.py b/src/nectarchain/makers/calibration/core.py index 7026d963..58f0c991 100644 --- a/src/nectarchain/makers/calibration/core.py +++ b/src/nectarchain/makers/calibration/core.py @@ -1,21 +1,23 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers import os +from collections.abc import Iterable +from copy import copy +from datetime import date from pathlib import Path -import numpy as np -from astropy.table import QTable,Column import astropy.units as u -from copy import copy -from datetime import date -from collections.abc import Iterable +import numpy as np +from astropy.table import Column, QTable from ..core import BaseMaker - __all__ = [""] + + class CalibrationMaker(BaseMaker): """ Mother class for all calibration makers that can be defined to compute calibration coefficients from data. @@ -51,13 +53,21 @@ def __init__(self, pixels_id, *args, **kwargs) -> None: pixels_id (iterable, np.ndarray): The list of pixels id. """ super().__init__() - if not(isinstance(pixels_id, Iterable)): + if not (isinstance(pixels_id, Iterable)): raise TypeError("pixels_id must be iterable") self.__pixels_id = np.array(pixels_id) self.__results = QTable() - self.__results.add_column(Column(self.__pixels_id, __class__.PIXELS_ID_COLUMN, unit=u.dimensionless_unscaled)) + self.__results.add_column( + Column( + self.__pixels_id, + __class__.PIXELS_ID_COLUMN, + unit=u.dimensionless_unscaled, + ) + ) self.__results.meta[__class__.NP_PIXELS] = self.npixels - self.__results.meta['comments'] = f'Produced with NectarChain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' + self.__results.meta[ + "comments" + ] = f'Produced with NectarChain, Credit : CTA NectarCam {date.today().strftime("%B %d, %Y")}' def save(self, path, **kwargs): """ @@ -72,8 +82,12 @@ def save(self, path, **kwargs): """ path = Path(path) path.mkdir(parents=True, exist_ok=True) - log.info(f'data saved in {path}') - self._results.write(f"{path}/results_{self._reduced_name}.ecsv", format='ascii.ecsv', overwrite=kwargs.get("overwrite", False)) + log.info(f"data saved in {path}") + self._results.write( + f"{path}/results_{self._reduced_name}.ecsv", + format="ascii.ecsv", + overwrite=kwargs.get("overwrite", False), + ) @property def _pixels_id(self): @@ -133,4 +147,4 @@ def results(self): Returns: QTable: A copy of the result table. """ - return copy(self.__results) \ No newline at end of file + return copy(self.__results) diff --git a/src/nectarchain/makers/calibration/flatfieldMakers.py b/src/nectarchain/makers/calibration/flatfieldMakers.py index 4df1e325..f7c1c5c8 100644 --- a/src/nectarchain/makers/calibration/flatfieldMakers.py +++ b/src/nectarchain/makers/calibration/flatfieldMakers.py @@ -1,15 +1,19 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers from .core import CalibrationMaker - __all__ = ["FlatfieldMaker"] -class FlatfieldMaker(CalibrationMaker) : - def __init__(self,*args,**kwargs) -> None: - super().__init__(*args,**kwargs) - def make(self) : - raise NotImplementedError("The computation of the flatfield calibration is not yet implemented, feel free to contribute !:)") \ No newline at end of file + +class FlatfieldMaker(CalibrationMaker): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def make(self): + raise NotImplementedError( + "The computation of the flatfield calibration is not yet implemented, feel free to contribute !:)" + ) diff --git a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py index 4529838b..31834773 100644 --- a/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/FlatFieldSPEMakers.py @@ -1,55 +1,46 @@ -import sys import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') -log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +import sys -import numpy as np -import astropy.units as u -from astropy.table import Column,QTable +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers import copy import os -import yaml import time - +from inspect import signature +from multiprocessing import Pool from typing import Tuple - +import astropy.units as u import matplotlib.pyplot as plt -from matplotlib.patches import Rectangle -from matplotlib.colors import to_rgba - import numpy as np - +import yaml +from astropy.table import Column, QTable from iminuit import Minuit - -from scipy.signal import find_peaks -from scipy.signal import savgol_filter +from matplotlib.colors import to_rgba +from matplotlib.patches import Rectangle from scipy.optimize import curve_fit +from scipy.signal import find_peaks, savgol_filter from scipy.special import gammainc -from multiprocessing import Pool - -from inspect import signature - -from .gainMakers import GainMaker - from ....data.container import ChargesContainer - from ...chargesMakers import ChargesMaker - +from .gainMakers import GainMaker from .parameters import Parameter, Parameters +from .utils import ( + MPE2, + MeanValueError, + PedestalValueError, + Statistics, + UtilsMinuit, + weight_gaussian, +) -from .utils import UtilsMinuit,weight_gaussian,Statistics,MPE2 - -from .utils import MeanValueError,PedestalValueError - - -__all__ = ["FlatFieldSingleHHVSPEMaker","FlatFieldSingleHHVStdSPEMaker"] +__all__ = ["FlatFieldSingleHHVSPEMaker", "FlatFieldSingleHHVStdSPEMaker"] -class FlatFieldSPEMaker(GainMaker) : +class FlatFieldSPEMaker(GainMaker): """ The `FlatFieldSPEMaker` class is used for flat field single photoelectron (SPE) calibration calculations on data. It inherits from the `GainMaker` class and adds functionality specific to flat field SPE calibration. @@ -82,17 +73,18 @@ class FlatFieldSPEMaker(GainMaker) : Attributes: - `_Windows_lenght`: A class attribute that represents the length of the windows used for smoothing the data. - `_Order`: A class attribute that represents the order of the polynomial used for smoothing the data. - + Members: - `npixels`: A property that returns the number of pixels. - `parameters`: A property that returns a deep copy of the internal parameters of the class. - `_parameters`: A property that returns the internal parameters of the class. """ + _Windows_lenght = 40 _Order = 2 -#constructors - def __init__(self,*args,**kwargs) -> None: + # constructors + def __init__(self, *args, **kwargs) -> None: """ Initializes the FlatFieldSPEMaker class. @@ -103,7 +95,7 @@ def __init__(self,*args,**kwargs) -> None: Returns: None """ - super().__init__(*args,**kwargs) + super().__init__(*args, **kwargs) self.__parameters = Parameters() @property @@ -136,7 +128,7 @@ def _parameters(self): """ return self.__parameters -#methods + # methods def read_param_from_yaml(self, parameters_file, only_update=False) -> None: """ Reads parameters from a YAML file and updates the internal parameters of the FlatFieldSPEMaker class. @@ -148,30 +140,36 @@ def read_param_from_yaml(self, parameters_file, only_update=False) -> None: Returns: None """ - with open(f"{os.path.dirname(os.path.abspath(__file__))}/{parameters_file}") as parameters: + with open( + f"{os.path.dirname(os.path.abspath(__file__))}/{parameters_file}" + ) as parameters: param = yaml.safe_load(parameters) if only_update: for i, name in enumerate(self.__parameters.parnames): dico = param.get(name, False) if dico: - self._parameters.parameters[i].value = dico.get('value') + self._parameters.parameters[i].value = dico.get("value") self._parameters.parameters[i].min = dico.get("min", np.nan) self._parameters.parameters[i].max = dico.get("max", np.nan) else: for name, dico in param.items(): - setattr(self, - f"__{name}", - Parameter( - name=name, - value=dico["value"], - min=dico.get("min", np.nan), - max=dico.get("max", np.nan), - unit=dico.get("unit", u.dimensionless_unscaled) - )) + setattr( + self, + f"__{name}", + Parameter( + name=name, + value=dico["value"], + min=dico.get("min", np.nan), + max=dico.get("max", np.nan), + unit=dico.get("unit", u.dimensionless_unscaled), + ), + ) self._parameters.append(eval(f"self.__{name}")) @staticmethod - def _update_parameters(parameters: Parameters, charge: np.ndarray, counts: np.ndarray, **kwargs) -> Parameters: + def _update_parameters( + parameters: Parameters, charge: np.ndarray, counts: np.ndarray, **kwargs + ) -> Parameters: """ Update the parameters of the FlatFieldSPEMaker class based on the input charge and counts data. @@ -185,8 +183,10 @@ def _update_parameters(parameters: Parameters, charge: np.ndarray, counts: np.nd Parameters: The updated parameters object with the pedestal and mean values and their corresponding limits. """ try: - coeff_ped, coeff_mean = __class__._get_mean_gaussian_fit(charge, counts, **kwargs) - pedestal = parameters['pedestal'] + coeff_ped, coeff_mean = __class__._get_mean_gaussian_fit( + charge, counts, **kwargs + ) + pedestal = parameters["pedestal"] pedestal.value = coeff_ped[1] pedestal.min = coeff_ped[1] - coeff_ped[2] pedestal.max = coeff_ped[1] + coeff_ped[2] @@ -196,9 +196,11 @@ def _update_parameters(parameters: Parameters, charge: np.ndarray, counts: np.nd pedestalWidth.max = 3 * pedestalWidth.value log.debug(f"pedestalWidth updated: {pedestalWidth.value}") - if (coeff_mean[1] - pedestal.value < 0) or ((coeff_mean[1] - coeff_mean[2]) - pedestal.max < 0): + if (coeff_mean[1] - pedestal.value < 0) or ( + (coeff_mean[1] - coeff_mean[2]) - pedestal.max < 0 + ): raise MeanValueError("mean gaussian fit not good") - mean = parameters['mean'] + mean = parameters["mean"] mean.value = coeff_mean[1] - pedestal.value mean.min = (coeff_mean[1] - coeff_mean[2]) - pedestal.max mean.max = (coeff_mean[1] + coeff_mean[2]) - pedestal.min @@ -208,11 +210,15 @@ def _update_parameters(parameters: Parameters, charge: np.ndarray, counts: np.nd log.warning("mean parameters limits and starting value not changed") except Exception as e: log.warning(e, exc_info=True) - log.warning("pedestal and mean parameters limits and starting value not changed") + log.warning( + "pedestal and mean parameters limits and starting value not changed" + ) return parameters @staticmethod - def _get_mean_gaussian_fit(charge: np.ndarray, counts: np.ndarray, extension: str = "", **kwargs) -> Tuple[np.ndarray, np.ndarray]: + def _get_mean_gaussian_fit( + charge: np.ndarray, counts: np.ndarray, extension: str = "", **kwargs + ) -> Tuple[np.ndarray, np.ndarray]: """ Perform a Gaussian fit on the data to determine the pedestal and mean values. @@ -239,44 +245,115 @@ def _get_mean_gaussian_fit(charge: np.ndarray, counts: np.ndarray, extension: st peaks = find_peaks(histo_smoothed, 10) peak_max = np.argmax(histo_smoothed[peaks[0]]) peak_pos, peak_value = charge[peaks[0][peak_max]], counts[peaks[0][peak_max]] - coeff, _ = curve_fit(weight_gaussian, charge[:peaks[0][peak_max]], histo_smoothed[:peaks[0][peak_max]], p0=[peak_value, peak_pos, 1]) + coeff, _ = curve_fit( + weight_gaussian, + charge[: peaks[0][peak_max]], + histo_smoothed[: peaks[0][peak_max]], + p0=[peak_value, peak_pos, 1], + ) if log.getEffectiveLevel() == logging.DEBUG and kwargs.get("display", False): - log.debug('plotting figures with prefit parameters computation') + log.debug("plotting figures with prefit parameters computation") fig, ax = plt.subplots(1, 1, figsize=(5, 5)) - ax.errorbar(charge, counts, np.sqrt(counts), zorder=0, fmt=".", label="data") - ax.plot(charge, histo_smoothed, label=f'smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})') - ax.plot(charge, weight_gaussian(charge, coeff[0], coeff[1], coeff[2]), label='gaussian fit of the pedestal, left tail only') + ax.errorbar( + charge, counts, np.sqrt(counts), zorder=0, fmt=".", label="data" + ) + ax.plot( + charge, + histo_smoothed, + label=f"smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})", + ) + ax.plot( + charge, + weight_gaussian(charge, coeff[0], coeff[1], coeff[2]), + label="gaussian fit of the pedestal, left tail only", + ) ax.set_xlim([peak_pos - 500, None]) - ax.vlines(coeff[1], 0, peak_value, label=f'pedestal initial value = {coeff[1]:.0f}', color='red') - ax.add_patch(Rectangle((coeff[1] - coeff[2], 0), 2 * coeff[2], peak_value, fc=to_rgba('red', 0.5))) + ax.vlines( + coeff[1], + 0, + peak_value, + label=f"pedestal initial value = {coeff[1]:.0f}", + color="red", + ) + ax.add_patch( + Rectangle( + (coeff[1] - coeff[2], 0), + 2 * coeff[2], + peak_value, + fc=to_rgba("red", 0.5), + ) + ) ax.set_xlabel("Charge (ADC)", size=15) ax.set_ylabel("Events", size=15) ax.legend(fontsize=7) - os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/", exist_ok=True) - fig.savefig(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/initialization_pedestal_pixel{extension}_{os.getpid()}.pdf") + os.makedirs( + f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/", + exist_ok=True, + ) + fig.savefig( + f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/initialization_pedestal_pixel{extension}_{os.getpid()}.pdf" + ) fig.clf() plt.close(fig) del fig, ax mask = charge > coeff[1] + 3 * coeff[2] peaks_mean = find_peaks(histo_smoothed[mask]) peak_max_mean = np.argmax(histo_smoothed[mask][peaks_mean[0]]) - peak_pos_mean, peak_value_mean = charge[mask][peaks_mean[0][peak_max_mean]], histo_smoothed[mask][peaks_mean[0][peak_max_mean]] - mask = (charge > ((coeff[1] + peak_pos_mean) / 2)) * (charge < (peak_pos_mean + (peak_pos_mean - coeff[1]) / 2)) - coeff_mean, _ = curve_fit(weight_gaussian, charge[mask], histo_smoothed[mask], p0=[peak_value_mean, peak_pos_mean, 1]) + peak_pos_mean, peak_value_mean = ( + charge[mask][peaks_mean[0][peak_max_mean]], + histo_smoothed[mask][peaks_mean[0][peak_max_mean]], + ) + mask = (charge > ((coeff[1] + peak_pos_mean) / 2)) * ( + charge < (peak_pos_mean + (peak_pos_mean - coeff[1]) / 2) + ) + coeff_mean, _ = curve_fit( + weight_gaussian, + charge[mask], + histo_smoothed[mask], + p0=[peak_value_mean, peak_pos_mean, 1], + ) if log.getEffectiveLevel() == logging.DEBUG and kwargs.get("display", False): - log.debug('plotting figures with prefit parameters computation') + log.debug("plotting figures with prefit parameters computation") fig, ax = plt.subplots(1, 1, figsize=(5, 5)) - ax.errorbar(charge, counts, np.sqrt(counts), zorder=0, fmt=".", label="data") - ax.plot(charge, histo_smoothed, label=f'smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})') - ax.plot(charge, weight_gaussian(charge, coeff_mean[0], coeff_mean[1], coeff_mean[2]), label='gaussian fit of the SPE') - ax.vlines(coeff_mean[1], 0, peak_value, label=f'mean initial value = {coeff_mean[1] - coeff[1]:.0f}', color="red") - ax.add_patch(Rectangle((coeff_mean[1] - coeff_mean[2], 0), 2 * coeff_mean[2], peak_value_mean, fc=to_rgba('red', 0.5))) + ax.errorbar( + charge, counts, np.sqrt(counts), zorder=0, fmt=".", label="data" + ) + ax.plot( + charge, + histo_smoothed, + label=f"smoothed data with savgol filter (windows lenght : {windows_lenght}, order : {order})", + ) + ax.plot( + charge, + weight_gaussian(charge, coeff_mean[0], coeff_mean[1], coeff_mean[2]), + label="gaussian fit of the SPE", + ) + ax.vlines( + coeff_mean[1], + 0, + peak_value, + label=f"mean initial value = {coeff_mean[1] - coeff[1]:.0f}", + color="red", + ) + ax.add_patch( + Rectangle( + (coeff_mean[1] - coeff_mean[2], 0), + 2 * coeff_mean[2], + peak_value_mean, + fc=to_rgba("red", 0.5), + ) + ) ax.set_xlim([peak_pos - 500, None]) ax.set_xlabel("Charge (ADC)", size=15) ax.set_ylabel("Events", size=15) ax.legend(fontsize=7) - os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/", exist_ok=True) - fig.savefig(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/initialization_mean_pixel{extension}_{os.getpid()}.pdf") + os.makedirs( + f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/", + exist_ok=True, + ) + fig.savefig( + f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures/initialization_mean_pixel{extension}_{os.getpid()}.pdf" + ) fig.clf() plt.close(fig) del fig, ax @@ -290,13 +367,23 @@ def _update_table_from_parameters(self) -> None: for param in self._parameters.parameters: if not (param.name in self._results.colnames): - self._results.add_column(Column(data=np.empty((self.npixels), dtype=np.float64), name=param.name, unit=param.unit)) - self._results.add_column(Column(data=np.empty((self.npixels), dtype=np.float64), name=f"{param.name}_error", unit=param.unit)) - - - - -class FlatFieldSingleHHVSPEMaker(FlatFieldSPEMaker) : + self._results.add_column( + Column( + data=np.empty((self.npixels), dtype=np.float64), + name=param.name, + unit=param.unit, + ) + ) + self._results.add_column( + Column( + data=np.empty((self.npixels), dtype=np.float64), + name=f"{param.name}_error", + unit=param.unit, + ) + ) + + +class FlatFieldSingleHHVSPEMaker(FlatFieldSPEMaker): """ This class represents a FlatFieldSingleHHVSPEMaker object. @@ -319,14 +406,14 @@ class FlatFieldSingleHHVSPEMaker(FlatFieldSPEMaker) : make(self, pixels_id=None, multiproc=True, display=True, **kwargs): Method that performs the fit on the specified pixels and returns the fit results. display(self, pixels_id, **kwargs): Method that plots the fit for the specified pixels. """ - __parameters_file = 'parameters_signal.yaml' + + __parameters_file = "parameters_signal.yaml" __fit_array = None _reduced_name = "FlatFieldSingleSPE" __nproc_default = 8 __chunksize_default = 1 - -#constructors + # constructors def __init__(self, charge, counts, *args, **kwargs) -> None: """ Initializes the FlatFieldSingleHHVSPEMaker object. @@ -347,18 +434,33 @@ def __init__(self, charge, counts, *args, **kwargs) -> None: self.__counts = np.ma.asarray(counts) self.__pedestal = Parameter( name="pedestal", - value=(np.min(self.__charge) + np.sum(self.__charge * self.__counts) / np.sum(self.__counts)) / 2, + value=( + np.min(self.__charge) + + np.sum(self.__charge * self.__counts) / np.sum(self.__counts) + ) + / 2, min=np.min(self.__charge), max=np.sum(self.__charge * self.__counts) / np.sum(self.__counts), - unit=u.dimensionless_unscaled + unit=u.dimensionless_unscaled, ) self._parameters.append(self.__pedestal) - self.read_param_from_yaml(kwargs.get('parameters_file', self.__parameters_file)) + self.read_param_from_yaml(kwargs.get("parameters_file", self.__parameters_file)) self._update_table_from_parameters() - self._results.add_column(Column(np.zeros((self.npixels), dtype=np.float64), "likelihood", - unit=u.dimensionless_unscaled)) - self._results.add_column(Column(np.zeros((self.npixels), dtype=np.float64), "pvalue", - unit=u.dimensionless_unscaled)) + self._results.add_column( + Column( + np.zeros((self.npixels), dtype=np.float64), + "likelihood", + unit=u.dimensionless_unscaled, + ) + ) + self._results.add_column( + Column( + np.zeros((self.npixels), dtype=np.float64), + "pvalue", + unit=u.dimensionless_unscaled, + ) + ) + @classmethod def create_from_chargesContainer(cls, signal: ChargesContainer, **kwargs): """ @@ -370,42 +472,46 @@ def create_from_chargesContainer(cls, signal: ChargesContainer, **kwargs): FlatFieldSingleHHVSPEMaker: An instance of FlatFieldSingleHHVSPEMaker. """ histo = ChargesMaker.histo_hg(signal, autoscale=True) - return cls(charge=histo[1], counts=histo[0], pixels_id=signal.pixels_id, **kwargs) - + return cls( + charge=histo[1], counts=histo[0], pixels_id=signal.pixels_id, **kwargs + ) + @classmethod - def create_from_run_number(cls, run_number : int, **kwargs) : - raise NotImplementedError("Need to implement here the use of the WaveformsMaker and ChargesMaker to produce the chargesContainer to be pass into the __ini__") - -#getters and setters + def create_from_run_number(cls, run_number: int, **kwargs): + raise NotImplementedError( + "Need to implement here the use of the WaveformsMaker and ChargesMaker to produce the chargesContainer to be pass into the __ini__" + ) + + # getters and setters @property - def charge(self) : + def charge(self): """ Returns a deep copy of the __charge attribute. """ return copy.deepcopy(self.__charge) @property - def _charge(self) : + def _charge(self): """ Returns the __charge attribute. """ return self.__charge @property - def counts(self) : + def counts(self): """ Returns a deep copy of the __counts attribute. """ return copy.deepcopy(self.__counts) @property - def _counts(self) : + def _counts(self): """ Returns the __counts attribute. """ return self.__counts -#methods + # methods def _fill_results_table_from_dict(self, dico: dict, pixels_id: np.ndarray) -> None: """ Populates the results table with fit values and errors for each pixel based on the dictionary provided as input. @@ -424,23 +530,46 @@ def _fill_results_table_from_dict(self, dico: dict, pixels_id: np.ndarray) -> No if not ((values is None) or (errors is None)): index = np.argmax(self._results["pixels_id"] == pixels_id[i]) if len(values) != len(chi2_sig.parameters): - e = Exception("the size out the minuit output parameters values array does not fit the signature of the minimized cost function") + e = Exception( + "the size out the minuit output parameters values array does not fit the signature of the minimized cost function" + ) log.error(e, exc_info=True) raise e for j, key in enumerate(chi2_sig.parameters): self._results[key][index] = values[j] self._results[f"{key}_error"][index] = errors[j] - if key == 'mean': + if key == "mean": self._high_gain[index] = values[j] - self._results[f"high_gain_error"][index] = [errors[j], errors[j]] + self._results[f"high_gain_error"][index] = [ + errors[j], + errors[j], + ] self._results[f"high_gain"][index] = values[j] - self._results['is_valid'][index] = True - self._results["likelihood"][index] = __class__.__fit_array[i].fcn(__class__.__fit_array[i].values) - ndof = self._counts.data[index][~self._counts.mask[index]].shape[0] - __class__.__fit_array[i].nfit - self._results["pvalue"][index] = Statistics.chi2_pvalue(ndof, __class__.__fit_array[i].fcn(__class__.__fit_array[i].values)) + self._results["is_valid"][index] = True + self._results["likelihood"][index] = __class__.__fit_array[i].fcn( + __class__.__fit_array[i].values + ) + ndof = ( + self._counts.data[index][~self._counts.mask[index]].shape[0] + - __class__.__fit_array[i].nfit + ) + self._results["pvalue"][index] = Statistics.chi2_pvalue( + ndof, __class__.__fit_array[i].fcn(__class__.__fit_array[i].values) + ) @staticmethod - def _NG_Likelihood_Chi2(pp : float,res: float,mu2: float,n: float,muped: float,sigped: float,lum: float,charge : np.ndarray,counts:np.ndarray,**kwargs) : + def _NG_Likelihood_Chi2( + pp: float, + res: float, + mu2: float, + n: float, + muped: float, + sigped: float, + lum: float, + charge: np.ndarray, + counts: np.ndarray, + **kwargs, + ): """ Calculates the chi-square value using the MPE2 function. Parameters: @@ -457,16 +586,18 @@ def _NG_Likelihood_Chi2(pp : float,res: float,mu2: float,n: float,muped: float,s Returns: float: The chi-square value. """ - pdf = MPE2(charge,pp,res,mu2,n,muped,sigped,lum,**kwargs) - #log.debug(f"pdf : {np.sum(pdf)}") + pdf = MPE2(charge, pp, res, mu2, n, muped, sigped, lum, **kwargs) + # log.debug(f"pdf : {np.sum(pdf)}") Ntot = np.sum(counts) - #log.debug(f'Ntot : {Ntot}') + # log.debug(f'Ntot : {Ntot}') mask = counts > 0 - Lik = np.sum(((pdf*Ntot-counts)[mask])**2/counts[mask]) #2 times faster + Lik = np.sum( + ((pdf * Ntot - counts)[mask]) ** 2 / counts[mask] + ) # 2 times faster return Lik @staticmethod - def cost(charge:np.ndarray,counts:np.ndarray) : + def cost(charge: np.ndarray, counts: np.ndarray): """ Defines a function called Chi2 that calculates the chi-square value using the _NG_Likelihood_Chi2 method. Parameters: @@ -475,18 +606,41 @@ def cost(charge:np.ndarray,counts:np.ndarray) : Returns: function: The Chi2 function. """ - def Chi2(pedestal: float,pp: float,luminosity: float,resolution: float,mean: float,n: float,pedestalWidth: float) : - #assert not(np.isnan(pp) or np.isnan(resolution) or np.isnan(mean) or np.isnan(n) or np.isnan(pedestal) or np.isnan(pedestalWidth) or np.isnan(luminosity)) + + def Chi2( + pedestal: float, + pp: float, + luminosity: float, + resolution: float, + mean: float, + n: float, + pedestalWidth: float, + ): + # assert not(np.isnan(pp) or np.isnan(resolution) or np.isnan(mean) or np.isnan(n) or np.isnan(pedestal) or np.isnan(pedestalWidth) or np.isnan(luminosity)) for i in range(1000): - if (gammainc(i+1,luminosity) < 1e-5): + if gammainc(i + 1, luminosity) < 1e-5: ntotalPE = i break - kwargs = {"ntotalPE" : ntotalPE} - return __class__._NG_Likelihood_Chi2(pp,resolution,mean,n,pedestal,pedestalWidth,luminosity,charge,counts,**kwargs) + kwargs = {"ntotalPE": ntotalPE} + return __class__._NG_Likelihood_Chi2( + pp, + resolution, + mean, + n, + pedestal, + pedestalWidth, + luminosity, + charge, + counts, + **kwargs, + ) + return Chi2 - - #@njit(parallel=True,nopython = True) - def _make_fit_array_from_parameters(self, pixels_id : np.ndarray = None, **kwargs) -> np.ndarray: + + # @njit(parallel=True,nopython = True) + def _make_fit_array_from_parameters( + self, pixels_id: np.ndarray = None, **kwargs + ) -> np.ndarray: """ Create an array of Minuit fit instances based on the parameters and data for each pixel. @@ -506,22 +660,39 @@ def _make_fit_array_from_parameters(self, pixels_id : np.ndarray = None, **kwarg for i, _id in enumerate(pixels_id): index = np.where(self.pixels_id == _id)[0][0] - parameters = __class__._update_parameters(self.parameters, self._charge[index].data[~self._charge[index].mask], self._counts[index].data[~self._charge[index].mask], pixel_id=_id, **kwargs) + parameters = __class__._update_parameters( + self.parameters, + self._charge[index].data[~self._charge[index].mask], + self._counts[index].data[~self._charge[index].mask], + pixel_id=_id, + **kwargs, + ) minuitParameters = UtilsMinuit.make_minuit_par_kwargs(parameters) - minuit_kwargs = {parname: minuitParameters['values'][parname] for parname in minuitParameters['values']} - log.info(f'creation of fit instance for pixel: {_id}') - fit_array[i] = Minuit(__class__.cost(self._charge[index].data[~self._charge[index].mask], self._counts[index].data[~self._charge[index].mask]), **minuit_kwargs) - log.debug('fit created') + minuit_kwargs = { + parname: minuitParameters["values"][parname] + for parname in minuitParameters["values"] + } + log.info(f"creation of fit instance for pixel: {_id}") + fit_array[i] = Minuit( + __class__.cost( + self._charge[index].data[~self._charge[index].mask], + self._counts[index].data[~self._charge[index].mask], + ), + **minuit_kwargs, + ) + log.debug("fit created") fit_array[i].errordef = Minuit.LIKELIHOOD fit_array[i].strategy = 0 fit_array[i].tol = 1e40 fit_array[i].print_level = 1 fit_array[i].throw_nan = True - UtilsMinuit.set_minuit_parameters_limits_and_errors(fit_array[i], minuitParameters) + UtilsMinuit.set_minuit_parameters_limits_and_errors( + fit_array[i], minuitParameters + ) log.debug(fit_array[i].values) log.debug(fit_array[i].limits) log.debug(fit_array[i].fixed) - + return fit_array @staticmethod @@ -544,11 +715,13 @@ def run_fit(i: int) -> dict: log.info("Finished") return {f"values_{i}": _values, f"errors_{i}": _errors} - def make(self, - pixels_id :np.ndarray= None, - multiproc : bool = True, - display : bool = True, - **kwargs) -> np.ndarray: + def make( + self, + pixels_id: np.ndarray = None, + multiproc: bool = True, + display: bool = True, + **kwargs, + ) -> np.ndarray: """ Perform a fit on specified pixels and return the fit results. @@ -572,70 +745,87 @@ def make(self, results = maker.make(pixels_id=[1, 2, 3]) """ log.info("running maker") - log.info('checking asked pixels id') - if pixels_id is None : + log.info("checking asked pixels id") + if pixels_id is None: pixels_id = self.pixels_id npix = self.npixels - else : - log.debug('checking that asked pixels id are in data') + else: + log.debug("checking that asked pixels id are in data") pixels_id = np.asarray(pixels_id) - mask = np.array([_id in self.pixels_id for _id in pixels_id],dtype = bool) - if False in mask : + mask = np.array([_id in self.pixels_id for _id in pixels_id], dtype=bool) + if False in mask: log.debug(f"The following pixels are not in data : {pixels_id[~mask]}") pixels_id = pixels_id[mask] npix = len(pixels_id) - if npix == 0 : - log.warning('The asked pixels id are all out of the data') + if npix == 0: + log.warning("The asked pixels id are all out of the data") return None - else : + else: log.info("creation of the fits instance array") __class__.__fit_array = self._make_fit_array_from_parameters( - pixels_id = pixels_id, - display = display, - **kwargs - ) + pixels_id=pixels_id, display=display, **kwargs + ) log.info("running fits") - if multiproc : - nproc = kwargs.get("nproc",__class__.__nproc_default) - chunksize = kwargs.get("chunksize",max(__class__.__chunksize_default,npix//(nproc*10))) + if multiproc: + nproc = kwargs.get("nproc", __class__.__nproc_default) + chunksize = kwargs.get( + "chunksize", + max(__class__.__chunksize_default, npix // (nproc * 10)), + ) log.info(f"pooling with nproc {nproc}, chunksize {chunksize}") t = time.time() - with Pool(nproc) as pool: - result = pool.starmap_async(__class__.run_fit, - [(i,) for i in range(npix)], - chunksize=chunksize) + with Pool(nproc) as pool: + result = pool.starmap_async( + __class__.run_fit, + [(i,) for i in range(npix)], + chunksize=chunksize, + ) result.wait() - try : + try: res = result.get() - except Exception as e : - log.error(e,exc_info=True) + except Exception as e: + log.error(e, exc_info=True) raise e log.debug(res) - log.info(f'time for multiproc with starmap_async execution is {time.time() - t:.2e} sec') - else : + log.info( + f"time for multiproc with starmap_async execution is {time.time() - t:.2e} sec" + ) + else: log.info("running in mono-cpu") t = time.time() res = [__class__.run_fit(i) for i in range(npix)] log.debug(res) - log.info(f'time for singleproc execution is {time.time() - t:.2e} sec') + log.info(f"time for singleproc execution is {time.time() - t:.2e} sec") log.info("filling result table from fits results") - self._fill_results_table_from_dict(res,pixels_id) + self._fill_results_table_from_dict(res, pixels_id) output = copy.copy(__class__.__fit_array) __class__.__fit_array = None - if display : + if display: log.info("plotting") - self.display(pixels_id,**kwargs) + self.display(pixels_id, **kwargs) return output - - def plot_single(pixel_id: int, charge: np.ndarray, counts: np.ndarray, pp: float, resolution: float, gain: float, gain_error: float, n: float, pedestal: float, pedestalWidth: float, luminosity: float, likelihood: float) -> tuple: + def plot_single( + pixel_id: int, + charge: np.ndarray, + counts: np.ndarray, + pp: float, + resolution: float, + gain: float, + gain_error: float, + n: float, + pedestal: float, + pedestalWidth: float, + luminosity: float, + likelihood: float, + ) -> tuple: """ Generate a plot of the data and a model fit for a specific pixel. @@ -658,20 +848,23 @@ def plot_single(pixel_id: int, charge: np.ndarray, counts: np.ndarray, pp: float """ fig, ax = plt.subplots(1, 1, figsize=(8, 8)) ax.errorbar(charge, counts, np.sqrt(counts), zorder=0, fmt=".", label="data") - ax.plot(charge, - np.trapz(counts, charge) * MPE2( - charge, - pp, - resolution, - gain, - n, - pedestal, - pedestalWidth, - luminosity, - ), - zorder=1, - linewidth=2, - label=f"SPE model fit \n gain : {gain - gain_error:.2f} < {gain:.2f} < {gain + gain_error:.2f} ADC/pe,\n likelihood : {likelihood:.2f}") + ax.plot( + charge, + np.trapz(counts, charge) + * MPE2( + charge, + pp, + resolution, + gain, + n, + pedestal, + pedestalWidth, + luminosity, + ), + zorder=1, + linewidth=2, + label=f"SPE model fit \n gain : {gain - gain_error:.2f} < {gain:.2f} < {gain + gain_error:.2f} ADC/pe,\n likelihood : {likelihood:.2f}", + ) ax.set_xlabel("Charge (ADC)", size=15) ax.set_ylabel("Events", size=15) ax.set_title(f"SPE fit pixel id : {pixel_id}") @@ -679,7 +872,6 @@ def plot_single(pixel_id: int, charge: np.ndarray, counts: np.ndarray, pp: float ax.legend(fontsize=18) return fig, ax - def display(self, pixels_id: np.ndarray, **kwargs) -> None: """ Display and save the plot for each specified pixel ID. @@ -689,23 +881,23 @@ def display(self, pixels_id: np.ndarray, **kwargs) -> None: **kwargs: Additional keyword arguments. figpath (str): The path to save the generated plot figures. Defaults to "/tmp/NectarGain_pid{os.getpid()}". """ - figpath = kwargs.get('figpath', f"/tmp/NectarGain_pid{os.getpid()}") + figpath = kwargs.get("figpath", f"/tmp/NectarGain_pid{os.getpid()}") os.makedirs(figpath, exist_ok=True) for _id in pixels_id: - index = np.argmax(self._results['pixels_id'] == _id) + index = np.argmax(self._results["pixels_id"] == _id) fig, ax = __class__.plot_single( _id, self._charge[index], self._counts[index], - self._results['pp'][index].value, - self._results['resolution'][index].value, - self._results['high_gain'][index].value, - self._results['high_gain_error'][index].value.mean(), - self._results['n'][index].value, - self._results['pedestal'][index].value, - self._results['pedestalWidth'][index].value, - self._results['luminosity'][index].value, - self._results['likelihood'][index], + self._results["pp"][index].value, + self._results["resolution"][index].value, + self._results["high_gain"][index].value, + self._results["high_gain_error"][index].value.mean(), + self._results["n"][index].value, + self._results["pedestal"][index].value, + self._results["pedestalWidth"][index].value, + self._results["luminosity"][index].value, + self._results["likelihood"][index], ) fig.savefig(f"{figpath}/fit_SPE_pixel{_id}.pdf") fig.clf() @@ -713,13 +905,13 @@ def display(self, pixels_id: np.ndarray, **kwargs) -> None: del fig, ax - class FlatFieldSingleHHVStdSPEMaker(FlatFieldSingleHHVSPEMaker): """class to perform fit of the SPE signal with n and pp fixed""" - __parameters_file = 'parameters_signalStd.yaml' + + __parameters_file = "parameters_signalStd.yaml" _reduced_name = "FlatFieldSingleStdSPE" - - def __init__(self, charge : np.ndarray, counts: np.ndarray, *args, **kwargs) -> None: + + def __init__(self, charge: np.ndarray, counts: np.ndarray, *args, **kwargs) -> None: """ Initializes a new instance of the FlatFieldSingleHHVStdSPEMaker class. @@ -732,7 +924,7 @@ def __init__(self, charge : np.ndarray, counts: np.ndarray, *args, **kwargs) -> super().__init__(charge, counts, *args, **kwargs) self.__fix_parameters() - def __fix_parameters(self) -> None: + def __fix_parameters(self) -> None: """ Fixes the values of the n and pp parameters by setting their frozen attribute to True. """ @@ -775,10 +967,18 @@ class FlatFieldSingleNominalSPEMaker(FlatFieldSingleHHVSPEMaker): maker.display(pixels_id=[1, 2, 3]) """ - __parameters_file = 'parameters_signal_fromHHVFit.yaml' + __parameters_file = "parameters_signal_fromHHVFit.yaml" _reduced_name = "FlatFieldSingleNominalSPE" - def __init__(self, charge:np.ndarray, counts:np.ndarray, nectarGainSPEresult: str, same_luminosity: bool = False, *args, **kwargs) -> None: + def __init__( + self, + charge: np.ndarray, + counts: np.ndarray, + nectarGainSPEresult: str, + same_luminosity: bool = False, + *args, + **kwargs, + ) -> None: """ Initializes an instance of FlatFieldSingleNominalSPEMaker. @@ -795,7 +995,9 @@ def __init__(self, charge:np.ndarray, counts:np.ndarray, nectarGainSPEresult: st self.__same_luminosity = same_luminosity self.__nectarGainSPEresult = self._read_SPEresult(nectarGainSPEresult) if len(self.__nectarGainSPEresult) == 0: - log.warning("The intersection between pixels id from the data and those valid from the SPE fit result is empty") + log.warning( + "The intersection between pixels id from the data and those valid from the SPE fit result is empty" + ) @property def nectarGainSPEresult(self): @@ -826,8 +1028,8 @@ def _read_SPEresult(self, nectarGainSPEresult: str): argsort = [] mask = [] for _id in self._pixels_id: - if _id in table['pixels_id']: - argsort.append(np.where(_id == table['pixels_id'])[0][0]) + if _id in table["pixels_id"]: + argsort.append(np.where(_id == table["pixels_id"])[0][0]) mask.append(True) else: mask.append(False) @@ -864,10 +1066,21 @@ def _make_fit_array_from_parameters(self, pixels_id=None, **kwargs): Returns: array-like: The fit array. """ - return super()._make_fit_array_from_parameters(pixels_id=pixels_id, nectarGainSPEresult=self.__nectarGainSPEresult, **kwargs) + return super()._make_fit_array_from_parameters( + pixels_id=pixels_id, + nectarGainSPEresult=self.__nectarGainSPEresult, + **kwargs, + ) @staticmethod - def _update_parameters(parameters:Parameters, charge:np.ndarray, counts:np.ndarray, pixel_id, nectarGainSPEresult:QTable, **kwargs): + def _update_parameters( + parameters: Parameters, + charge: np.ndarray, + counts: np.ndarray, + pixel_id, + nectarGainSPEresult: QTable, + **kwargs, + ): """ Updates the parameters with the fixed values from the fitted data obtained from a 1400V run. @@ -896,4 +1109,4 @@ def _update_parameters(parameters:Parameters, charge:np.ndarray, counts:np.ndarr if luminosity.frozen: luminosity.value = nectarGainSPEresult[index]["luminosity"].value - return param \ No newline at end of file + return param diff --git a/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py b/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py index 9ec2aba9..3aa17e54 100644 --- a/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py +++ b/src/nectarchain/makers/calibration/gain/PhotoStatisticMakers.py @@ -1,30 +1,28 @@ -import sys import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +import sys + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers -import numpy as np -from scipy.stats import linregress -from matplotlib import pyplot as plt -import astropy.units as u -from astropy.visualization import quantity_support -from astropy.table import QTable,Column +import copy import os from datetime import date from pathlib import Path -import copy +import astropy.units as u +import numpy as np +from astropy.table import Column, QTable +from astropy.visualization import quantity_support from ctapipe_io_nectarcam import constants +from matplotlib import pyplot as plt +from scipy.stats import linregress -from ....data.container import ChargesContainer,ChargesContainerIO - +from ....data.container import ChargesContainer, ChargesContainerIO from ...chargesMakers import ChargesMaker - from .gainMakers import GainMaker - __all__ = ["PhotoStatisticMaker"] @@ -76,19 +74,21 @@ class PhotoStatisticMaker(GainMaker): - `BLG`: Property to calculate the BLG value. - `gainLG`: Property to calculate the gain for low gain. """ + _reduced_name = "PhotoStatistic" -#constructors - def __init__(self, - FFcharge_hg : np.ndarray, - FFcharge_lg : np.ndarray, - Pedcharge_hg: np.ndarray, - Pedcharge_lg: np.ndarray, - coefCharge_FF_Ped : float, - SPE_resolution, - *args, - **kwargs - ) -> None: + # constructors + def __init__( + self, + FFcharge_hg: np.ndarray, + FFcharge_lg: np.ndarray, + Pedcharge_hg: np.ndarray, + Pedcharge_lg: np.ndarray, + coefCharge_FF_Ped: float, + SPE_resolution, + *args, + **kwargs, + ) -> None: """ Initializes the instance of the PhotoStatisticMaker class with charge data and other parameters. @@ -106,73 +106,80 @@ def __init__(self, Returns: None """ - super().__init__(*args,**kwargs) + super().__init__(*args, **kwargs) self.__coefCharge_FF_Ped = coefCharge_FF_Ped - + self.__FFcharge_hg = FFcharge_hg self.__FFcharge_lg = FFcharge_lg self.__Pedcharge_hg = Pedcharge_hg self.__Pedcharge_lg = Pedcharge_lg - if isinstance(SPE_resolution,np.ndarray) and len(SPE_resolution) == self.npixels : + if ( + isinstance(SPE_resolution, np.ndarray) + and len(SPE_resolution) == self.npixels + ): self.__SPE_resolution = SPE_resolution - elif isinstance(SPE_resolution,list) and len(SPE_resolution) == self.npixels : + elif isinstance(SPE_resolution, list) and len(SPE_resolution) == self.npixels: self.__SPE_resolution = np.array(SPE_resolution) - elif isinstance(SPE_resolution,float) : - self.__SPE_resolution = SPE_resolution * np.ones((self.npixels)) - else : - e = TypeError("SPE_resolution must be a float, a numpy.ndarray or list instance") + elif isinstance(SPE_resolution, float): + self.__SPE_resolution = SPE_resolution * np.ones((self.npixels)) + else: + e = TypeError( + "SPE_resolution must be a float, a numpy.ndarray or list instance" + ) raise e self.__check_shape() - @classmethod - def create_from_chargeContainer(cls, - FFcharge : ChargesContainer, - Pedcharge : ChargesContainer, - coefCharge_FF_Ped : float, - SPE_result, - **kwargs) : + def create_from_chargeContainer( + cls, + FFcharge: ChargesContainer, + Pedcharge: ChargesContainer, + coefCharge_FF_Ped: float, + SPE_result, + **kwargs, + ): """ Create an instance of the PhotoStatisticMaker class from Pedestal and Flatfield runs stored in ChargesContainer. - + Args: FFcharge (ChargesContainer): Array of charge data for the FF image. Pedcharge (ChargesContainer): Array of charge data for the Ped image. coefCharge_FF_Ped (float): Coefficient to convert FF charge to Ped charge. SPE_result (str or Path): Path to the SPE result file (optional). **kwargs: Additional keyword arguments for initializing the PhotoStatisticMaker instance. - + Returns: PhotoStatisticMaker: An instance of the PhotoStatisticMaker class created from the ChargesContainer instances. """ - if isinstance(SPE_result , str) or isinstance(SPE_result , Path) : - SPE_resolution,SPE_pixels_id = __class__.__readSPE(SPE_result) - else : + if isinstance(SPE_result, str) or isinstance(SPE_result, Path): + SPE_resolution, SPE_pixels_id = __class__.__readSPE(SPE_result) + else: SPE_pixels_id = None - kwargs_init = __class__.__get_charges_FF_Ped_reshaped(FFcharge, - Pedcharge, - SPE_resolution, - SPE_pixels_id) + kwargs_init = __class__.__get_charges_FF_Ped_reshaped( + FFcharge, Pedcharge, SPE_resolution, SPE_pixels_id + ) kwargs.update(kwargs_init) - return cls(coefCharge_FF_Ped = coefCharge_FF_Ped, **kwargs) + return cls(coefCharge_FF_Ped=coefCharge_FF_Ped, **kwargs) @classmethod - def create_from_run_numbers(cls, FFrun: int, Pedrun: int, SPE_result: str, **kwargs): + def create_from_run_numbers( + cls, FFrun: int, Pedrun: int, SPE_result: str, **kwargs + ): """ Create an instance of the PhotoStatisticMaker class by reading the FF (Flat Field) and Ped (Pedestal) charge data from run numbers. - + Args: FFrun (int): The run number for the FF charge data. Pedrun (int): The run number for the Ped charge data. SPE_result (str): The path to the SPE result file. **kwargs: Additional keyword arguments. - + Returns: PhotoStatisticMaker: An instance of the PhotoStatisticMaker class created from the FF and Ped charge data and the SPE result file. """ @@ -182,9 +189,9 @@ def create_from_run_numbers(cls, FFrun: int, Pedrun: int, SPE_result: str, **kwa kwargs.update(Pedkwargs) return cls.create_from_chargeContainer(SPE_result=SPE_result, **kwargs) -#methods + # methods @staticmethod - def __readSPE(SPEresults) -> tuple: + def __readSPE(SPEresults) -> tuple: """ Reads the SPE resolution from a file and returns the resolution values and corresponding pixel IDs. @@ -194,13 +201,21 @@ def __readSPE(SPEresults) -> tuple: Returns: tuple: A tuple containing the SPE resolution values and corresponding pixel IDs. """ - log.info(f'reading SPE resolution from {SPEresults}') + log.info(f"reading SPE resolution from {SPEresults}") table = QTable.read(SPEresults) - table.sort('pixels_id') - return table['resolution'][table['is_valid']].value,table['pixels_id'][table['is_valid']].value + table.sort("pixels_id") + return ( + table["resolution"][table["is_valid"]].value, + table["pixels_id"][table["is_valid"]].value, + ) @staticmethod - def __get_charges_FF_Ped_reshaped( FFcharge : ChargesContainer, Pedcharge : ChargesContainer, SPE_resolution : np.ndarray, SPE_pixels_id: np.ndarray)-> dict : + def __get_charges_FF_Ped_reshaped( + FFcharge: ChargesContainer, + Pedcharge: ChargesContainer, + SPE_resolution: np.ndarray, + SPE_pixels_id: np.ndarray, + ) -> dict: """ Reshapes the FF (Flat Field) and Ped (Pedestal) charges based on the intersection of pixel IDs between the two charges. Selects the charges for the high-gain and low-gain channels and returns them along with the common pixel IDs. @@ -217,18 +232,32 @@ def __get_charges_FF_Ped_reshaped( FFcharge : ChargesContainer, Pedcharge : Char log.info("reshape of SPE, Ped and FF data with intersection of pixel ids") out = {} - FFped_intersection = np.intersect1d(Pedcharge.pixels_id,FFcharge.pixels_id) - if not(SPE_pixels_id is None) : - SPEFFPed_intersection = np.intersect1d(FFped_intersection,SPE_pixels_id) - mask_SPE = np.array([SPE_pixels_id[i] in SPEFFPed_intersection for i in range(len(SPE_pixels_id))],dtype = bool) + FFped_intersection = np.intersect1d(Pedcharge.pixels_id, FFcharge.pixels_id) + if not (SPE_pixels_id is None): + SPEFFPed_intersection = np.intersect1d(FFped_intersection, SPE_pixels_id) + mask_SPE = np.array( + [ + SPE_pixels_id[i] in SPEFFPed_intersection + for i in range(len(SPE_pixels_id)) + ], + dtype=bool, + ) out["SPE_resolution"] = SPE_resolution[mask_SPE] out["pixels_id"] = SPEFFPed_intersection - out["FFcharge_hg"] = ChargesMaker.select_charges_hg(FFcharge,SPEFFPed_intersection) - out["FFcharge_lg"] = ChargesMaker.select_charges_lg(FFcharge,SPEFFPed_intersection) - out["Pedcharge_hg"] = ChargesMaker.select_charges_hg(Pedcharge,SPEFFPed_intersection) - out["Pedcharge_lg"] = ChargesMaker.select_charges_lg(Pedcharge,SPEFFPed_intersection) - + out["FFcharge_hg"] = ChargesMaker.select_charges_hg( + FFcharge, SPEFFPed_intersection + ) + out["FFcharge_lg"] = ChargesMaker.select_charges_lg( + FFcharge, SPEFFPed_intersection + ) + out["Pedcharge_hg"] = ChargesMaker.select_charges_hg( + Pedcharge, SPEFFPed_intersection + ) + out["Pedcharge_lg"] = ChargesMaker.select_charges_lg( + Pedcharge, SPEFFPed_intersection + ) + log.info(f"data have {len(SPEFFPed_intersection)} pixels in common") return out @@ -242,12 +271,16 @@ def __readFF(FFRun: int, **kwargs) -> dict: Returns: - dict: A dictionary containing the FF charge data (`FFcharge`) and the coefficient for the FF charge (`coefCharge_FF_Ped`). """ - log.info('reading FF data') - method = kwargs.get('method', 'FullWaveformSum') - FFchargeExtractorWindowLength = kwargs.get('FFchargeExtractorWindowLength', None) - if method != 'FullWaveformSum': + log.info("reading FF data") + method = kwargs.get("method", "FullWaveformSum") + FFchargeExtractorWindowLength = kwargs.get( + "FFchargeExtractorWindowLength", None + ) + if method != "FullWaveformSum": if FFchargeExtractorWindowLength is None: - e = Exception(f"we have to specify FFchargeExtractorWindowLength argument if charge extractor method is not FullwaveformSum") + e = Exception( + f"we have to specify FFchargeExtractorWindowLength argument if charge extractor method is not FullwaveformSum" + ) log.error(e, exc_info=True) raise e else: @@ -256,8 +289,10 @@ def __readFF(FFRun: int, **kwargs) -> dict: coefCharge_FF_Ped = 1 if isinstance(FFRun, int): try: - FFcharge = ChargesContainerIO.load(f"{os.environ['NECTARCAMDATA']}/charges/{method}", FFRun) - log.info(f'charges have ever been computed for FF run {FFRun}') + FFcharge = ChargesContainerIO.load( + f"{os.environ['NECTARCAMDATA']}/charges/{method}", FFRun + ) + log.info(f"charges have ever been computed for FF run {FFRun}") except Exception as e: log.error("charge have not been yet computed") raise e @@ -266,6 +301,7 @@ def __readFF(FFRun: int, **kwargs) -> dict: log.error(e, exc_info=True) raise e return {"FFcharge": FFcharge, "coefCharge_FF_Ped": coefCharge_FF_Ped} + @staticmethod def __readPed(PedRun: int, **kwargs) -> dict: """ @@ -276,12 +312,14 @@ def __readPed(PedRun: int, **kwargs) -> dict: Returns: - dict: A dictionary containing the Ped charge data (`Pedcharge`). """ - log.info('reading Ped data') - method = 'FullWaveformSum' # kwargs.get('method','std') + log.info("reading Ped data") + method = "FullWaveformSum" # kwargs.get('method','std') if isinstance(PedRun, int): try: - Pedcharge = ChargesContainerIO.load(f"{os.environ['NECTARCAMDATA']}/charges/{method}", PedRun) - log.info(f'charges have ever been computed for Ped run {PedRun}') + Pedcharge = ChargesContainerIO.load( + f"{os.environ['NECTARCAMDATA']}/charges/{method}", PedRun + ) + log.info(f"charges have ever been computed for Ped run {PedRun}") except Exception as e: log.error("charge have not been yet computed") raise e @@ -291,14 +329,16 @@ def __readPed(PedRun: int, **kwargs) -> dict: raise e return {"Pedcharge": Pedcharge} - def __check_shape(self) -> None: + def __check_shape(self) -> None: """ Checks the shape of certain attributes and raises an exception if the shape is not as expected. """ - try : - self.__FFcharge_hg[0] * self.__FFcharge_lg[0] * self.__Pedcharge_hg[0] * self.__Pedcharge_lg[0] * self.__SPE_resolution * self._pixels_id - except Exception as e : - log.error(e,exc_info = True) + try: + self.__FFcharge_hg[0] * self.__FFcharge_lg[0] * self.__Pedcharge_hg[ + 0 + ] * self.__Pedcharge_lg[0] * self.__SPE_resolution * self._pixels_id + except Exception as e: + log.error(e, exc_info=True) raise e def make(self, **kwargs) -> None: @@ -311,13 +351,14 @@ def make(self, **kwargs) -> None: Returns: None """ - log.info('running photo statistic method') + log.info("running photo statistic method") self._results["high_gain"] = self.gainHG self._results["low_gain"] = self.gainLG # self._results["is_valid"] = self._SPEvalid - - def plot_correlation(photoStat_gain: np.ndarray, SPE_gain: np.ndarray) -> plt.Figure: + def plot_correlation( + photoStat_gain: np.ndarray, SPE_gain: np.ndarray + ) -> plt.Figure: """ Plot the correlation between the photo statistic gain and the single photoelectron (SPE) gain. @@ -333,7 +374,9 @@ def plot_correlation(photoStat_gain: np.ndarray, SPE_gain: np.ndarray) -> plt.Fi mask = (photoStat_gain > 20) * (SPE_gain > 0) * (photoStat_gain < 80) # Perform a linear regression analysis on the filtered data points - a, b, r, p_value, std_err = linregress(photoStat_gain[mask], SPE_gain[mask], 'greater') + a, b, r, p_value, std_err = linregress( + photoStat_gain[mask], SPE_gain[mask], "greater" + ) # Generate a range of x-values for the linear fit line x = np.linspace(photoStat_gain[mask].min(), photoStat_gain[mask].max(), 1000) @@ -347,11 +390,15 @@ def plot_correlation(photoStat_gain: np.ndarray, SPE_gain: np.ndarray) -> plt.Fi ax.scatter(photoStat_gain[mask], SPE_gain[mask], marker=".") # Plot the linear fit line using the x-values and the lambda function - ax.plot(x, y(x), color='red', - label=f"linear fit,\n a = {a:.2e},\n b = {b:.2e},\n r = {r:.2e},\n p_value = {p_value:.2e},\n std_err = {std_err:.2e}") + ax.plot( + x, + y(x), + color="red", + label=f"linear fit,\n a = {a:.2e},\n b = {b:.2e},\n r = {r:.2e},\n p_value = {p_value:.2e},\n std_err = {std_err:.2e}", + ) # Plot the line y = x - ax.plot(x, x, color='black', label="y = x") + ax.plot(x, x, color="black", label="y = x") ax.set_xlabel("Gain Photo stat (ADC)", size=15) ax.set_ylabel("Gain SPE fit (ADC)", size=15) @@ -359,6 +406,7 @@ def plot_correlation(photoStat_gain: np.ndarray, SPE_gain: np.ndarray) -> plt.Fi return fig + @property def SPE_resolution(self) -> float: """ @@ -423,7 +471,14 @@ def BHG(self) -> float: float: The BHG value. """ min_events = np.min((self.__FFcharge_hg.shape[0], self.__Pedcharge_hg.shape[0])) - upper = (np.power(self.__FFcharge_hg.mean(axis=1)[:min_events] - self.__Pedcharge_hg.mean(axis=1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeHG.mean(), 2)).mean(axis=0) + upper = ( + np.power( + self.__FFcharge_hg.mean(axis=1)[:min_events] + - self.__Pedcharge_hg.mean(axis=1)[:min_events] * self.__coefCharge_FF_Ped + - self.meanChargeHG.mean(), + 2, + ) + ).mean(axis=0) lower = np.power(self.meanChargeHG.mean(), 2) return np.sqrt(upper / lower) @@ -436,8 +491,11 @@ def gainHG(self) -> float: Returns: float: The gain for high gain charge data. """ - return ((np.power(self.sigmaChargeHG, 2) - np.power(self.sigmaPedHG, 2) - np.power(self.BHG * self.meanChargeHG, 2)) - / (self.meanChargeHG * (1 + np.power(self.SPE_resolution, 2)))) + return ( + np.power(self.sigmaChargeHG, 2) + - np.power(self.sigmaPedHG, 2) + - np.power(self.BHG * self.meanChargeHG, 2) + ) / (self.meanChargeHG * (1 + np.power(self.SPE_resolution, 2))) @property @@ -493,7 +551,14 @@ def BLG(self) -> float: float: The BLG value. """ min_events = np.min((self.__FFcharge_lg.shape[0], self.__Pedcharge_lg.shape[0])) - upper = (np.power(self.__FFcharge_lg.mean(axis=1)[:min_events] - self.__Pedcharge_lg.mean(axis=1)[:min_events] * self.__coefCharge_FF_Ped - self.meanChargeLG.mean(), 2)).mean(axis=0) + upper = ( + np.power( + self.__FFcharge_lg.mean(axis=1)[:min_events] + - self.__Pedcharge_lg.mean(axis=1)[:min_events] * self.__coefCharge_FF_Ped + - self.meanChargeLG.mean(), + 2, + ) + ).mean(axis=0) lower = np.power(self.meanChargeLG.mean(), 2) return np.sqrt(upper / lower) @@ -506,6 +571,8 @@ def gainLG(self) -> float: Returns: float: The gain for low gain charge data. """ - return ((np.power(self.sigmaChargeLG, 2) - np.power(self.sigmaPedLG, 2) - np.power(self.BLG * self.meanChargeLG, 2)) - / (self.meanChargeLG * (1 + np.power(self.SPE_resolution, 2)))) - + return ( + np.power(self.sigmaChargeLG, 2) + - np.power(self.sigmaPedLG, 2) + - np.power(self.BLG * self.meanChargeLG, 2) + ) / (self.meanChargeLG * (1 + np.power(self.SPE_resolution, 2))) diff --git a/src/nectarchain/makers/calibration/gain/WhiteTargetSPEMakers.py b/src/nectarchain/makers/calibration/gain/WhiteTargetSPEMakers.py index 0e455b1a..be726219 100644 --- a/src/nectarchain/makers/calibration/gain/WhiteTargetSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/WhiteTargetSPEMakers.py @@ -1,15 +1,19 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers from .gainMakers import GainMaker - __all__ = ["FlatfieldMaker"] -class WhiteTargetSPEMaker(GainMaker) : - def __init__(self,*args,**kwargs) -> None: - super().__init__(*args,**kwargs) - def make(self) : - raise NotImplementedError("The computation of the white target calibration is not yet implemented, feel free to contribute !:)") \ No newline at end of file + +class WhiteTargetSPEMaker(GainMaker): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def make(self): + raise NotImplementedError( + "The computation of the white target calibration is not yet implemented, feel free to contribute !:)" + ) diff --git a/src/nectarchain/makers/calibration/gain/__init__.py b/src/nectarchain/makers/calibration/gain/__init__.py index a4100bfb..26159534 100644 --- a/src/nectarchain/makers/calibration/gain/__init__.py +++ b/src/nectarchain/makers/calibration/gain/__init__.py @@ -1,3 +1,4 @@ from .FlatFieldSPEMakers import * -#from .WhiteTargetSPEMakers import * -from .PhotoStatisticMakers import * \ No newline at end of file + +# from .WhiteTargetSPEMakers import * +from .PhotoStatisticMakers import * diff --git a/src/nectarchain/makers/calibration/gain/gainMakers.py b/src/nectarchain/makers/calibration/gain/gainMakers.py index b48e0de4..ea8c31af 100644 --- a/src/nectarchain/makers/calibration/gain/gainMakers.py +++ b/src/nectarchain/makers/calibration/gain/gainMakers.py @@ -1,17 +1,20 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers + +from copy import copy +import astropy.units as u import numpy as np -from copy import copy from astropy.table import Column -import astropy.units as u from ..core import CalibrationMaker __all__ = ["GainMaker"] + class GainMaker(CalibrationMaker): """ A class for gain calibration calculations on data. @@ -41,11 +44,35 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.__high_gain = np.empty((self.npixels), dtype=np.float64) self.__low_gain = np.empty((self.npixels), dtype=np.float64) - self._results.add_column(Column(data=self.__high_gain, name="high_gain", unit=u.dimensionless_unscaled)) - self._results.add_column(Column(np.empty((self.npixels, 2), dtype=np.float64), "high_gain_error", unit=u.dimensionless_unscaled)) - self._results.add_column(Column(data=self.__low_gain, name="low_gain", unit=u.dimensionless_unscaled)) - self._results.add_column(Column(np.empty((self.npixels, 2), dtype=np.float64), "low_gain_error", unit=u.dimensionless_unscaled)) - self._results.add_column(Column(np.zeros((self.npixels), dtype=bool), "is_valid", unit=u.dimensionless_unscaled)) + self._results.add_column( + Column( + data=self.__high_gain, name="high_gain", unit=u.dimensionless_unscaled + ) + ) + self._results.add_column( + Column( + np.empty((self.npixels, 2), dtype=np.float64), + "high_gain_error", + unit=u.dimensionless_unscaled, + ) + ) + self._results.add_column( + Column(data=self.__low_gain, name="low_gain", unit=u.dimensionless_unscaled) + ) + self._results.add_column( + Column( + np.empty((self.npixels, 2), dtype=np.float64), + "low_gain_error", + unit=u.dimensionless_unscaled, + ) + ) + self._results.add_column( + Column( + np.zeros((self.npixels), dtype=bool), + "is_valid", + unit=u.dimensionless_unscaled, + ) + ) @property def _high_gain(self): @@ -106,4 +133,3 @@ def low_gain(self): ndarray: A copy of the low gain values. """ return copy(self.__low_gain) - \ No newline at end of file diff --git a/src/nectarchain/makers/calibration/gain/parameters.py b/src/nectarchain/makers/calibration/gain/parameters.py index 96d6ddba..28d384a3 100644 --- a/src/nectarchain/makers/calibration/gain/parameters.py +++ b/src/nectarchain/makers/calibration/gain/parameters.py @@ -1,24 +1,27 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) import copy -import numpy as np import astropy.units as u +import numpy as np + +__all__ = ["Parameter", "Parameters"] + -__all__ = ["Parameter","Parameters"] - -class Parameter() : - def __init__(self, - name:str, - value, - min = np.nan, - max = np.nan, - error = np.nan, - unit = u.dimensionless_unscaled, - frozen : bool = False - )-> None: +class Parameter: + def __init__( + self, + name: str, + value, + min=np.nan, + max=np.nan, + error=np.nan, + unit=u.dimensionless_unscaled, + frozen: bool = False, + ) -> None: self.__name = name self.__value = value self.__error = error @@ -28,85 +31,115 @@ def __init__(self, self.__frozen = frozen @classmethod - def from_instance(cls,parameter): - return cls(parameter.name,parameter.value,parameter.min,parameter.max,parameter.unit,parameter.frozen) + def from_instance(cls, parameter): + return cls( + parameter.name, + parameter.value, + parameter.min, + parameter.max, + parameter.unit, + parameter.frozen, + ) def __str__(self): return f"name : {self.__name}, value : {self.__value}, error : {self.__error}, unit : {self.__unit}, min : {self.__min}, max : {self.__max},frozen : {self.__frozen}" - @property - def name(self) : return self.__name + def name(self): + return self.__name + @name.setter - def name(self,value: str) : self.__name = value + def name(self, value: str): + self.__name = value @property - def value(self) : return self.__value + def value(self): + return self.__value + @value.setter - def value(self,value) : self.__value = value + def value(self, value): + self.__value = value @property - def min(self): return self.__min + def min(self): + return self.__min + @min.setter - def min(self,value) : self.__min = value - + def min(self, value): + self.__min = value + @property - def max(self): return self.__max + def max(self): + return self.__max + @max.setter - def max(self,value) : self.__max = value + def max(self, value): + self.__max = value @property - def unit(self) : return self.__unit + def unit(self): + return self.__unit + @unit.setter - def unit(self,value:u.Unit) : self.__unit = value + def unit(self, value: u.Unit): + self.__unit = value @property - def error(self) : return self.__error + def error(self): + return self.__error + @error.setter - def error(self,value) : self.__error = value + def error(self, value): + self.__error = value @property - def frozen(self) : return self.__frozen + def frozen(self): + return self.__frozen + @frozen.setter - def frozen(self,value : bool) : self.__frozen = value + def frozen(self, value: bool): + self.__frozen = value + -class Parameters() : - def __init__(self,parameters_liste : list = []) -> None: +class Parameters: + def __init__(self, parameters_liste: list = []) -> None: self.__parameters = copy.deepcopy(parameters_liste) - - - def append(self,parameter : Parameter) -> None: + + def append(self, parameter: Parameter) -> None: self.__parameters.append(parameter) - def __getitem__(self,key:str) : - for parameter in self.__parameters : - if parameter.name == key : + def __getitem__(self, key: str): + for parameter in self.__parameters: + if parameter.name == key: return parameter return [] - + def __str__(self): - string="" - for parameter in self.__parameters : - string += str(parameter)+"\n" + string = "" + for parameter in self.__parameters: + string += str(parameter) + "\n" return string - @property - def parameters(self) : return self.__parameters + def parameters(self): + return self.__parameters @property - def size(self) : return len(self.__parameters) + def size(self): + return len(self.__parameters) @property - def parnames(self) : return [parameter.name for parameter in self.__parameters] + def parnames(self): + return [parameter.name for parameter in self.__parameters] @property - def parvalues(self) : return [parameter.value for parameter in self.__parameters] + def parvalues(self): + return [parameter.value for parameter in self.__parameters] @property - def unfrozen(self) : + def unfrozen(self): parameters = Parameters() - for parameter in self.__parameters : - if not(parameter.frozen) : - parameters.append(parameter) + for parameter in self.__parameters: + if not (parameter.frozen): + parameters.append(parameter) return parameters diff --git a/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py b/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py index 3b3a0c52..2bd60e2b 100644 --- a/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py +++ b/src/nectarchain/makers/calibration/gain/tests/test_FlatFieldSPEMakers.py @@ -1,59 +1,66 @@ -from nectarchain.makers.calibration.gain.FlatFieldSPEMakers import FlatFieldSPEMaker -from nectarchain.makers.calibration.gain.parameters import Parameter,Parameters -from nectarchain.makers.calibration.gain import FlatFieldSingleHHVSPEMaker,FlatFieldSingleHHVStdSPEMaker import astropy.units as u -from nectarchain.data.container import ChargesContainer import numpy as np - import pytest -class FlatFieldSPEMakerforTest(FlatFieldSPEMaker) : - def make() : +from nectarchain.data.container import ChargesContainer +from nectarchain.makers.calibration.gain import ( + FlatFieldSingleHHVSPEMaker, + FlatFieldSingleHHVStdSPEMaker, +) +from nectarchain.makers.calibration.gain.FlatFieldSPEMakers import FlatFieldSPEMaker +from nectarchain.makers.calibration.gain.parameters import Parameter, Parameters + + +class FlatFieldSPEMakerforTest(FlatFieldSPEMaker): + def make(): pass -def create_fake_chargeContainer() : - pixels_id = np.array([2,4,3,8,6,9,7,1,5,10]) + +def create_fake_chargeContainer(): + pixels_id = np.array([2, 4, 3, 8, 6, 9, 7, 1, 5, 10]) nevents = 40 npixels = 10 rng = np.random.default_rng() - charges_hg = rng.integers(low=0, high=1000, size= (nevents,npixels)) - charges_lg = rng.integers(low=0, high=1000, size= (nevents,npixels)) - peak_hg = rng.integers(low=0, high=60, size= (nevents,npixels)) - peak_lg = rng.integers(low=0, high=60, size= (nevents,npixels)) + charges_hg = rng.integers(low=0, high=1000, size=(nevents, npixels)) + charges_lg = rng.integers(low=0, high=1000, size=(nevents, npixels)) + peak_hg = rng.integers(low=0, high=60, size=(nevents, npixels)) + peak_lg = rng.integers(low=0, high=60, size=(nevents, npixels)) run_number = 1234 return ChargesContainer( - charges_hg = charges_hg , - charges_lg = charges_lg, - peak_hg = peak_hg, - peak_lg = peak_lg, - run_number = run_number, - pixels_id = pixels_id, - nevents = nevents, - npixels = npixels + charges_hg=charges_hg, + charges_lg=charges_lg, + peak_hg=peak_hg, + peak_lg=peak_lg, + run_number=run_number, + pixels_id=pixels_id, + nevents=nevents, + npixels=npixels, ) -class TestFlatFieldSPEMaker: +class TestFlatFieldSPEMaker: # Tests that the object can be initialized without errors def test_initialize_object(self): - pixels_id = [2,3,5] + pixels_id = [2, 3, 5] flat_field_spe_maker = FlatFieldSPEMakerforTest(pixels_id) assert isinstance(flat_field_spe_maker, FlatFieldSPEMakerforTest) # Tests that parameters can be read from a YAML file def test_read_parameters_from_yaml(self): - pixels_id = [2,3,5] + pixels_id = [2, 3, 5] flat_field_spe_maker = FlatFieldSPEMakerforTest(pixels_id) flat_field_spe_maker.read_param_from_yaml("parameters_signal.yaml") assert flat_field_spe_maker.parameters.size == 6 - assert isinstance(flat_field_spe_maker.parameters,Parameters) + assert isinstance(flat_field_spe_maker.parameters, Parameters) # Tests that parameters can be updated from a YAML file def test_update_parameters_from_yaml(self): - pixels_id = [2,3,5] + pixels_id = [2, 3, 5] flat_field_spe_maker = FlatFieldSPEMakerforTest(pixels_id) flat_field_spe_maker.read_param_from_yaml("parameters_signal.yaml") - flat_field_spe_maker.read_param_from_yaml("parameters_signalStd.yaml",only_update = True) + flat_field_spe_maker.read_param_from_yaml( + "parameters_signalStd.yaml", only_update=True + ) assert flat_field_spe_maker.parameters.parameters[-2].value == 0.697 # Tests that parameters can be updated from a fit @@ -61,14 +68,22 @@ def test_update_parameters_from_fit(self): pixels_id = [2] flat_field_spe_maker = FlatFieldSPEMakerforTest(pixels_id) flat_field_spe_maker.read_param_from_yaml("parameters_signal.yaml") - updated_parameters = flat_field_spe_maker._update_parameters(flat_field_spe_maker.parameters, charge=[1, 2, 3,4,5,6,7,8,9,10], counts=[1,3,9,5,3,5,6,3,2,1]) + updated_parameters = flat_field_spe_maker._update_parameters( + flat_field_spe_maker.parameters, + charge=[1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + counts=[1, 3, 9, 5, 3, 5, 6, 3, 2, 1], + ) # Tests that the table can be updated from parameters def test_update_table_from_parameters(self): - pixels_id = [2,3,5] + pixels_id = [2, 3, 5] flat_field_spe_maker = FlatFieldSPEMakerforTest(pixels_id) - flat_field_spe_maker._parameters.append(Parameter(name="param1", value=1, unit=u.dimensionless_unscaled)) - flat_field_spe_maker._parameters.append(Parameter(name="param2", value=2, unit=u.dimensionless_unscaled)) + flat_field_spe_maker._parameters.append( + Parameter(name="param1", value=1, unit=u.dimensionless_unscaled) + ) + flat_field_spe_maker._parameters.append( + Parameter(name="param2", value=2, unit=u.dimensionless_unscaled) + ) flat_field_spe_maker._update_table_from_parameters() @@ -79,12 +94,11 @@ def test_update_table_from_parameters(self): class TestFlatFieldSingleHHVSPEMaker: - # Tests that creating an instance of FlatFieldSingleHHVSPEMaker with valid input parameters is successful def test_create_instance_valid_input(self): charge = [1, 2, 3] counts = [10, 20, 30] - pixels_id = [2,3,5] + pixels_id = [2, 3, 5] maker = FlatFieldSingleHHVSPEMaker(charge, counts, pixels_id) assert isinstance(maker, FlatFieldSingleHHVSPEMaker) @@ -92,10 +106,10 @@ def test_create_instance_valid_input(self): def test_create_instance_invalid_input(self): charge = [1, 2, 3] counts = [10, 20] # Invalid input, counts and charge must have the same length - pixels_id = [2,3,5] + pixels_id = [2, 3, 5] with pytest.raises(Exception): - FlatFieldSingleHHVSPEMaker(charge, counts,pixels_id) + FlatFieldSingleHHVSPEMaker(charge, counts, pixels_id) # Tests that calling create_from_chargeContainer method with valid input parameters is successful def test_create_from_ChargeContainer_valid_input(self): @@ -103,24 +117,42 @@ def test_create_from_ChargeContainer_valid_input(self): maker = FlatFieldSingleHHVSPEMaker.create_from_chargesContainer(chargeContainer) assert isinstance(maker, FlatFieldSingleHHVSPEMaker) - def test_fill_results_table_from_dict(self): pass - def test_NG_Likelihood_Chi2(self):pass - def test_cost(self):pass - def test_make_fit_array_from_parameters(self):pass - def test_run_fit(self) : pass - def test_make(self):pass - def test_plot_single(self) : pass - def test_display(self) : pass + + def test_NG_Likelihood_Chi2(self): + pass + + def test_cost(self): + pass + + def test_make_fit_array_from_parameters(self): + pass + + def test_run_fit(self): + pass + + def test_make(self): + pass + + def test_plot_single(self): + pass + + def test_display(self): + pass + class TestFlatFieldSingleHHVStdSPEMaker: def test_create_instance(self): charge = [1, 2, 3] - counts = [10, 20,30] # Invalid input, counts and charge must have the same length - pixels_id = [2,3,5] - instance = FlatFieldSingleHHVStdSPEMaker(charge, counts,pixels_id) - assert isinstance(instance,FlatFieldSingleHHVStdSPEMaker) + counts = [ + 10, + 20, + 30, + ] # Invalid input, counts and charge must have the same length + pixels_id = [2, 3, 5] + instance = FlatFieldSingleHHVStdSPEMaker(charge, counts, pixels_id) + assert isinstance(instance, FlatFieldSingleHHVStdSPEMaker) class TestFlatFieldSingleNominalSPEMaker: diff --git a/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py b/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py index 1acbac89..16be40b0 100644 --- a/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py +++ b/src/nectarchain/makers/calibration/gain/tests/test_gainMakers.py @@ -1,17 +1,20 @@ - # Generated by CodiumAI -from nectarchain.makers.calibration.gain.gainMakers import GainMaker -import numpy as np from pathlib import Path + +import numpy as np from astropy.table import QTable -class GainMakerforTest(GainMaker) : +from nectarchain.makers.calibration.gain.gainMakers import GainMaker + + +class GainMakerforTest(GainMaker): _reduced_name = "test" - def make() : + + def make(): pass -class TestGainMaker: +class TestGainMaker: # Tests that an instance of GainMaker can be created with a list of pixel ids as input. def test_create_instance_with_pixel_ids(self): pixel_ids = [1, 2, 3, 4, 5] @@ -36,7 +39,7 @@ def test_set_and_get_low_gain_values(self): assert np.array_equal(gain_maker.low_gain, low_gain_values) # Tests that the results can be saved to a file. - def test_save_results_to_file(self, tmp_path = Path(f'/tmp/{np.random.rand()}')): + def test_save_results_to_file(self, tmp_path=Path(f"/tmp/{np.random.rand()}")): pixel_ids = [1, 2, 3, 4, 5] gain_maker = GainMakerforTest(pixel_ids) high_gain_values = np.array([0.5, 0.6, 0.7, 0.8, 0.9]) @@ -52,5 +55,6 @@ def test_get_copy_of_result_table(self): gain_maker = GainMakerforTest(pixel_ids) result_table_copy = gain_maker.results assert isinstance(result_table_copy, QTable) - assert np.array_equal(result_table_copy[GainMakerforTest.PIXELS_ID_COLUMN], np.array(pixel_ids)) - + assert np.array_equal( + result_table_copy[GainMakerforTest.PIXELS_ID_COLUMN], np.array(pixel_ids) + ) diff --git a/src/nectarchain/makers/calibration/gain/utils/__init__.py b/src/nectarchain/makers/calibration/gain/utils/__init__.py index b4996a25..6f3aa3a4 100644 --- a/src/nectarchain/makers/calibration/gain/utils/__init__.py +++ b/src/nectarchain/makers/calibration/gain/utils/__init__.py @@ -1,2 +1,2 @@ -from .error import * -from .utils import * \ No newline at end of file +from .error import * +from .utils import * diff --git a/src/nectarchain/makers/calibration/gain/utils/error.py b/src/nectarchain/makers/calibration/gain/utils/error.py index a136e7cc..9e1c9e75 100644 --- a/src/nectarchain/makers/calibration/gain/utils/error.py +++ b/src/nectarchain/makers/calibration/gain/utils/error.py @@ -1,23 +1,32 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers + +class DifferentPixelsID(Exception): + def __init__(self, message): + self.__message = message -class DifferentPixelsID(Exception) : - def __init__(self,message) : - self.__message = message @property - def message(self) : return self.__message + def message(self): + return self.__message + + +class PedestalValueError(ValueError): + def __init__(self, message): + self.__message = message -class PedestalValueError(ValueError) : - def __init__(self,message) : - self.__message = message @property - def message(self) : return self.__message + def message(self): + return self.__message + + +class MeanValueError(ValueError): + def __init__(self, message): + self.__message = message -class MeanValueError(ValueError) : - def __init__(self,message) : - self.__message = message @property - def message(self) : return self.__message + def message(self): + return self.__message diff --git a/src/nectarchain/makers/calibration/gain/utils/utils.py b/src/nectarchain/makers/calibration/gain/utils/utils.py index 54825935..d5577bac 100644 --- a/src/nectarchain/makers/calibration/gain/utils/utils.py +++ b/src/nectarchain/makers/calibration/gain/utils/utils.py @@ -5,36 +5,39 @@ from iminuit import Minuit from scipy import interpolate, signal from scipy.special import gammainc -from scipy.stats import norm,chi2 +from scipy.stats import chi2, norm -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers -#__all__ = ['UtilsMinuit','Multiprocessing'] +# __all__ = ['UtilsMinuit','Multiprocessing'] from ..parameters import Parameters -class Multiprocessing() : + +class Multiprocessing: @staticmethod def custom_error_callback(error): - log.error(f'Got an error: {error}') - log.error(error,exc_info=True) + log.error(f"Got an error: {error}") + log.error(error, exc_info=True) + -class Statistics() : +class Statistics: @staticmethod - def chi2_pvalue(ndof : int, likelihood : float) : - return 1 - chi2(df = ndof).cdf(likelihood) + def chi2_pvalue(ndof: int, likelihood: float): + return 1 - chi2(df=ndof).cdf(likelihood) -class UtilsMinuit() : + +class UtilsMinuit: @staticmethod - def make_minuit_par_kwargs(parameters : Parameters): + def make_minuit_par_kwargs(parameters: Parameters): """Create *Parameter Keyword Arguments* for the `Minuit` constructor. updated for Minuit >2.0 """ names = parameters.parnames - kwargs = {"names": names,"values" : {}} + kwargs = {"names": names, "values": {}} for parameter in parameters.parameters: kwargs["values"][parameter.name] = parameter.value @@ -43,35 +46,38 @@ def make_minuit_par_kwargs(parameters : Parameters): error = 0.1 if np.isnan(parameter.error) else parameter.error kwargs[f"limit_{parameter.name}"] = (min_, max_) kwargs[f"error_{parameter.name}"] = error - if parameter.frozen : + if parameter.frozen: kwargs[f"fix_{parameter.name}"] = True return kwargs - + @staticmethod - def set_minuit_parameters_limits_and_errors(m : Minuit,parameters : dict) : + def set_minuit_parameters_limits_and_errors(m: Minuit, parameters: dict): """function to set minuit parameter limits and errors with Minuit >2.0 Args: m (Minuit): a Minuit instance parameters (dict): dict containing parameters names, limits errors and values """ - for name in parameters["names"] : + for name in parameters["names"]: m.limits[name] = parameters[f"limit_{name}"] m.errors[name] = parameters[f"error_{name}"] - if parameters.get(f"fix_{name}",False) : + if parameters.get(f"fix_{name}", False): m.fixed[name] = True # Usefull fucntions for the fit def gaussian(x, mu, sig): - #return (1./(sig*np.sqrt(2*math.pi)))*np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.))) - return norm.pdf(x,loc = mu,scale = sig) + # return (1./(sig*np.sqrt(2*math.pi)))*np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.))) + return norm.pdf(x, loc=mu, scale=sig) -def weight_gaussian(x,N, mu, sig) : + +def weight_gaussian(x, N, mu, sig): return N * gaussian(x, mu, sig) -def doubleGauss(x,sig1,mu2,sig2,p): - return p *2 *gaussian(x, 0, sig1) + (1-p) * gaussian(x, mu2, sig2) + +def doubleGauss(x, sig1, mu2, sig2, p): + return p * 2 * gaussian(x, 0, sig1) + (1 - p) * gaussian(x, mu2, sig2) + def PMax(r): """p_{max} in equation 6 in Caroff et al. (2019) @@ -82,13 +88,14 @@ def PMax(r): Returns: float : p_{max} """ - if r > np.sqrt((np.pi -2 )/ 2) : - pmax = np.pi/(2 * (r**2 + 1)) - else : - pmax = np.pi*r**2/(np.pi*r**2 + np.pi - 2*r**2 - 2) + if r > np.sqrt((np.pi - 2) / 2): + pmax = np.pi / (2 * (r**2 + 1)) + else: + pmax = np.pi * r**2 / (np.pi * r**2 + np.pi - 2 * r**2 - 2) return pmax -def ax(p,res): + +def ax(p, res): """a in equation 4 in Caroff et al. (2019) Args: @@ -98,9 +105,10 @@ def ax(p,res): Returns: float : a """ - return ((2/np.pi)*p**2-p/(res**2+1)) + return (2 / np.pi) * p**2 - p / (res**2 + 1) + -def bx(p,mu2): +def bx(p, mu2): """b in equation 4 in Caroff et al. (2019) Args: @@ -110,11 +118,12 @@ def bx(p,mu2): Returns: float : b """ - return (np.sqrt(2/np.pi)*2*p*(1-p)*mu2) + return np.sqrt(2 / np.pi) * 2 * p * (1 - p) * mu2 -def cx(sig2,mu2,res,p): + +def cx(sig2, mu2, res, p): """c in equation 4 in Caroff et al. (2019) - Note : There is a typo in the article 1-p**2 -> (1-p)**2 + Note : There is a typo in the article 1-p**2 -> (1-p)**2 Args: sig2 (float): width of the high charge Gaussian @@ -125,9 +134,10 @@ def cx(sig2,mu2,res,p): Returns: float : c """ - return (1-p)**2*mu2**2 - (1-p)*(sig2**2+mu2**2)/(res**2+1) + return (1 - p) ** 2 * mu2**2 - (1 - p) * (sig2**2 + mu2**2) / (res**2 + 1) + -def delta(p,res,sig2,mu2): +def delta(p, res, sig2, mu2): """well known delta in 2nd order polynom Args: @@ -139,9 +149,10 @@ def delta(p,res,sig2,mu2): Returns: float : b**2 - 4*a*c """ - return bx(p,mu2)*bx(p,mu2) - 4*ax(p,res)*cx(sig2,mu2,res,p) + return bx(p, mu2) * bx(p, mu2) - 4 * ax(p, res) * cx(sig2, mu2, res, p) + -def ParamU(p,r): +def ParamU(p, r): """d in equation 6 in Caroff et al. (2019) Args: @@ -151,9 +162,12 @@ def ParamU(p,r): Returns: float : d """ - return ((8*(1-p)**2*p**2)/np.pi - 4*(2*p**2/np.pi - p/(r**2+1))*((1-p)**2-(1-p)/(r**2+1))) + return (8 * (1 - p) ** 2 * p**2) / np.pi - 4 * ( + 2 * p**2 / np.pi - p / (r**2 + 1) + ) * ((1 - p) ** 2 - (1 - p) / (r**2 + 1)) -def ParamS(p,r): + +def ParamS(p, r): """e in equation 6 in Caroff et al. (2019) Args: @@ -163,10 +177,11 @@ def ParamS(p,r): Returns: float : e """ - e = (4*(2*p**2/np.pi - p/(r**2+1))*(1-p))/(r**2+1) + e = (4 * (2 * p**2 / np.pi - p / (r**2 + 1)) * (1 - p)) / (r**2 + 1) return e -def SigMin(p,res,mu2): + +def SigMin(p, res, mu2): """sigma_{high,min} in equation 6 in Caroff et al. (2019) Args: @@ -177,9 +192,12 @@ def SigMin(p,res,mu2): Returns: float : sigma_{high,min} """ - return mu2*np.sqrt((-ParamU(p,res)+(bx(p,mu2)**2/mu2**2))/(ParamS(p,res))) + return mu2 * np.sqrt( + (-ParamU(p, res) + (bx(p, mu2) ** 2 / mu2**2)) / (ParamS(p, res)) + ) -def SigMax(p,res,mu2): + +def SigMax(p, res, mu2): """sigma_{high,min} in equation 6 in Caroff et al. (2019) Args: @@ -190,15 +208,16 @@ def SigMax(p,res,mu2): Returns: float : sigma_{high,min} """ - temp = (-ParamU(p,res))/(ParamS(p,res)) - if temp < 0 : + temp = (-ParamU(p, res)) / (ParamS(p, res)) + if temp < 0: err = ValueError("-d/e must be < 0") - log.error(err,exc_info=True) + log.error(err, exc_info=True) raise err - else : - return mu2*np.sqrt(temp) + else: + return mu2 * np.sqrt(temp) -def sigma1(p,res,sig2,mu2): + +def sigma1(p, res, sig2, mu2): """sigma_{low} in equation 5 in Caroff et al. (2019) Args: @@ -210,9 +229,10 @@ def sigma1(p,res,sig2,mu2): Returns: float : sigma_{low} """ - return (-bx(p,mu2)+np.sqrt(delta(p,res,sig2,mu2)))/(2*ax(p,res)) + return (-bx(p, mu2) + np.sqrt(delta(p, res, sig2, mu2))) / (2 * ax(p, res)) + -def sigma2(n,p,res,mu2): +def sigma2(n, p, res, mu2): """sigma_{high} in equation 7 in Caroff et al. (2019) Args: @@ -224,39 +244,56 @@ def sigma2(n,p,res,mu2): Returns: float : sigma_{high} """ - if ((-ParamU(p,res)+(bx(p,mu2)**2/mu2**2))/(ParamS(p,res)) > 0): - return SigMin(p,res,mu2)+n*(SigMax(p,res,mu2)-SigMin(p,res,mu2)) + if (-ParamU(p, res) + (bx(p, mu2) ** 2 / mu2**2)) / (ParamS(p, res)) > 0: + return SigMin(p, res, mu2) + n * (SigMax(p, res, mu2) - SigMin(p, res, mu2)) else: - return n*SigMax(p,res,mu2) + return n * SigMax(p, res, mu2) # The real final model callign all the above for luminosity (lum) + PED, wil return probability of number of Spe -def MPE2(x,pp,res,mu2,n,muped,sigped,lum,**kwargs): - log.debug(f"pp = {pp}, res = {res}, mu2 = {mu2}, n = {n}, muped = {muped}, sigped = {sigped}, lum = {lum}") + + +def MPE2(x, pp, res, mu2, n, muped, sigped, lum, **kwargs): + log.debug( + f"pp = {pp}, res = {res}, mu2 = {mu2}, n = {n}, muped = {muped}, sigped = {sigped}, lum = {lum}" + ) f = 0 - ntotalPE = kwargs.get("ntotalPE",0) - if ntotalPE == 0 : - #about 1sec + ntotalPE = kwargs.get("ntotalPE", 0) + if ntotalPE == 0: + # about 1sec for i in range(1000): - if (gammainc(i+1,lum) < 1e-5): + if gammainc(i + 1, lum) < 1e-5: ntotalPE = i break - #print(ntotalPE) - #about 8 sec, 1 sec by nPEPDF call - #for i in range(ntotalPE): + # print(ntotalPE) + # about 8 sec, 1 sec by nPEPDF call + # for i in range(ntotalPE): # f = f + ((lum**i)/math.factorial(i)) * np.exp(-lum) * nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) - f = np.sum([(lum**i)/math.factorial(i) * np.exp(-lum) * nPEPDF(x,pp,res,mu2,n,muped,sigped,i,int(mu2*ntotalPE+10*mu2)) for i in range(ntotalPE)],axis = 0) # 10 % faster + f = np.sum( + [ + (lum**i) + / math.factorial(i) + * np.exp(-lum) + * nPEPDF( + x, pp, res, mu2, n, muped, sigped, i, int(mu2 * ntotalPE + 10 * mu2) + ) + for i in range(ntotalPE) + ], + axis=0, + ) # 10 % faster return f - + + # Fnal model shape/function (for one SPE) -def doubleGaussConstrained(x,pp,res,mu2,n): - p = pp*PMax(res) - sig2 = sigma2(n,p,res,mu2) - sig1 = sigma1(p,res,sig2,mu2) - return doubleGauss(x,sig1,mu2,sig2,p) +def doubleGaussConstrained(x, pp, res, mu2, n): + p = pp * PMax(res) + sig2 = sigma2(n, p, res, mu2) + sig1 = sigma1(p, res, sig2, mu2) + return doubleGauss(x, sig1, mu2, sig2, p) + # Get the gain from the parameters model -def Gain(pp,res,mu2,n): +def Gain(pp, res, mu2, n): """analytic gain computatuon Args: @@ -268,32 +305,35 @@ def Gain(pp,res,mu2,n): Returns: float : gain """ - p = pp*PMax(res) - sig2 = sigma2(n,p,res,mu2) - return (1-p)*mu2 + 2*p*sigma1(p,res,sig2,mu2)/np.sqrt(2*np.pi) + p = pp * PMax(res) + sig2 = sigma2(n, p, res, mu2) + return (1 - p) * mu2 + 2 * p * sigma1(p, res, sig2, mu2) / np.sqrt(2 * np.pi) + -def nPEPDF(x,pp,res,mu2,n,muped,sigped,nph,size_charge): - allrange = np.linspace(-1 * size_charge,size_charge,size_charge*2) +def nPEPDF(x, pp, res, mu2, n, muped, sigped, nph, size_charge): + allrange = np.linspace(-1 * size_charge, size_charge, size_charge * 2) spe = [] - #about 2 sec this is the main pb - #for i in range(len(allrange)): + # about 2 sec this is the main pb + # for i in range(len(allrange)): # if (allrange[i]>=0): # spe.append(doubleGaussConstrained(allrange[i],pp,res,mu2,n)) # else: # spe.append(0) - - spe = doubleGaussConstrained(allrange,pp,res,mu2,n) * (allrange>=0 * np.ones(allrange.shape)) #100 times faster + + spe = doubleGaussConstrained(allrange, pp, res, mu2, n) * ( + allrange >= 0 * np.ones(allrange.shape) + ) # 100 times faster # ~ plt.plot(allrange,spe) - #npe = semi_gaussian(allrange, muped, sigped) - npe = gaussian(allrange, 0, sigped) + # npe = semi_gaussian(allrange, muped, sigped) + npe = gaussian(allrange, 0, sigped) # ~ plt.plot(allrange,npe) # ~ plt.show() for i in range(nph): - #npe = np.convolve(npe,spe,"same") - npe = signal.fftconvolve(npe,spe,"same") + # npe = np.convolve(npe,spe,"same") + npe = signal.fftconvolve(npe, spe, "same") # ~ plt.plot(allrange,npe) # ~ plt.show() - fff = interpolate.UnivariateSpline(allrange,npe,ext=1,k=3,s=0) - norm = np.trapz(fff(allrange),allrange) - return fff(x-muped)/norm \ No newline at end of file + fff = interpolate.UnivariateSpline(allrange, npe, ext=1, k=3, s=0) + norm = np.trapz(fff(allrange), allrange) + return fff(x - muped) / norm diff --git a/src/nectarchain/makers/calibration/pedestalMakers.py b/src/nectarchain/makers/calibration/pedestalMakers.py index 5936d76c..15a8af1b 100644 --- a/src/nectarchain/makers/calibration/pedestalMakers.py +++ b/src/nectarchain/makers/calibration/pedestalMakers.py @@ -1,16 +1,19 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers from .core import CalibrationMaker __all__ = ["PedestalMaker"] -class PedestalMaker(CalibrationMaker) : - def __init__(self,*args,**kwargs) -> None: - super().__init__(*args,**kwargs) - def make(self) : - raise NotImplementedError("The computation of the pedestal calibration is not yet implemented, feel free to contribute !:)") - \ No newline at end of file +class PedestalMaker(CalibrationMaker): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def make(self): + raise NotImplementedError( + "The computation of the pedestal calibration is not yet implemented, feel free to contribute !:)" + ) diff --git a/src/nectarchain/makers/calibration/tests/test_core.py b/src/nectarchain/makers/calibration/tests/test_core.py index a9e8a01e..d2a27fab 100644 --- a/src/nectarchain/makers/calibration/tests/test_core.py +++ b/src/nectarchain/makers/calibration/tests/test_core.py @@ -1,27 +1,33 @@ -from nectarchain.makers.calibration.core import CalibrationMaker -import numpy as np from pathlib import Path +import numpy as np import pytest -class CalibrationMakerforTest(CalibrationMaker) : +from nectarchain.makers.calibration.core import CalibrationMaker + + +class CalibrationMakerforTest(CalibrationMaker): _reduced_name = "test" - def make() : + + def make(): pass class TestflatfieldMaker: - # Tests that the constructor initializes the object with the correct attributes and metadata when valid input is provided def test_constructor_with_valid_input(self): pixels_id = [1, 2, 3] calibration_maker = CalibrationMakerforTest(pixels_id) - - assert np.equal(calibration_maker._pixels_id,pixels_id).all() - assert np.equal(calibration_maker._results[calibration_maker.PIXELS_ID_COLUMN],np.array(pixels_id)).all() - assert calibration_maker._results.meta[calibration_maker.NP_PIXELS] == len(pixels_id) - assert isinstance(calibration_maker._results.meta['comments'],str) + assert np.equal(calibration_maker._pixels_id, pixels_id).all() + assert np.equal( + calibration_maker._results[calibration_maker.PIXELS_ID_COLUMN], + np.array(pixels_id), + ).all() + assert calibration_maker._results.meta[calibration_maker.NP_PIXELS] == len( + pixels_id + ) + assert isinstance(calibration_maker._results.meta["comments"], str) # Tests that the constructor raises an error when a non-iterable pixels_id is provided def test_constructor_with_non_iterable_pixels_id(self): @@ -30,14 +36,14 @@ def test_constructor_with_non_iterable_pixels_id(self): CalibrationMakerforTest(pixels_id) # Tests that saving the results to an existing file with overwrite=False raises an error - def test_save_to_existing_file_with_overwrite_false(self, tmp_path = Path('/tmp')): + def test_save_to_existing_file_with_overwrite_false(self, tmp_path=Path("/tmp")): pixels_id = [1, 2, 3] calibration_maker = CalibrationMakerforTest(pixels_id) # Create a temporary file file_path = tmp_path / "results_Calibration.ecsv" file_path.touch() - + with pytest.raises(FileExistsError): calibration_maker.save(file_path, overwrite=False) @@ -45,9 +51,8 @@ def test_save_to_existing_file_with_overwrite_false(self, tmp_path = Path('/tmp' def test_change_pixels_id_attribute(self): pixels_id = [1, 2, 3] calibration_maker = CalibrationMakerforTest(pixels_id) - + new_pixels_id = [4, 5, 6] calibration_maker._pixels_id = np.array(new_pixels_id) - - assert np.equal(calibration_maker._pixels_id,new_pixels_id).all() + assert np.equal(calibration_maker._pixels_id, new_pixels_id).all() diff --git a/src/nectarchain/makers/chargesMakers.py b/src/nectarchain/makers/chargesMakers.py index 91ebef31..2742781c 100644 --- a/src/nectarchain/makers/chargesMakers.py +++ b/src/nectarchain/makers/chargesMakers.py @@ -1,59 +1,58 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers +import time from argparse import ArgumentError + import numpy as np import numpy.ma as ma -import time - +from ctapipe.containers import EventType +from ctapipe.image.extractor import ( + BaselineSubtractedNeighborPeakWindowSum, + FixedWindowSum, + FullWaveformSum, + GlobalPeakWindowSum, + LocalPeakWindowSum, + NeighborPeakWindowSum, + SlidingWindowMaxSum, + TwoPassWindowSum, +) +from ctapipe.instrument.subarray import SubarrayDescription from ctapipe_io_nectarcam import constants from ctapipe_io_nectarcam.containers import NectarCAMDataContainer -from ctapipe.instrument.subarray import SubarrayDescription -from ctapipe.containers import EventType -from ctapipe.image.extractor import (FullWaveformSum, - FixedWindowSum, - GlobalPeakWindowSum, - LocalPeakWindowSum, - SlidingWindowMaxSum, - NeighborPeakWindowSum, - BaselineSubtractedNeighborPeakWindowSum, - TwoPassWindowSum) - - -from numba import guvectorize, float64, int64, bool_ - -from ..data.container import WaveformsContainer,ChargesContainer +from numba import bool_, float64, guvectorize, int64 +from ..data.container import ChargesContainer, WaveformsContainer from .core import ArrayDataMaker from .extractor.utils import CtapipeExtractor +__all__ = ["ChargesMaker"] -__all__ = ['ChargesMaker'] - -list_ctapipe_charge_extractor = ["FullWaveformSum", - "FixedWindowSum", - "GlobalPeakWindowSum", - "LocalPeakWindowSum", - "SlidingWindowMaxSum", - "NeighborPeakWindowSum", - "BaselineSubtractedNeighborPeakWindowSum", - "TwoPassWindowSum"] - +list_ctapipe_charge_extractor = [ + "FullWaveformSum", + "FixedWindowSum", + "GlobalPeakWindowSum", + "LocalPeakWindowSum", + "SlidingWindowMaxSum", + "NeighborPeakWindowSum", + "BaselineSubtractedNeighborPeakWindowSum", + "TwoPassWindowSum", +] -list_nectarchain_charge_extractor = ['gradient_extractor'] +list_nectarchain_charge_extractor = ["gradient_extractor"] @guvectorize( -[ - (int64[:], float64[:], bool_, bool_[:], int64[:]), -], - -"(s),(n),()->(n),(n)", -nopython=True, -cache=True, + [ + (int64[:], float64[:], bool_, bool_[:], int64[:]), + ], + "(s),(n),()->(n),(n)", + nopython=True, + cache=True, ) def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): """compute histogram of charge with numba @@ -65,41 +64,47 @@ def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): _mask (np.ndarray(pixels,nbins)): mask hist_ma_data (np.ndarray(pixels,nbins)): histogram """ - #print(f"charge.shape = {charge.shape[0]}") - #print(f"_mask.shape = {_mask.shape[0]}") - #print(f"_mask[0] = {_mask[0]}") - #print(f"hist_ma_data[0] = {hist_ma_data[0]}") - #print(f"mask_broken_pix = {mask_broken_pix}") - - if not(mask_broken_pix) : - #print("this pixel is not broken, let's continue computation") - hist,_charge = np.histogram(charge,bins=np.arange(np.uint16(np.min(charge)) - 1, np.uint16(np.max(charge)) + 2,1)) - #print(f"hist.shape[0] = {hist.shape[0]}") - #print(f"charge.shape[0] = {_charge.shape[0]}") - charge_edges = np.array([np.mean(_charge[i:i+2]) for i in range(_charge.shape[0]-1)]) - #print(f"charge_edges.shape[0] = {charge_edges.shape[0]}") + # print(f"charge.shape = {charge.shape[0]}") + # print(f"_mask.shape = {_mask.shape[0]}") + # print(f"_mask[0] = {_mask[0]}") + # print(f"hist_ma_data[0] = {hist_ma_data[0]}") + # print(f"mask_broken_pix = {mask_broken_pix}") + + if not (mask_broken_pix): + # print("this pixel is not broken, let's continue computation") + hist, _charge = np.histogram( + charge, + bins=np.arange( + np.uint16(np.min(charge)) - 1, np.uint16(np.max(charge)) + 2, 1 + ), + ) + # print(f"hist.shape[0] = {hist.shape[0]}") + # print(f"charge.shape[0] = {_charge.shape[0]}") + charge_edges = np.array( + [np.mean(_charge[i : i + 2]) for i in range(_charge.shape[0] - 1)] + ) + # print(f"charge_edges.shape[0] = {charge_edges.shape[0]}") mask = (all_range >= charge_edges[0]) * (all_range <= charge_edges[-1]) - #print(f"all_range = {int(all_range[0])}-{int(all_range[-1])}") - #print(f"charge_edges[0] = {int(charge_edges[0])}") - #print(f"charge_edges[-1] = {int(charge_edges[-1])}") - #print(f"mask[0] = {mask[0]}") - #print(f"mask[-1] = {mask[-1]}") - - #MASK THE DATA - #print(f"mask.shape = {mask.shape[0]}") + # print(f"all_range = {int(all_range[0])}-{int(all_range[-1])}") + # print(f"charge_edges[0] = {int(charge_edges[0])}") + # print(f"charge_edges[-1] = {int(charge_edges[-1])}") + # print(f"mask[0] = {mask[0]}") + # print(f"mask[-1] = {mask[-1]}") + + # MASK THE DATA + # print(f"mask.shape = {mask.shape[0]}") _mask[:] = ~mask - #print(f"_mask[0] = {_mask[0]}") - #print(f"_mask[-1] = {_mask[-1]}") - #FILL THE DATA + # print(f"_mask[0] = {_mask[0]}") + # print(f"_mask[-1] = {_mask[-1]}") + # FILL THE DATA hist_ma_data[mask] = hist - #print("work done") - else : - #print("this pixel is broken, skipped") + # print("work done") + else: + # print("this pixel is broken, skipped") pass - -class ChargesMaker(ArrayDataMaker) : +class ChargesMaker(ArrayDataMaker): """ The `ChargesMaker` class is a subclass of `ArrayDataMaker` and is responsible for computing charges and peaks from waveforms. It provides methods for initializing the class, making charges and peaks for events, creating output containers, sorting charges containers, selecting charges for specific pixels, computing histograms of charges, and more. @@ -151,8 +156,16 @@ class ChargesMaker(ArrayDataMaker) : - __peak_hg: Dictionary to store the peaks in the high-gain channel for each trigger type. - __peak_lg: Dictionary to store the peaks in the low-gain channel for each trigger type. """ -#constructors - def __init__(self,run_number : int,max_events : int = None,run_file:str = None,*args,**kwargs): + + # constructors + def __init__( + self, + run_number: int, + max_events: int = None, + run_file: str = None, + *args, + **kwargs, + ): """construtor Args: @@ -161,12 +174,12 @@ def __init__(self,run_number : int,max_events : int = None,run_file:str = None,* nevents (int, optional) : number of events in run if known (parameter used to save computing time) run_file (optional) : if provided, will load this run file """ - super().__init__(run_number,max_events,run_file,*args,**kwargs) + super().__init__(run_number, max_events, run_file, *args, **kwargs) self.__charges_hg = {} self.__charges_lg = {} self.__peak_hg = {} self.__peak_lg = {} - + def _init_trigger_type(self, trigger_type: EventType, **kwargs): """ Initializes the ChargesMaker based on the trigger type. @@ -186,8 +199,15 @@ def _init_trigger_type(self, trigger_type: EventType, **kwargs): self.__peak_hg[f"{name}"] = [] self.__peak_lg[f"{name}"] = [] - - def make(self, n_events=np.inf, trigger_type: list = None, restart_from_beginning: bool = False, method: str = "FullWaveformSum", *args, **kwargs): + def make( + self, + n_events=np.inf, + trigger_type: list = None, + restart_from_beginning: bool = False, + method: str = "FullWaveformSum", + *args, + **kwargs, + ): """ Makes charges based on the specified arguments. @@ -203,54 +223,72 @@ def make(self, n_events=np.inf, trigger_type: list = None, restart_from_beginnin The result of the parent class's make method. """ kwargs["method"] = method - return super().make(n_events=n_events, trigger_type=trigger_type, restart_from_beginning=restart_from_beginning, *args, **kwargs) - def _make_event(self, - event : NectarCAMDataContainer, - trigger : EventType, - method: str = "FullWaveformSum", - *args, - **kwargs - ) : + return super().make( + n_events=n_events, + trigger_type=trigger_type, + restart_from_beginning=restart_from_beginning, + *args, + **kwargs, + ) + + def _make_event( + self, + event: NectarCAMDataContainer, + trigger: EventType, + method: str = "FullWaveformSum", + *args, + **kwargs, + ): """ Make charges and peaks for a single event. - + Args: event (NectarCAMDataContainer): The event container that contains the data for a single event. trigger (EventType): The type of trigger for the event. method (str, optional): The method to use for computing charges and peaks. Defaults to "FullWaveformSum". *args: Additional positional arguments. **kwargs: Additional keyword arguments. - + Returns: None - + Summary: This method extracts waveforms from the event, computes broken pixels, and then uses an image extractor to calculate charges and peaks for the high-gain and low-gain channels. The results are stored in dictionaries for each trigger type. ``` """ - wfs_hg_tmp,wfs_lg_tmp = super()._make_event(event = event, - trigger = trigger, - return_wfs = True, - *args,**kwargs) + wfs_hg_tmp, wfs_lg_tmp = super()._make_event( + event=event, trigger=trigger, return_wfs=True, *args, **kwargs + ) name = __class__._get_name_trigger(trigger) - broken_pixels_hg,broken_pixels_lg = __class__._compute_broken_pixels_event(event,self._pixels_id) - self._broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) - self._broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) - + broken_pixels_hg, broken_pixels_lg = __class__._compute_broken_pixels_event( + event, self._pixels_id + ) + self._broken_pixels_hg[f"{name}"].append(broken_pixels_hg.tolist()) + self._broken_pixels_lg[f"{name}"].append(broken_pixels_lg.tolist()) - imageExtractor = __class__._get_imageExtractor(method,self._reader.subarray,**kwargs) + imageExtractor = __class__._get_imageExtractor( + method, self._reader.subarray, **kwargs + ) - __image = CtapipeExtractor.get_image_peak_time(imageExtractor(wfs_hg_tmp,__class__.TEL_ID,constants.HIGH_GAIN,broken_pixels_hg)) + __image = CtapipeExtractor.get_image_peak_time( + imageExtractor( + wfs_hg_tmp, __class__.TEL_ID, constants.HIGH_GAIN, broken_pixels_hg + ) + ) self.__charges_hg[f"{name}"].append(__image[0].tolist()) self.__peak_hg[f"{name}"].append(__image[1].tolist()) - __image = CtapipeExtractor.get_image_peak_time(imageExtractor(wfs_lg_tmp,__class__.TEL_ID,constants.LOW_GAIN,broken_pixels_lg)) + __image = CtapipeExtractor.get_image_peak_time( + imageExtractor( + wfs_lg_tmp, __class__.TEL_ID, constants.LOW_GAIN, broken_pixels_lg + ) + ) self.__charges_lg[f"{name}"].append(__image[0].tolist()) self.__peak_lg[f"{name}"].append(__image[1].tolist()) @staticmethod - def _get_imageExtractor(method : str,subarray : SubarrayDescription,**kwargs) : + def _get_imageExtractor(method: str, subarray: SubarrayDescription, **kwargs): """ Create an instance of a charge extraction method based on the provided method name and subarray description. Args: @@ -260,19 +298,30 @@ def _get_imageExtractor(method : str,subarray : SubarrayDescription,**kwargs) : Returns: imageExtractor: An instance of the charge extraction method specified by `method` with the provided subarray description and keyword arguments. """ - if not(method in list_ctapipe_charge_extractor or method in list_nectarchain_charge_extractor) : + if not ( + method in list_ctapipe_charge_extractor + or method in list_nectarchain_charge_extractor + ): raise ArgumentError(f"method must be in {list_ctapipe_charge_extractor}") extractor_kwargs = {} - for key in eval(method).class_own_traits().keys() : - if key in kwargs.keys() : + for key in eval(method).class_own_traits().keys(): + if key in kwargs.keys(): extractor_kwargs[key] = kwargs[key] - if "apply_integration_correction" in eval(method).class_own_traits().keys() : #to change the default behavior of ctapipe extractor - extractor_kwargs["apply_integration_correction"] = kwargs.get("apply_integration_correction",False) - log.debug(f"Extracting charges with method {method} and extractor_kwargs {extractor_kwargs}") - imageExtractor = eval(method)(subarray,**extractor_kwargs) + if ( + "apply_integration_correction" in eval(method).class_own_traits().keys() + ): # to change the default behavior of ctapipe extractor + extractor_kwargs["apply_integration_correction"] = kwargs.get( + "apply_integration_correction", False + ) + log.debug( + f"Extracting charges with method {method} and extractor_kwargs {extractor_kwargs}" + ) + imageExtractor = eval(method)(subarray, **extractor_kwargs) return imageExtractor - def _make_output_container(self,trigger_type : EventType, method : str,*args,**kwargs) : + def _make_output_container( + self, trigger_type: EventType, method: str, *args, **kwargs + ): """ Create an output container for the specified trigger type and method. Args: @@ -284,34 +333,34 @@ def _make_output_container(self,trigger_type : EventType, method : str,*args,** list: A list of ChargesContainer objects. """ output = [] - for trigger in trigger_type : + for trigger in trigger_type: chargesContainer = ChargesContainer( - run_number = self._run_number, - npixels = self._npixels, - camera = self.CAMERA_NAME, - pixels_id = self._pixels_id, - method = method, - nevents = self.nevents(trigger), - charges_hg = self.charges_hg(trigger), - charges_lg = self.charges_lg(trigger), - peak_hg = self.peak_hg(trigger), - peak_lg = self.peak_lg(trigger), - broken_pixels_hg = self.broken_pixels_hg(trigger), - broken_pixels_lg = self.broken_pixels_lg(trigger), - ucts_timestamp = self.ucts_timestamp(trigger), - ucts_busy_counter = self.ucts_busy_counter(trigger), - ucts_event_counter = self.ucts_event_counter(trigger), - event_type = self.event_type(trigger), - event_id = self.event_id(trigger), - trig_pattern_all = self.trig_pattern_all(trigger), - trig_pattern = self.trig_pattern(trigger), - multiplicity = self.multiplicity(trigger) + run_number=self._run_number, + npixels=self._npixels, + camera=self.CAMERA_NAME, + pixels_id=self._pixels_id, + method=method, + nevents=self.nevents(trigger), + charges_hg=self.charges_hg(trigger), + charges_lg=self.charges_lg(trigger), + peak_hg=self.peak_hg(trigger), + peak_lg=self.peak_lg(trigger), + broken_pixels_hg=self.broken_pixels_hg(trigger), + broken_pixels_lg=self.broken_pixels_lg(trigger), + ucts_timestamp=self.ucts_timestamp(trigger), + ucts_busy_counter=self.ucts_busy_counter(trigger), + ucts_event_counter=self.ucts_event_counter(trigger), + event_type=self.event_type(trigger), + event_id=self.event_id(trigger), + trig_pattern_all=self.trig_pattern_all(trigger), + trig_pattern=self.trig_pattern(trigger), + multiplicity=self.multiplicity(trigger), ) output.append(chargesContainer) return output @staticmethod - def sort(chargesContainer :ChargesContainer, method :str = 'event_id') : + def sort(chargesContainer: ChargesContainer, method: str = "event_id"): """ Sorts the charges in a ChargesContainer object based on the specified method. Args: @@ -319,29 +368,39 @@ def sort(chargesContainer :ChargesContainer, method :str = 'event_id') : method (str, optional): The sorting method. Defaults to 'event_id'. Returns: ChargesContainer: A new ChargesContainer object with the charges sorted based on the specified method. - + Raises: ArgumentError: If the specified method is not valid. """ output = ChargesContainer( - run_number = chargesContainer.run_number, - npixels = chargesContainer.npixels, - camera = chargesContainer.camera, - pixels_id = chargesContainer.pixels_id, - nevents = chargesContainer.nevents, - method = chargesContainer.method + run_number=chargesContainer.run_number, + npixels=chargesContainer.npixels, + camera=chargesContainer.camera, + pixels_id=chargesContainer.pixels_id, + nevents=chargesContainer.nevents, + method=chargesContainer.method, ) - if method == 'event_id' : + if method == "event_id": index = np.argsort(chargesContainer.event_id) - for field in chargesContainer.keys() : - if not(field in ["run_number","npixels","camera","pixels_id","nevents","method"]) : + for field in chargesContainer.keys(): + if not ( + field + in [ + "run_number", + "npixels", + "camera", + "pixels_id", + "nevents", + "method", + ] + ): output[field] = chargesContainer[field][index] - else : + else: raise ArgumentError(f"{method} is not a valid method for sorting") return output @staticmethod - def select_charges_hg(chargesContainer :ChargesContainer,pixel_id : np.ndarray) : + def select_charges_hg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): """ Selects the charges from the ChargesContainer object for the given pixel_id and returns the result transposed. Args: @@ -350,11 +409,14 @@ def select_charges_hg(chargesContainer :ChargesContainer,pixel_id : np.ndarray) Returns: np.ndarray: The selected charges from the ChargesContainer object for the given pixel_id, transposed. """ - res = __class__.select_container_array_field(container = chargesContainer,pixel_id = pixel_id,field = 'charges_hg') - res = res.transpose(1,0) + res = __class__.select_container_array_field( + container=chargesContainer, pixel_id=pixel_id, field="charges_hg" + ) + res = res.transpose(1, 0) return res + @staticmethod - def select_charges_lg(chargesContainer : ChargesContainer,pixel_id : np.ndarray) : + def select_charges_lg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): """ Selects the charges from the ChargesContainer object for the given pixel_id and returns the result transposed. Args: @@ -363,10 +425,13 @@ def select_charges_lg(chargesContainer : ChargesContainer,pixel_id : np.ndarray) Returns: np.ndarray: The selected charges from the ChargesContainer object for the given pixel_id, transposed. """ - res = __class__.select_container_array_field(container = chargesContainer,pixel_id = pixel_id,field = 'charges_lg') - res = res.transpose(1,0) + res = __class__.select_container_array_field( + container=chargesContainer, pixel_id=pixel_id, field="charges_lg" + ) + res = res.transpose(1, 0) return res - def charges_hg(self,trigger : EventType) : + + def charges_hg(self, trigger: EventType): """ Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. Args: @@ -374,8 +439,11 @@ def charges_hg(self,trigger : EventType) : Returns: np.ndarray: The charges for the specific trigger type. """ - return np.array(self.__charges_hg[__class__._get_name_trigger(trigger)],dtype = np.uint16) - def charges_lg(self,trigger : EventType) : + return np.array( + self.__charges_hg[__class__._get_name_trigger(trigger)], dtype=np.uint16 + ) + + def charges_lg(self, trigger: EventType): """ Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. Args: @@ -383,8 +451,11 @@ def charges_lg(self,trigger : EventType) : Returns: np.ndarray: The charges for the specific trigger type. """ - return np.array(self.__charges_lg[__class__._get_name_trigger(trigger)],dtype = np.uint16) - def peak_hg(self,trigger : EventType) : + return np.array( + self.__charges_lg[__class__._get_name_trigger(trigger)], dtype=np.uint16 + ) + + def peak_hg(self, trigger: EventType): """ Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. Args: @@ -392,8 +463,11 @@ def peak_hg(self,trigger : EventType) : Returns: np.ndarray: The peak charges for the specific trigger type. """ - return np.array(self.__peak_hg[__class__._get_name_trigger(trigger)],dtype = np.uint16) - def peak_lg(self,trigger : EventType) : + return np.array( + self.__peak_hg[__class__._get_name_trigger(trigger)], dtype=np.uint16 + ) + + def peak_lg(self, trigger: EventType): """ Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. Args: @@ -401,11 +475,16 @@ def peak_lg(self,trigger : EventType) : Returns: np.ndarray: The peak charges for the specific trigger type. """ - return np.array(self.__peak_lg[__class__._get_name_trigger(trigger)],dtype = np.uint16) - + return np.array( + self.__peak_lg[__class__._get_name_trigger(trigger)], dtype=np.uint16 + ) @staticmethod - def create_from_waveforms(waveformsContainer: WaveformsContainer, method: str = "FullWaveformSum", **kwargs) -> ChargesContainer: + def create_from_waveforms( + waveformsContainer: WaveformsContainer, + method: str = "FullWaveformSum", + **kwargs, + ) -> ChargesContainer: """ Create a ChargesContainer object from waveforms using the specified charge extraction method. Args: @@ -416,14 +495,18 @@ def create_from_waveforms(waveformsContainer: WaveformsContainer, method: str = ChargesContainer: The charges container object containing the computed charges and peak times. """ chargesContainer = ChargesContainer() - for field in waveformsContainer.keys() : - if not(field in ["subarray","nsamples","wfs_hg","wfs_lg"]) : - chargesContainer[field] = waveformsContainer[field] + for field in waveformsContainer.keys(): + if not (field in ["subarray", "nsamples", "wfs_hg", "wfs_lg"]): + chargesContainer[field] = waveformsContainer[field] log.info(f"computing hg charge with {method} method") - charges_hg, peak_hg = __class__.compute_charge(waveformsContainer, constants.HIGH_GAIN, method, **kwargs) + charges_hg, peak_hg = __class__.compute_charge( + waveformsContainer, constants.HIGH_GAIN, method, **kwargs + ) charges_hg = np.array(charges_hg, dtype=np.uint16) log.info(f"computing lg charge with {method} method") - charges_lg, peak_lg = __class__.compute_charge(waveformsContainer, constants.LOW_GAIN, method, **kwargs) + charges_lg, peak_lg = __class__.compute_charge( + waveformsContainer, constants.LOW_GAIN, method, **kwargs + ) charges_lg = np.array(charges_lg, dtype=np.uint16) chargesContainer.charges_hg = charges_hg chargesContainer.charges_lg = charges_lg @@ -431,9 +514,14 @@ def create_from_waveforms(waveformsContainer: WaveformsContainer, method: str = chargesContainer.peak_lg = peak_lg chargesContainer.method = method return chargesContainer - - @staticmethod - def compute_charge(waveformContainer : WaveformsContainer,channel : int,method : str = "FullWaveformSum" ,**kwargs) : + + @staticmethod + def compute_charge( + waveformContainer: WaveformsContainer, + channel: int, + method: str = "FullWaveformSum", + **kwargs, + ): """ Compute charge from waveforms. Args: @@ -447,82 +535,167 @@ def compute_charge(waveformContainer : WaveformsContainer,channel : int,method : Returns: tuple: A tuple containing the computed charges and peak times. """ - #import is here for fix issue with pytest (TypeError : inference is not possible with python <3.9 (Numba conflict bc there is no inference...)) + # import is here for fix issue with pytest (TypeError : inference is not possible with python <3.9 (Numba conflict bc there is no inference...)) from .extractor.utils import CtapipeExtractor - - imageExtractor = __class__._get_imageExtractor(method = method,subarray = waveformContainer.subarray,**kwargs) + + imageExtractor = __class__._get_imageExtractor( + method=method, subarray=waveformContainer.subarray, **kwargs + ) if channel == constants.HIGH_GAIN: - out = np.array([CtapipeExtractor.get_image_peak_time(imageExtractor(waveformContainer.wfs_hg[i],__class__.TEL_ID,channel,waveformContainer.broken_pixels_hg)) for i in range(len(waveformContainer.wfs_hg))]).transpose(1,0,2) - return out[0],out[1] + out = np.array( + [ + CtapipeExtractor.get_image_peak_time( + imageExtractor( + waveformContainer.wfs_hg[i], + __class__.TEL_ID, + channel, + waveformContainer.broken_pixels_hg, + ) + ) + for i in range(len(waveformContainer.wfs_hg)) + ] + ).transpose(1, 0, 2) + return out[0], out[1] elif channel == constants.LOW_GAIN: - out = np.array([CtapipeExtractor.get_image_peak_time(imageExtractor(waveformContainer.wfs_lg[i],__class__.TEL_ID,channel,waveformContainer.broken_pixels_lg)) for i in range(len(waveformContainer.wfs_lg))]).transpose(1,0,2) - return out[0],out[1] - else : - raise ArgumentError(f"channel must be {constants.LOW_GAIN} or {constants.HIGH_GAIN}") - + out = np.array( + [ + CtapipeExtractor.get_image_peak_time( + imageExtractor( + waveformContainer.wfs_lg[i], + __class__.TEL_ID, + channel, + waveformContainer.broken_pixels_lg, + ) + ) + for i in range(len(waveformContainer.wfs_lg)) + ] + ).transpose(1, 0, 2) + return out[0], out[1] + else: + raise ArgumentError( + f"channel must be {constants.LOW_GAIN} or {constants.HIGH_GAIN}" + ) + @staticmethod - def histo_hg(chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True) -> ma.masked_array: + def histo_hg( + chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True + ) -> ma.masked_array: """ Computes histogram of high gain charges from a ChargesContainer object. - + Args: chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. - + Returns: ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. """ - return __class__._histo(chargesContainer=chargesContainer, field='charges_hg', n_bins=n_bins, autoscale=autoscale) - + return __class__._histo( + chargesContainer=chargesContainer, + field="charges_hg", + n_bins=n_bins, + autoscale=autoscale, + ) + @staticmethod - def histo_lg(chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True) -> ma.masked_array: + def histo_lg( + chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True + ) -> ma.masked_array: """ Computes histogram of low gain charges from a ChargesContainer object. - + Args: chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. - + Returns: ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. """ - return __class__._histo(chargesContainer=chargesContainer, field='charges_lg', n_bins=n_bins, autoscale=autoscale) - + return __class__._histo( + chargesContainer=chargesContainer, + field="charges_lg", + n_bins=n_bins, + autoscale=autoscale, + ) + @staticmethod - def _histo(chargesContainer: ChargesContainer, field: str, n_bins: int = 1000, autoscale: bool = True) -> ma.masked_array: + def _histo( + chargesContainer: ChargesContainer, + field: str, + n_bins: int = 1000, + autoscale: bool = True, + ) -> ma.masked_array: """ Computes histogram of charges for a given field from a ChargesContainer object. Numba is used to compute histograms in a vectorized way. - + Args: chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. field (str): The field name for which the histogram is computed. n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. - + Returns: ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. """ - mask_broken_pix = np.array((chargesContainer[field] == chargesContainer[field].mean(axis=0)).mean(axis=0), dtype=bool) - log.debug(f"there are {mask_broken_pix.sum()} broken pixels (charge stays at same level for each events)") - - if autoscale: - all_range = np.arange(np.uint16(np.min(chargesContainer[field].T[~mask_broken_pix].T)) - 0.5, np.uint16(np.max(chargesContainer[field].T[~mask_broken_pix].T)) + 1.5, 1) - charge_ma = ma.masked_array((all_range.reshape(all_range.shape[0], 1) @ np.ones((1, chargesContainer[field].shape[1]))).T, mask=np.zeros((chargesContainer[field].shape[1], all_range.shape[0]), dtype=bool)) - broxen_pixels_mask = np.array([mask_broken_pix for i in range(charge_ma.shape[1])]).T + mask_broken_pix = np.array( + (chargesContainer[field] == chargesContainer[field].mean(axis=0)).mean( + axis=0 + ), + dtype=bool, + ) + log.debug( + f"there are {mask_broken_pix.sum()} broken pixels (charge stays at same level for each events)" + ) + + if autoscale: + all_range = np.arange( + np.uint16(np.min(chargesContainer[field].T[~mask_broken_pix].T)) - 0.5, + np.uint16(np.max(chargesContainer[field].T[~mask_broken_pix].T)) + 1.5, + 1, + ) + charge_ma = ma.masked_array( + ( + all_range.reshape(all_range.shape[0], 1) + @ np.ones((1, chargesContainer[field].shape[1])) + ).T, + mask=np.zeros( + (chargesContainer[field].shape[1], all_range.shape[0]), dtype=bool + ), + ) + broxen_pixels_mask = np.array( + [mask_broken_pix for i in range(charge_ma.shape[1])] + ).T start = time.time() - _mask, hist_ma_data = make_histo(chargesContainer[field].T, all_range, mask_broken_pix) + _mask, hist_ma_data = make_histo( + chargesContainer[field].T, all_range, mask_broken_pix + ) charge_ma.mask = np.logical_or(_mask, broxen_pixels_mask) hist_ma = ma.masked_array(hist_ma_data, mask=charge_ma.mask) - log.debug(f"histogram hg computation time : {time.time() - start} sec") - + log.debug(f"histogram hg computation time : {time.time() - start} sec") + return ma.masked_array((hist_ma, charge_ma)) - - else: - hist = np.array([np.histogram(chargesContainer[field].T[i], bins=n_bins)[0] for i in range(chargesContainer[field].shape[1])]) - charge = np.array([np.histogram(chargesContainer[field].T[i], bins=n_bins)[1] for i in range(chargesContainer[field].shape[1])]) - charge_edges = np.array([np.mean(charge.T[i:i+2], axis=0) for i in range(charge.shape[1]-1)]).T - - return np.array((hist, charge_edges)) \ No newline at end of file + + else: + hist = np.array( + [ + np.histogram(chargesContainer[field].T[i], bins=n_bins)[0] + for i in range(chargesContainer[field].shape[1]) + ] + ) + charge = np.array( + [ + np.histogram(chargesContainer[field].T[i], bins=n_bins)[1] + for i in range(chargesContainer[field].shape[1]) + ] + ) + charge_edges = np.array( + [ + np.mean(charge.T[i : i + 2], axis=0) + for i in range(charge.shape[1] - 1) + ] + ).T + + return np.array((hist, charge_edges)) diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index e945cb8a..30a9260f 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -1,21 +1,18 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers +import copy from abc import ABC, abstractmethod -from ctapipe_io_nectarcam import NectarCAMEventSource import numpy as np -import copy - from ctapipe.containers import EventType from ctapipe.instrument import CameraGeometry -from ctapipe_io_nectarcam import constants +from ctapipe_io_nectarcam import NectarCAMEventSource, constants from ctapipe_io_nectarcam.containers import NectarCAMDataContainer - - from ..data import DataManagement from ..data.container.core import ArrayDataContainer @@ -25,18 +22,22 @@ It includes the `BaseMaker` abstract class, the `EventsLoopMaker` and `ArrayDataMaker` subclasses. These classes are used to perform computations on data from a specific run.""" + class BaseMaker(ABC): - """Mother class for all the makers, the role of makers is to do computation on the data. - """ + """Mother class for all the makers, the role of makers is to do computation on the data.""" + @abstractmethod def make(self, *args, **kwargs): """ Abstract method that needs to be implemented by subclasses. - This method is the main one, which computes and does the work. + This method is the main one, which computes and does the work. """ pass + @staticmethod - def load_run(run_number : int,max_events : int = None, run_file = None) -> NectarCAMEventSource: + def load_run( + run_number: int, max_events: int = None, run_file=None + ) -> NectarCAMEventSource: """Static method to load from $NECTARCAMDATA directory data for specified run with max_events Args:self.__run_number = run_number @@ -47,13 +48,17 @@ def load_run(run_number : int,max_events : int = None, run_file = None) -> Necta List[ctapipe_io_nectarcam.NectarCAMEventSource]: List of EventSource for each run files """ # Load the data from the run file. - if run_file is None : - generic_filename,_ = DataManagement.findrun(run_number) + if run_file is None: + generic_filename, _ = DataManagement.findrun(run_number) log.info(f"{str(generic_filename)} will be loaded") - eventsource = NectarCAMEventSource(input_url=generic_filename,max_events=max_events) - else : + eventsource = NectarCAMEventSource( + input_url=generic_filename, max_events=max_events + ) + else: log.info(f"{run_file} will be loaded") - eventsource = NectarCAMEventSource(input_url=run_file,max_events=max_events) + eventsource = NectarCAMEventSource( + input_url=run_file, max_events=max_events + ) return eventsource @@ -71,7 +76,9 @@ class EventsLoopMaker(BaseMaker): maker.make(n_events=500) """ - def __init__(self, run_number: int, max_events: int = None, run_file=None, *args, **kwargs): + def __init__( + self, run_number: int, max_events: int = None, run_file=None, *args, **kwargs + ): """ Constructor method that initializes the EventsLoopMaker object. @@ -94,7 +101,9 @@ def __init__(self, run_number: int, max_events: int = None, run_file=None, *args log.info(f"N pixels : {self.npixels}") - def make(self, n_events=np.inf, restart_from_beginning : bool =False, *args, **kwargs): + def make( + self, n_events=np.inf, restart_from_beginning: bool = False, *args, **kwargs + ): """ Method to iterate over the events and perform computations on each event. @@ -103,8 +112,10 @@ def make(self, n_events=np.inf, restart_from_beginning : bool =False, *args, **k restart_from_beginning (bool, optional): Whether to restart from the beginning of the run. Defaults to False. """ if restart_from_beginning: - log.debug('restart from beginning : creation of the EventSource reader') - self.__reader = __class__.load_run(self.__run_number, self.__max_events, run_file=self.__run_file) + log.debug("restart from beginning : creation of the EventSource reader") + self.__reader = __class__.load_run( + self.__run_number, self.__max_events, run_file=self.__run_file + ) n_traited_events = 0 for i, event in enumerate(self.__reader): @@ -209,11 +220,10 @@ def run_number(self): """ return copy.deepcopy(self.__run_number) - -class ArrayDataMaker(EventsLoopMaker) : +class ArrayDataMaker(EventsLoopMaker): """ - Class used to loop over the events of a run and to extract informations that are stored in arrays. + Class used to loop over the events of a run and to extract informations that are stored in arrays. Example Usage: - Create an instance of the ArrayDataMaker class maker = ArrayDataMaker(run_number=1234, max_events=1000) @@ -244,7 +254,10 @@ class ArrayDataMaker(EventsLoopMaker) : TEL_ID = 0 CAMERA_NAME = "NectarCam-003" CAMERA = CameraGeometry.from_name(CAMERA_NAME) - def __init__(self,run_number : int,max_events : int = None,run_file = None,*args,**kwargs): + + def __init__( + self, run_number: int, max_events: int = None, run_file=None, *args, **kwargs + ): """construtor Args: @@ -253,10 +266,10 @@ def __init__(self,run_number : int,max_events : int = None,run_file = None,*args nevents (int, optional) : number of events in run if known (parameter used to save computing time) run_file (optional) : if provided, will load this run file """ - super().__init__(run_number,max_events,run_file,*args,**kwargs) - self.__nsamples = self._reader.camera_config.num_samples - - #data we want to compute + super().__init__(run_number, max_events, run_file, *args, **kwargs) + self.__nsamples = self._reader.camera_config.num_samples + + # data we want to compute self.__ucts_timestamp = {} self.__ucts_busy_counter = {} self.__ucts_event_counter = {} @@ -266,7 +279,7 @@ def __init__(self,run_number : int,max_events : int = None,run_file = None,*args self.__broken_pixels_hg = {} self.__broken_pixels_lg = {} - def _init_trigger_type(self, trigger : EventType, **kwargs): + def _init_trigger_type(self, trigger: EventType, **kwargs): """ Initializes empty lists for different trigger types in the ArrayDataMaker class. @@ -286,7 +299,6 @@ def _init_trigger_type(self, trigger : EventType, **kwargs): self.__broken_pixels_hg[f"{name}"] = [] self.__broken_pixels_lg[f"{name}"] = [] - @staticmethod def _compute_broken_pixels(wfs_hg, wfs_lg, **kwargs): """ @@ -299,25 +311,31 @@ def _compute_broken_pixels(wfs_hg, wfs_lg, **kwargs): tuple: Two arrays of zeros with the same shape as `wfs_hg` (or `wfs_lg`) but without the last dimension. """ log.warning("computation of broken pixels is not yet implemented") - return np.zeros((wfs_hg.shape[:-1]), dtype=bool), np.zeros((wfs_hg.shape[:-1]), dtype=bool) - + return np.zeros((wfs_hg.shape[:-1]), dtype=bool), np.zeros( + (wfs_hg.shape[:-1]), dtype=bool + ) + @staticmethod - def _compute_broken_pixels_event(event: NectarCAMDataContainer, pixels_id : np.ndarray, **kwargs): - """ - Computes broken pixels for a specific event and pixel IDs. - Args: - event (NectarCAMDataContainer): An event. - pixels_id (list or np.ndarray): IDs of pixels. - **kwargs: Additional keyword arguments. - Returns: - tuple: Two arrays of zeros with the length of `pixels_id`. - """ - log.warning("computation of broken pixels is not yet implemented") - return np.zeros((len(pixels_id)), dtype=bool), np.zeros((len(pixels_id)), dtype=bool) + def _compute_broken_pixels_event( + event: NectarCAMDataContainer, pixels_id: np.ndarray, **kwargs + ): + """ + Computes broken pixels for a specific event and pixel IDs. + Args: + event (NectarCAMDataContainer): An event. + pixels_id (list or np.ndarray): IDs of pixels. + **kwargs: Additional keyword arguments. + Returns: + tuple: Two arrays of zeros with the length of `pixels_id`. + """ + log.warning("computation of broken pixels is not yet implemented") + return np.zeros((len(pixels_id)), dtype=bool), np.zeros( + (len(pixels_id)), dtype=bool + ) @staticmethod def _get_name_trigger(trigger: EventType): - """ + """ Gets the name of a trigger event. Args: trigger (EventType): A trigger event. @@ -330,8 +348,14 @@ def _get_name_trigger(trigger: EventType): name = trigger.name return name - - def make(self, n_events=np.inf, trigger_type: list = None, restart_from_begining : bool=False, *args, **kwargs): + def make( + self, + n_events=np.inf, + trigger_type: list = None, + restart_from_begining: bool = False, + *args, + **kwargs, + ): """ Method to extract data from the EventSource. @@ -346,15 +370,19 @@ def make(self, n_events=np.inf, trigger_type: list = None, restart_from_begining The output container created by the _make_output_container method. """ if ~np.isfinite(n_events): - log.warning('no needed events number specified, it may cause a memory error') + log.warning( + "no needed events number specified, it may cause a memory error" + ) if isinstance(trigger_type, EventType) or trigger_type is None: trigger_type = [trigger_type] for _trigger_type in trigger_type: self._init_trigger_type(_trigger_type) if restart_from_begining: - log.debug('restart from begining : creation of the EventSource reader') - self._reader = __class__.load_run(self._run_number, self._max_events, run_file=self._run_file) + log.debug("restart from begining : creation of the EventSource reader") + self._reader = __class__.load_run( + self._run_number, self._max_events, run_file=self._run_file + ) n_traited_events = 0 for i, event in enumerate(self._reader): @@ -369,8 +397,9 @@ def make(self, n_events=np.inf, trigger_type: list = None, restart_from_begining return self._make_output_container(trigger_type, *args, **kwargs) - - def _make_event(self, event : NectarCAMDataContainer, trigger : EventType, *args, **kwargs): + def _make_event( + self, event: NectarCAMDataContainer, trigger: EventType, *args, **kwargs + ): """ Method to extract data from the event. @@ -384,24 +413,34 @@ def _make_event(self, event : NectarCAMDataContainer, trigger : EventType, *args If the return_wfs keyword argument is True, the method returns the high and low gain waveforms from the event. """ name = __class__._get_name_trigger(trigger) - self.__event_id[f'{name}'].append(np.uint16(event.index.event_id)) - self.__ucts_timestamp[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_timestamp) - self.__event_type[f'{name}'].append(event.trigger.event_type.value) - self.__ucts_busy_counter[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_busy_counter) - self.__ucts_event_counter[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_event_counter) - self.__trig_patter_all[f'{name}'].append(event.nectarcam.tel[__class__.TEL_ID].evt.trigger_pattern.T) + self.__event_id[f"{name}"].append(np.uint16(event.index.event_id)) + self.__ucts_timestamp[f"{name}"].append( + event.nectarcam.tel[__class__.TEL_ID].evt.ucts_timestamp + ) + self.__event_type[f"{name}"].append(event.trigger.event_type.value) + self.__ucts_busy_counter[f"{name}"].append( + event.nectarcam.tel[__class__.TEL_ID].evt.ucts_busy_counter + ) + self.__ucts_event_counter[f"{name}"].append( + event.nectarcam.tel[__class__.TEL_ID].evt.ucts_event_counter + ) + self.__trig_patter_all[f"{name}"].append( + event.nectarcam.tel[__class__.TEL_ID].evt.trigger_pattern.T + ) if kwargs.get("return_wfs", False): get_wfs_hg = event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id] get_wfs_lg = event.r0.tel[0].waveform[constants.LOW_GAIN][self.pixels_id] return get_wfs_hg, get_wfs_lg - @abstractmethod - def _make_output_container(self) : pass + def _make_output_container(self): + pass @staticmethod - def select_container_array_field(container: ArrayDataContainer, pixel_id: np.ndarray, field: str) -> np.ndarray: + def select_container_array_field( + container: ArrayDataContainer, pixel_id: np.ndarray, field: str + ) -> np.ndarray: """ Selects specific fields from an ArrayDataContainer object based on a given list of pixel IDs. @@ -413,222 +452,255 @@ def select_container_array_field(container: ArrayDataContainer, pixel_id: np.nda Returns: ndarray: An array containing the selected data for the given pixel IDs. """ - mask_contain_pixels_id = np.array([pixel in container.pixels_id for pixel in pixel_id], dtype=bool) + mask_contain_pixels_id = np.array( + [pixel in container.pixels_id for pixel in pixel_id], dtype=bool + ) for pixel in pixel_id[~mask_contain_pixels_id]: - log.warning(f"You asked for pixel_id {pixel} but it is not present in this container, skip this one") - res = np.array([np.take(container[field], np.where(container.pixels_id == pixel)[0][0], axis=1) for pixel in pixel_id[mask_contain_pixels_id]]) + log.warning( + f"You asked for pixel_id {pixel} but it is not present in this container, skip this one" + ) + res = np.array( + [ + np.take( + container[field], + np.where(container.pixels_id == pixel)[0][0], + axis=1, + ) + for pixel in pixel_id[mask_contain_pixels_id] + ] + ) ####could be nice to return np.ma.masked_array(data = res, mask = container.broken_pixels_hg.transpose(res.shape[1],res.shape[0],res.shape[2])) return res - - @staticmethod - def merge(container_a : ArrayDataContainer,container_b : ArrayDataContainer) -> ArrayDataContainer : + def merge( + container_a: ArrayDataContainer, container_b: ArrayDataContainer + ) -> ArrayDataContainer: """method to merge 2 ArrayDataContainer into one single ArrayDataContainer Returns: ArrayDataContainer: the merged object """ - if type(container_a) != type(container_b) : + if type(container_a) != type(container_b): raise Exception("The containers have to be instnace of the same class") - if np.array_equal(container_a.pixels_id,container_b.pixels_id) : + if np.array_equal(container_a.pixels_id, container_b.pixels_id): raise Exception("The containers have not the same pixels ids") - + merged_container = container_a.__class__.__new__() - for field in container_a.keys() : - if ~isinstance(container_a[field],np.ndarray) : - if container_a[field] != container_b[field] : - raise Exception(f"merge impossible because of {field} filed (values are {container_a[field]} and {container_b[field]}") - - for field in container_a.keys() : - if isinstance(container_a[field],np.ndarray) : - merged_container[field] = np.concatenate(container_a[field],container_a[field],axis = 0) - else : + for field in container_a.keys(): + if ~isinstance(container_a[field], np.ndarray): + if container_a[field] != container_b[field]: + raise Exception( + f"merge impossible because of {field} filed (values are {container_a[field]} and {container_b[field]}" + ) + + for field in container_a.keys(): + if isinstance(container_a[field], np.ndarray): + merged_container[field] = np.concatenate( + container_a[field], container_a[field], axis=0 + ) + else: merged_container[field] = container_a[field] return merged_container - - @property - def nsamples(self) : + def nsamples(self): """ Returns a deep copy of the nsamples attribute. - + Returns: np.ndarray: A deep copy of the nsamples attribute. """ return copy.deepcopy(self.__nsamples) @property - def _nsamples(self) : + def _nsamples(self): """ Returns the nsamples attribute. - + Returns: np.ndarray: The nsamples attribute. """ return self.__nsamples - def nevents(self,trigger : EventType) : + def nevents(self, trigger: EventType): """ Returns the number of events for the specified trigger type. - + Args: trigger (EventType): The trigger type for which the number of events is requested. - + Returns: int: The number of events for the specified trigger type. """ return len(self.__event_id[__class__._get_name_trigger(trigger)]) @property - def _broken_pixels_hg(self) : + def _broken_pixels_hg(self): """ Returns the broken_pixels_hg attribute. - + Returns: np.ndarray: The broken_pixels_hg attribute. """ return self.__broken_pixels_hg - def broken_pixels_hg(self,trigger : EventType) : + def broken_pixels_hg(self, trigger: EventType): """ Returns an array of broken pixels for high gain for the specified trigger type. - + Args: trigger (EventType): The trigger type for which the broken pixels for high gain are requested. - + Returns: np.ndarray: An array of broken pixels for high gain for the specified trigger type. """ - return np.array(self.__broken_pixels_hg[__class__._get_name_trigger(trigger)],dtype = bool) + return np.array( + self.__broken_pixels_hg[__class__._get_name_trigger(trigger)], dtype=bool + ) @property - def _broken_pixels_lg(self) : + def _broken_pixels_lg(self): """ Returns the broken_pixels_lg attribute. - + Returns: np.ndarray: The broken_pixels_lg attribute. """ return self.__broken_pixels_lg - def broken_pixels_lg(self,trigger : EventType) : + def broken_pixels_lg(self, trigger: EventType): """ Returns an array of broken pixels for low gain for the specified trigger type. - + Args: trigger (EventType): The trigger type for which the broken pixels for low gain are requested. - + Returns: np.ndarray: An array of broken pixels for low gain for the specified trigger type. """ - return np.array(self.__broken_pixels_lg[__class__._get_name_trigger(trigger)],dtype = bool) + return np.array( + self.__broken_pixels_lg[__class__._get_name_trigger(trigger)], dtype=bool + ) - def ucts_timestamp(self,trigger : EventType) : + def ucts_timestamp(self, trigger: EventType): """ Returns an array of UCTS timestamps for the specified trigger type. - + Args: trigger (EventType): The trigger type for which the UCTS timestamps are requested. - + Returns: np.ndarray: An array of UCTS timestamps for the specified trigger type. """ - return np.array(self.__ucts_timestamp[__class__._get_name_trigger(trigger)],dtype = np.uint64) + return np.array( + self.__ucts_timestamp[__class__._get_name_trigger(trigger)], dtype=np.uint64 + ) - def ucts_busy_counter(self,trigger : EventType) : + def ucts_busy_counter(self, trigger: EventType): """ Returns an array of UCTS busy counters for the specified trigger type. - + Args: trigger (EventType): The trigger type for which the UCTS busy counters are requested. - + Returns: np.ndarray: An array of UCTS busy counters for the specified trigger type. """ - return np.array(self.__ucts_busy_counter[__class__._get_name_trigger(trigger)],dtype = np.uint32) + return np.array( + self.__ucts_busy_counter[__class__._get_name_trigger(trigger)], + dtype=np.uint32, + ) - def ucts_event_counter(self,trigger : EventType) : + def ucts_event_counter(self, trigger: EventType): """ Returns an array of UCTS event counters for the specified trigger type. - + Args: trigger (EventType): The trigger type for which the UCTS event counters are requested. - + Returns: np.ndarray: An array of UCTS event counters for the specified trigger type. """ - return np.array(self.__ucts_event_counter[__class__._get_name_trigger(trigger)],dtype = np.uint32) + return np.array( + self.__ucts_event_counter[__class__._get_name_trigger(trigger)], + dtype=np.uint32, + ) - def event_type(self,trigger : EventType) : + def event_type(self, trigger: EventType): """ Returns an array of event types for the specified trigger type. - + Args: trigger (EventType): The trigger type for which the event types are requested. - + Returns: np.ndarray: An array of event types for the specified trigger type. """ - return np.array(self.__event_type[__class__._get_name_trigger(trigger)],dtype = np.uint8) + return np.array( + self.__event_type[__class__._get_name_trigger(trigger)], dtype=np.uint8 + ) - def event_id(self,trigger : EventType) : + def event_id(self, trigger: EventType): """ Returns an array of event IDs for the specified trigger type. - + Args: trigger (EventType): The trigger type for which the event IDs are requested. - + Returns: np.ndarray: An array of event IDs for the specified trigger type. """ - return np.array(self.__event_id[__class__._get_name_trigger(trigger)],dtype = np.uint32) + return np.array( + self.__event_id[__class__._get_name_trigger(trigger)], dtype=np.uint32 + ) - def multiplicity(self,trigger : EventType) : + def multiplicity(self, trigger: EventType): """ Returns an array of multiplicities for the specified trigger type. - + Args: trigger (EventType): The trigger type for which the multiplicities are requested. - + Returns: np.ndarray: An array of multiplicities for the specified trigger type. """ tmp = self.trig_pattern(trigger) - if len(tmp) == 0 : + if len(tmp) == 0: return np.array([]) - else : - return np.uint16(np.count_nonzero(tmp,axis = 1)) + else: + return np.uint16(np.count_nonzero(tmp, axis=1)) - def trig_pattern(self,trigger : EventType) : + def trig_pattern(self, trigger: EventType): """ Returns an array of trigger patterns for the specified trigger type. - + Args: trigger (EventType): The trigger type for which the trigger patterns are requested. - + Returns: np.ndarray: An array of trigger patterns for the specified trigger type. """ tmp = self.trig_pattern_all(trigger) - if len(tmp) == 0 : + if len(tmp) == 0: return np.array([]) - else : - return tmp.any(axis = 2) + else: + return tmp.any(axis=2) - def trig_pattern_all(self,trigger : EventType) : + def trig_pattern_all(self, trigger: EventType): """ Returns an array of trigger patterns for all events for the specified trigger type. - + Args: trigger (EventType): The trigger type for which the trigger patterns for all events are requested. - + Returns: np.ndarray: An array of trigger patterns for all events for the specified trigger type. """ - return np.array(self.__trig_patter_all[f"{__class__._get_name_trigger(trigger)}"],dtype = bool) - + return np.array( + self.__trig_patter_all[f"{__class__._get_name_trigger(trigger)}"], + dtype=bool, + ) diff --git a/src/nectarchain/makers/extractor/__init__.py b/src/nectarchain/makers/extractor/__init__.py index 6a54cdfe..2c75a364 100644 --- a/src/nectarchain/makers/extractor/__init__.py +++ b/src/nectarchain/makers/extractor/__init__.py @@ -1 +1 @@ -#from .charge_extractor import * \ No newline at end of file +# from .charge_extractor import * diff --git a/src/nectarchain/makers/extractor/charge_extractor.py b/src/nectarchain/makers/extractor/charge_extractor.py index 3f75c101..670234c5 100644 --- a/src/nectarchain/makers/extractor/charge_extractor.py +++ b/src/nectarchain/makers/extractor/charge_extractor.py @@ -1,92 +1,128 @@ -from ctapipe.image import ImageExtractor -import numpy as np import logging -from scipy.interpolate import InterpolatedUnivariateSpline -from scipy.signal import find_peaks +import numpy as np +from ctapipe.image import ImageExtractor from numba import guvectorize +from scipy.interpolate import InterpolatedUnivariateSpline +from scipy.signal import find_peaks -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers -__all__ = ['gradient_extractor'] +__all__ = ["gradient_extractor"] -class gradient_extractor(ImageExtractor) : - - fixed_window=np.uint8(16) - height_peak=np.uint8(10) - - def __call__(self, waveforms, telid, selected_gain_channel,substract_ped = False): +class gradient_extractor(ImageExtractor): + fixed_window = np.uint8(16) + height_peak = np.uint8(10) + def __call__(self, waveforms, telid, selected_gain_channel, substract_ped=False): shape = waveforms.shape - waveforms = waveforms.reshape(shape[0]*shape[1],shape[2]) - - if substract_ped : - log.info('substracting pedestal') - #calculate pedestal - ped_mean = np.mean(waveforms[:,:16],axis = 1).reshape(shape[0]*shape[1],1) + waveforms = waveforms.reshape(shape[0] * shape[1], shape[2]) + + if substract_ped: + log.info("substracting pedestal") + # calculate pedestal + ped_mean = np.mean(waveforms[:, :16], axis=1).reshape( + shape[0] * shape[1], 1 + ) - y = waveforms - (ped_mean)@(np.ones(1,shape[2])) # waveforms without pedestal - else : - log.info('do not substract pedestal') + y = waveforms - (ped_mean) @ ( + np.ones(1, shape[2]) + ) # waveforms without pedestal + else: + log.info("do not substract pedestal") y = waveforms - waveforms.reshape(shape[0],shape[1],shape[2]) + waveforms.reshape(shape[0], shape[1], shape[2]) - peak_time, charge_integral, charge_sum, charge_integral_fixed_window, charge_sum_fixed_window = extract_charge(waveforms, self.height_peak, self.fixed_window) + ( + peak_time, + charge_integral, + charge_sum, + charge_integral_fixed_window, + charge_sum_fixed_window, + ) = extract_charge(waveforms, self.height_peak, self.fixed_window) return peak_time, charge_integral @guvectorize( [ - '(np.uint16[:], np.uint8, np.uint8, np.uint16[:])', + "(np.uint16[:], np.uint8, np.uint8, np.uint16[:])", ], "(n,p,s),(),()->(n,p)", nopython=True, cache=True, ) -def extract_charge(y,height_peak,fixed_window) : +def extract_charge(y, height_peak, fixed_window): x = np.linspace(0, len(y), len(y)) xi = np.linspace(0, len(y), 251) ius = InterpolatedUnivariateSpline(x, y) yi = ius(xi) peaks, _ = find_peaks(yi, height=height_peak) # find the peak - if len(peaks)>0: + if len(peaks) > 0: # find the max peak # max_peak = max(yi[peaks]) max_peak_index = np.argmax(yi[peaks], axis=0) # Check if there is not a peak but a plateaux # 1. divide for the maximum to round to the first digit # 2. create a new array with normalized and rounded values, yi_rounded - yi_rounded = np.around(yi[peaks]/max(yi[peaks]),1) + yi_rounded = np.around(yi[peaks] / max(yi[peaks]), 1) maxima_peak_index = np.argwhere(yi_rounded == np.amax(yi_rounded)) - if len(maxima_peak_index)>1: + if len(maxima_peak_index) > 1: # saturated event max_peak_index = int(np.median(maxima_peak_index)) # width_peak = 20 - if (xi[peaks[max_peak_index]]>20)&(xi[peaks[max_peak_index]]<40): # Search the adaptive integration window + if (xi[peaks[max_peak_index]] > 20) & ( + xi[peaks[max_peak_index]] < 40 + ): # Search the adaptive integration window # calculate total gradients (not used, only for plot) yi_grad_tot = np.gradient(yi, 1) maxposition = peaks[max_peak_index] # calcualte grandients starting from the max peak and going to the left to find the left margin of the window yi_left = yi[:maxposition] yi_grad_left = np.gradient(yi_left[::-1], 0.9) - change_grad_pos_left = (np.where(yi_grad_left[:-1] * yi_grad_left[1:] < 0 )[0] +1)[0] + change_grad_pos_left = ( + np.where(yi_grad_left[:-1] * yi_grad_left[1:] < 0)[0] + 1 + )[0] # calcualte grandients starting from the max peak and going to the right to find the right margin of the window yi_right = yi[maxposition:] yi_grad_right = np.gradient(yi_right, 0.5) - change_grad_pos_right = (np.where(yi_grad_right[:-1] * yi_grad_right[1:] < 0 )[0] +1)[0] - charge_integral = ius.integral(xi[peaks[max_peak_index]-(change_grad_pos_left)], xi[peaks[max_peak_index]+ change_grad_pos_right]) - charge_sum = yi[(peaks[max_peak_index]-(change_grad_pos_left)):peaks[max_peak_index]+change_grad_pos_right].sum()/(change_grad_pos_left+change_grad_pos_right) # simple sum integration + change_grad_pos_right = ( + np.where(yi_grad_right[:-1] * yi_grad_right[1:] < 0)[0] + 1 + )[0] + charge_integral = ius.integral( + xi[peaks[max_peak_index] - (change_grad_pos_left)], + xi[peaks[max_peak_index] + change_grad_pos_right], + ) + charge_sum = yi[ + (peaks[max_peak_index] - (change_grad_pos_left)) : peaks[max_peak_index] + + change_grad_pos_right + ].sum() / ( + change_grad_pos_left + change_grad_pos_right + ) # simple sum integration adaptive_window = change_grad_pos_right + change_grad_pos_left - window_right = (fixed_window-6)/2 - window_left = (fixed_window-6)/2 + 6 - charge_integral_fixed_window = ius.integral(xi[peaks[max_peak_index]-(window_left)], xi[peaks[max_peak_index]+window_right]) - charge_sum_fixed_window = yi[(peaks[max_peak_index]-(window_left)):peaks[max_peak_index]+window_right].sum()/(fixed_window) # simple sum integration + window_right = (fixed_window - 6) / 2 + window_left = (fixed_window - 6) / 2 + 6 + charge_integral_fixed_window = ius.integral( + xi[peaks[max_peak_index] - (window_left)], + xi[peaks[max_peak_index] + window_right], + ) + charge_sum_fixed_window = yi[ + (peaks[max_peak_index] - (window_left)) : peaks[max_peak_index] + + window_right + ].sum() / ( + fixed_window + ) # simple sum integration else: - log.info('No peak found, maybe it is a pedestal or noisy run!') - return adaptive_window, charge_integral, charge_sum, charge_integral_fixed_window, charge_sum_fixed_window \ No newline at end of file + log.info("No peak found, maybe it is a pedestal or noisy run!") + return ( + adaptive_window, + charge_integral, + charge_sum, + charge_integral_fixed_window, + charge_sum_fixed_window, + ) diff --git a/src/nectarchain/makers/extractor/tests/test_utils.py b/src/nectarchain/makers/extractor/tests/test_utils.py index ee4835f0..fc7dcf4d 100644 --- a/src/nectarchain/makers/extractor/tests/test_utils.py +++ b/src/nectarchain/makers/extractor/tests/test_utils.py @@ -1,20 +1,22 @@ import pytest + class TestCtapipeExtractor: - - @pytest.mark.skip('numba conflict') + @pytest.mark.skip("numba conflict") # Tests that the function returns the image and peak_time values from a valid DL1CameraContainer object. def test_get_image_peak_time_valid_object(self): from ctapipe.containers import DL1CameraContainer + from nectarchain.makers.extractor.utils import CtapipeExtractor + # Create a valid DL1CameraContainer object container = DL1CameraContainer() container.image = [1, 2, 3, 4, 5] container.peak_time = [10, 4, 5, 6, 9] - + # Call the function under test result_image, result_peak_time = CtapipeExtractor.get_image_peak_time(container) - + # Check the result assert result_image == [1, 2, 3, 4, 5] - assert result_peak_time == [10, 4, 5, 6, 9] \ No newline at end of file + assert result_peak_time == [10, 4, 5, 6, 9] diff --git a/src/nectarchain/makers/extractor/utils.py b/src/nectarchain/makers/extractor/utils.py index de096258..0fab1212 100644 --- a/src/nectarchain/makers/extractor/utils.py +++ b/src/nectarchain/makers/extractor/utils.py @@ -1,11 +1,13 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers + +from ctapipe.containers import DL1CameraContainer -from ctapipe.containers import DL1CameraContainer -class CtapipeExtractor(): +class CtapipeExtractor: """ A class to extract the image and peak time from a DL1CameraContainer object. """ diff --git a/src/nectarchain/makers/tests/test_chargesMakers.py b/src/nectarchain/makers/tests/test_chargesMakers.py index 659413ea..04409263 100644 --- a/src/nectarchain/makers/tests/test_chargesMakers.py +++ b/src/nectarchain/makers/tests/test_chargesMakers.py @@ -1,33 +1,43 @@ -from nectarchain.makers import ChargesMaker,WaveformsMaker -from nectarchain.data.container import ChargesContainer,ChargesContainerIO -from ctapipe.containers import EventType -import numpy as np import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level = logging.DEBUG) + +import numpy as np +from ctapipe.containers import EventType + +from nectarchain.data.container import ChargesContainer, ChargesContainerIO +from nectarchain.makers import ChargesMaker, WaveformsMaker + +logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG +) log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers + class TestChargesMaker: run_number = 3938 max_events = 100 - def test_instance(self) : - chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, - max_events = TestChargesMaker.max_events) - assert isinstance(chargesMaker,ChargesMaker) + def test_instance(self): + chargesMaker = ChargesMaker( + run_number=TestChargesMaker.run_number, + max_events=TestChargesMaker.max_events, + ) + assert isinstance(chargesMaker, ChargesMaker) def test_shape_valid(self): - chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, - max_events = TestChargesMaker.max_events) + chargesMaker = ChargesMaker( + run_number=TestChargesMaker.run_number, + max_events=TestChargesMaker.max_events, + ) chargesContainer = chargesMaker.make()[0] assert chargesContainer.nevents <= TestChargesMaker.max_events assert chargesContainer.run_number == TestChargesMaker.run_number - assert chargesContainer.ucts_timestamp.shape == (chargesContainer.nevents,) + assert chargesContainer.ucts_timestamp.shape == (chargesContainer.nevents,) assert chargesContainer.ucts_busy_counter.shape == (chargesContainer.nevents,) - assert chargesContainer.ucts_event_counter.shape == (chargesContainer.nevents,) - assert chargesContainer.event_type.shape == (chargesContainer.nevents,) - assert chargesContainer.event_id.shape == (chargesContainer.nevents,) + assert chargesContainer.ucts_event_counter.shape == (chargesContainer.nevents,) + assert chargesContainer.event_type.shape == (chargesContainer.nevents,) + assert chargesContainer.event_id.shape == (chargesContainer.nevents,) assert chargesContainer.trig_pattern_all.shape[0] == chargesContainer.nevents assert chargesContainer.trig_pattern_all.shape[2] == 4 @@ -38,91 +48,150 @@ def test_shape_valid(self): assert chargesContainer.charges_lg.mean() != 0 assert chargesContainer.peak_hg.mean() != 0 assert chargesContainer.peak_lg.mean() != 0 - assert chargesContainer.charges_hg.shape == (chargesContainer.nevents,chargesContainer.npixels) - assert chargesContainer.charges_lg.shape == (chargesContainer.nevents,chargesContainer.npixels) - assert chargesContainer.peak_hg.shape == (chargesContainer.nevents,chargesContainer.npixels) - assert chargesContainer.peak_lg.shape == (chargesContainer.nevents,chargesContainer.npixels) - - assert chargesContainer.broken_pixels_hg.shape == (chargesContainer.nevents,chargesContainer.npixels) - assert chargesContainer.broken_pixels_lg.shape == (chargesContainer.nevents,chargesContainer.npixels) - - def test_make_restart_eventsource(self) : - chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, - max_events = TestChargesMaker.max_events) - chargesContainer_list = chargesMaker.make(restart_from_begining = True) - assert isinstance(chargesContainer_list[0],ChargesContainer) - - def test_make_LocalPeakWindowSum(self) : - chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, - max_events = TestChargesMaker.max_events) - chargesContainer_list = chargesMaker.make(method = "LocalPeakWindowSum",window_shift = -4, window_length = 16) - assert isinstance(chargesContainer_list[0],ChargesContainer) - - def test_all_multiple_trigger(self) : + assert chargesContainer.charges_hg.shape == ( + chargesContainer.nevents, + chargesContainer.npixels, + ) + assert chargesContainer.charges_lg.shape == ( + chargesContainer.nevents, + chargesContainer.npixels, + ) + assert chargesContainer.peak_hg.shape == ( + chargesContainer.nevents, + chargesContainer.npixels, + ) + assert chargesContainer.peak_lg.shape == ( + chargesContainer.nevents, + chargesContainer.npixels, + ) + + assert chargesContainer.broken_pixels_hg.shape == ( + chargesContainer.nevents, + chargesContainer.npixels, + ) + assert chargesContainer.broken_pixels_lg.shape == ( + chargesContainer.nevents, + chargesContainer.npixels, + ) + + def test_make_restart_eventsource(self): + chargesMaker = ChargesMaker( + run_number=TestChargesMaker.run_number, + max_events=TestChargesMaker.max_events, + ) + chargesContainer_list = chargesMaker.make(restart_from_begining=True) + assert isinstance(chargesContainer_list[0], ChargesContainer) + + def test_make_LocalPeakWindowSum(self): + chargesMaker = ChargesMaker( + run_number=TestChargesMaker.run_number, + max_events=TestChargesMaker.max_events, + ) + chargesContainer_list = chargesMaker.make( + method="LocalPeakWindowSum", window_shift=-4, window_length=16 + ) + assert isinstance(chargesContainer_list[0], ChargesContainer) + + def test_all_multiple_trigger(self): trigger1 = EventType.FLATFIELD trigger2 = EventType.SKY_PEDESTAL - chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, - max_events = TestChargesMaker.max_events) - chargesContainer_list = chargesMaker.make(trigger_type = [trigger1,trigger2]) - for chargesContainer in chargesContainer_list : - assert isinstance(chargesContainer,ChargesContainer) - - - def test_all_trigger_None(self) : - chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, - max_events = TestChargesMaker.max_events) + chargesMaker = ChargesMaker( + run_number=TestChargesMaker.run_number, + max_events=TestChargesMaker.max_events, + ) + chargesContainer_list = chargesMaker.make(trigger_type=[trigger1, trigger2]) + for chargesContainer in chargesContainer_list: + assert isinstance(chargesContainer, ChargesContainer) + + def test_all_trigger_None(self): + chargesMaker = ChargesMaker( + run_number=TestChargesMaker.run_number, + max_events=TestChargesMaker.max_events, + ) chargesContainer_list = chargesMaker.make() - assert isinstance(chargesContainer_list[0],ChargesContainer) + assert isinstance(chargesContainer_list[0], ChargesContainer) - def test_create_from_waveforms(self) : - waveformsMaker = WaveformsMaker(run_number = TestChargesMaker.run_number, - max_events = TestChargesMaker.max_events) + def test_create_from_waveforms(self): + waveformsMaker = WaveformsMaker( + run_number=TestChargesMaker.run_number, + max_events=TestChargesMaker.max_events, + ) waveformsContainer_list = waveformsMaker.make() - chargesContainer = ChargesMaker.create_from_waveforms(waveformsContainer_list[0],method = "LocalPeakWindowSum",window_shift = -4, window_length = 16) - assert isinstance(chargesContainer,ChargesContainer) - - def test_select_charges(self) : - chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, - max_events = TestChargesMaker.max_events) + chargesContainer = ChargesMaker.create_from_waveforms( + waveformsContainer_list[0], + method="LocalPeakWindowSum", + window_shift=-4, + window_length=16, + ) + assert isinstance(chargesContainer, ChargesContainer) + + def test_select_charges(self): + chargesMaker = ChargesMaker( + run_number=TestChargesMaker.run_number, + max_events=TestChargesMaker.max_events, + ) chargesContainer_list = chargesMaker.make() - pixel_id = np.array([3,67,87]) - assert isinstance(ChargesMaker.select_charges_hg(chargesContainer_list[0],pixel_id),np.ndarray) - assert isinstance(ChargesMaker.select_charges_lg(chargesContainer_list[0],pixel_id),np.ndarray) - - def test_histo(self) : - chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, - max_events = TestChargesMaker.max_events) + pixel_id = np.array([3, 67, 87]) + assert isinstance( + ChargesMaker.select_charges_hg(chargesContainer_list[0], pixel_id), + np.ndarray, + ) + assert isinstance( + ChargesMaker.select_charges_lg(chargesContainer_list[0], pixel_id), + np.ndarray, + ) + + def test_histo(self): + chargesMaker = ChargesMaker( + run_number=TestChargesMaker.run_number, + max_events=TestChargesMaker.max_events, + ) chargesContainer_list = chargesMaker.make() histo = ChargesMaker.histo_hg(chargesContainer_list[0]) - assert isinstance(histo,np.ndarray) + assert isinstance(histo, np.ndarray) assert histo.mean() != 0 assert histo.shape[0] == 2 assert histo.shape[1] == chargesContainer_list[0].npixels histo = ChargesMaker.histo_lg(chargesContainer_list[0]) - assert isinstance(histo,np.ndarray) + assert isinstance(histo, np.ndarray) assert histo.mean() != 0 assert histo.shape[0] == 2 assert histo.shape[1] == chargesContainer_list[0].npixels - - def test_sort_ChargesContainer(self) : - chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, - max_events = TestChargesMaker.max_events) + def test_sort_ChargesContainer(self): + chargesMaker = ChargesMaker( + run_number=TestChargesMaker.run_number, + max_events=TestChargesMaker.max_events, + ) chargesContainer_list = chargesMaker.make() - sortWfs = ChargesMaker.sort(chargesContainer_list[0],method = 'event_id') - assert np.array_equal(sortWfs.event_id ,np.sort(chargesContainer_list[0].event_id)) - - def test_write_load_container(self) : - chargesMaker = ChargesMaker(run_number = TestChargesMaker.run_number, - max_events = TestChargesMaker.max_events) + sortWfs = ChargesMaker.sort(chargesContainer_list[0], method="event_id") + assert np.array_equal( + sortWfs.event_id, np.sort(chargesContainer_list[0].event_id) + ) + + def test_write_load_container(self): + chargesMaker = ChargesMaker( + run_number=TestChargesMaker.run_number, + max_events=TestChargesMaker.max_events, + ) chargesContainer_list = chargesMaker.make() - ChargesContainerIO.write("/tmp/test_charge_container/",chargesContainer_list[0],overwrite = True) - loaded_charge = ChargesContainerIO.load(f"/tmp/test_charge_container",run_number = TestChargesMaker.run_number) - assert np.array_equal(chargesContainer_list[0].charges_hg,loaded_charge.charges_hg) - -if __name__ == '__main__' : + ChargesContainerIO.write( + "/tmp/test_charge_container/", chargesContainer_list[0], overwrite=True + ) + loaded_charge = ChargesContainerIO.load( + f"/tmp/test_charge_container", run_number=TestChargesMaker.run_number + ) + assert np.array_equal( + chargesContainer_list[0].charges_hg, loaded_charge.charges_hg + ) + + +if __name__ == "__main__": import logging - logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level = logging.DEBUG) + + logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG + ) log = logging.getLogger(__name__) - log.handlers = logging.getLogger('__main__').handlers - TestChargesMaker().test_write_load_container() \ No newline at end of file + log.handlers = logging.getLogger("__main__").handlers + TestChargesMaker().test_write_load_container() diff --git a/src/nectarchain/makers/tests/test_core.py b/src/nectarchain/makers/tests/test_core.py index d1aa0e05..98ae818f 100644 --- a/src/nectarchain/makers/tests/test_core.py +++ b/src/nectarchain/makers/tests/test_core.py @@ -1,32 +1,43 @@ -from nectarchain.makers import ChargesMaker,WaveformsMaker,ArrayDataMaker -from nectarchain.data.container import ChargesContainer import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level = logging.DEBUG) + +from nectarchain.data.container import ChargesContainer +from nectarchain.makers import ArrayDataMaker, ChargesMaker, WaveformsMaker + +logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG +) log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers + class TestArrayDataMaker: run_number = 3938 max_events = 100 - def test_merge(self) : - chargesMaker = ChargesMaker(run_number = TestArrayDataMaker.run_number, - max_events = TestArrayDataMaker.max_events) + def test_merge(self): + chargesMaker = ChargesMaker( + run_number=TestArrayDataMaker.run_number, + max_events=TestArrayDataMaker.max_events, + ) charges_1 = chargesMaker.make() - chargesMaker_2 = ChargesMaker(run_number = TestArrayDataMaker.run_number, - max_events = TestArrayDataMaker.max_events) + chargesMaker_2 = ChargesMaker( + run_number=TestArrayDataMaker.run_number, + max_events=TestArrayDataMaker.max_events, + ) charges_2 = chargesMaker_2.make() - merged = ArrayDataMaker.merge(charges_1,charges_2) - assert isinstance(merged,ChargesContainer) + merged = ArrayDataMaker.merge(charges_1, charges_2) + assert isinstance(merged, ChargesContainer) - def test_merge_different_container(self) : - chargesMaker = ChargesMaker(run_number = TestArrayDataMaker.run_number, - max_events = TestArrayDataMaker.max_events) + def test_merge_different_container(self): + chargesMaker = ChargesMaker( + run_number=TestArrayDataMaker.run_number, + max_events=TestArrayDataMaker.max_events, + ) charges_1 = chargesMaker.make() - wfsMaker_2 = WaveformsMaker(run_number = TestArrayDataMaker.run_number, - max_events = TestArrayDataMaker.max_events) + wfsMaker_2 = WaveformsMaker( + run_number=TestArrayDataMaker.run_number, + max_events=TestArrayDataMaker.max_events, + ) wfs_2 = wfsMaker_2.make() - merged = ArrayDataMaker.merge(charges_1,wfs_2) - - \ No newline at end of file + merged = ArrayDataMaker.merge(charges_1, wfs_2) diff --git a/src/nectarchain/makers/tests/test_waveformsMakers.py b/src/nectarchain/makers/tests/test_waveformsMakers.py index 9a983926..35aaec16 100644 --- a/src/nectarchain/makers/tests/test_waveformsMakers.py +++ b/src/nectarchain/makers/tests/test_waveformsMakers.py @@ -1,34 +1,54 @@ -from nectarchain.makers.waveformsMakers import WaveformsMaker -from nectarchain.data.container import WaveformsContainer,WaveformsContainerIO,ArrayDataContainer -from ctapipe.containers import EventType -import numpy as np import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level = logging.DEBUG) + +import numpy as np +from ctapipe.containers import EventType + +from nectarchain.data.container import ( + ArrayDataContainer, + WaveformsContainer, + WaveformsContainerIO, +) +from nectarchain.makers.waveformsMakers import WaveformsMaker + +logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG +) log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers + class TestWaveformsMaker: run_number = 3938 max_events = 100 - def test_instance(self) : - waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, - max_events = TestWaveformsMaker.max_events) - assert isinstance(waveformsMaker,WaveformsMaker) + def test_instance(self): + waveformsMaker = WaveformsMaker( + run_number=TestWaveformsMaker.run_number, + max_events=TestWaveformsMaker.max_events, + ) + assert isinstance(waveformsMaker, WaveformsMaker) def test_shape_valid(self): - waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, - max_events = TestWaveformsMaker.max_events) + waveformsMaker = WaveformsMaker( + run_number=TestWaveformsMaker.run_number, + max_events=TestWaveformsMaker.max_events, + ) waveformsContainer = waveformsMaker.make()[0] assert waveformsContainer.nevents <= TestWaveformsMaker.max_events assert waveformsContainer.run_number == TestWaveformsMaker.run_number - assert waveformsContainer.ucts_timestamp.shape == (waveformsContainer.nevents,) - assert waveformsContainer.ucts_busy_counter.shape == (waveformsContainer.nevents,) - assert waveformsContainer.ucts_event_counter.shape == (waveformsContainer.nevents,) - assert waveformsContainer.event_type.shape == (waveformsContainer.nevents,) - assert waveformsContainer.event_id.shape == (waveformsContainer.nevents,) - assert waveformsContainer.trig_pattern_all.shape[0] == waveformsContainer.nevents + assert waveformsContainer.ucts_timestamp.shape == (waveformsContainer.nevents,) + assert waveformsContainer.ucts_busy_counter.shape == ( + waveformsContainer.nevents, + ) + assert waveformsContainer.ucts_event_counter.shape == ( + waveformsContainer.nevents, + ) + assert waveformsContainer.event_type.shape == (waveformsContainer.nevents,) + assert waveformsContainer.event_id.shape == (waveformsContainer.nevents,) + assert ( + waveformsContainer.trig_pattern_all.shape[0] == waveformsContainer.nevents + ) assert waveformsContainer.trig_pattern_all.shape[2] == 4 assert waveformsContainer.trig_pattern.shape[0] == waveformsContainer.nevents @@ -36,71 +56,112 @@ def test_shape_valid(self): assert waveformsContainer.wfs_hg.mean() != 0 assert waveformsContainer.wfs_lg.mean() != 0 - assert waveformsContainer.wfs_hg.shape == (waveformsContainer.nevents,waveformsContainer.npixels,waveformsContainer.nsamples) - assert waveformsContainer.wfs_lg.shape == (waveformsContainer.nevents,waveformsContainer.npixels,waveformsContainer.nsamples) - assert waveformsContainer.broken_pixels_hg.shape == (waveformsContainer.nevents,waveformsContainer.npixels) - assert waveformsContainer.broken_pixels_lg.shape == (waveformsContainer.nevents,waveformsContainer.npixels) - + assert waveformsContainer.wfs_hg.shape == ( + waveformsContainer.nevents, + waveformsContainer.npixels, + waveformsContainer.nsamples, + ) + assert waveformsContainer.wfs_lg.shape == ( + waveformsContainer.nevents, + waveformsContainer.npixels, + waveformsContainer.nsamples, + ) + assert waveformsContainer.broken_pixels_hg.shape == ( + waveformsContainer.nevents, + waveformsContainer.npixels, + ) + assert waveformsContainer.broken_pixels_lg.shape == ( + waveformsContainer.nevents, + waveformsContainer.npixels, + ) - - def test_all_multiple_trigger(self) : + def test_all_multiple_trigger(self): trigger1 = EventType.FLATFIELD trigger2 = EventType.SKY_PEDESTAL - waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, - max_events = TestWaveformsMaker.max_events) - waveformsContainer_list = waveformsMaker.make(trigger_type = [trigger1,trigger2],restart_from_begining = True) - for waveformsContainer in waveformsContainer_list : - assert isinstance(waveformsContainer,WaveformsContainer) + waveformsMaker = WaveformsMaker( + run_number=TestWaveformsMaker.run_number, + max_events=TestWaveformsMaker.max_events, + ) + waveformsContainer_list = waveformsMaker.make( + trigger_type=[trigger1, trigger2], restart_from_begining=True + ) + for waveformsContainer in waveformsContainer_list: + assert isinstance(waveformsContainer, WaveformsContainer) assert waveformsContainer.wfs_hg.mean() != 0 - - - def test_all_trigger_None(self) : - waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, - max_events = TestWaveformsMaker.max_events) - waveformsContainer_list = waveformsMaker.make() - assert isinstance(waveformsContainer_list[0],WaveformsContainer) - def test_select_waveforms_hg(self) : - waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, - max_events = TestWaveformsMaker.max_events) + def test_all_trigger_None(self): + waveformsMaker = WaveformsMaker( + run_number=TestWaveformsMaker.run_number, + max_events=TestWaveformsMaker.max_events, + ) waveformsContainer_list = waveformsMaker.make() - pixel_id = np.array([3,67,87]) - assert isinstance(WaveformsMaker.select_waveforms_hg(waveformsContainer_list[0],pixel_id),np.ndarray) - assert isinstance(WaveformsMaker.select_waveforms_lg(waveformsContainer_list[0],pixel_id),np.ndarray) + assert isinstance(waveformsContainer_list[0], WaveformsContainer) + def test_select_waveforms_hg(self): + waveformsMaker = WaveformsMaker( + run_number=TestWaveformsMaker.run_number, + max_events=TestWaveformsMaker.max_events, + ) + waveformsContainer_list = waveformsMaker.make() + pixel_id = np.array([3, 67, 87]) + assert isinstance( + WaveformsMaker.select_waveforms_hg(waveformsContainer_list[0], pixel_id), + np.ndarray, + ) + assert isinstance( + WaveformsMaker.select_waveforms_lg(waveformsContainer_list[0], pixel_id), + np.ndarray, + ) - def test_sort_WaveformsContainer(self) : - waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, - max_events = TestWaveformsMaker.max_events) + def test_sort_WaveformsContainer(self): + waveformsMaker = WaveformsMaker( + run_number=TestWaveformsMaker.run_number, + max_events=TestWaveformsMaker.max_events, + ) waveformsContainer_list = waveformsMaker.make() - sortWfs = WaveformsMaker.sort(waveformsContainer_list[0],method = 'event_id') - assert np.array_equal(sortWfs.event_id ,np.sort(waveformsContainer_list[0].event_id)) + sortWfs = WaveformsMaker.sort(waveformsContainer_list[0], method="event_id") + assert np.array_equal( + sortWfs.event_id, np.sort(waveformsContainer_list[0].event_id) + ) - def test_write_load_container(self) : - waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, - max_events = TestWaveformsMaker.max_events) + def test_write_load_container(self): + waveformsMaker = WaveformsMaker( + run_number=TestWaveformsMaker.run_number, + max_events=TestWaveformsMaker.max_events, + ) waveformsContainer_list = waveformsMaker.make() - WaveformsContainerIO.write("/tmp/test_wfs_container/",waveformsContainer_list[0],overwrite = True) - loaded_wfs = WaveformsContainerIO.load(f"/tmp/test_wfs_container",TestWaveformsMaker.run_number) - assert np.array_equal(waveformsContainer_list[0].wfs_hg,loaded_wfs.wfs_hg) + WaveformsContainerIO.write( + "/tmp/test_wfs_container/", waveformsContainer_list[0], overwrite=True + ) + loaded_wfs = WaveformsContainerIO.load( + f"/tmp/test_wfs_container", TestWaveformsMaker.run_number + ) + assert np.array_equal(waveformsContainer_list[0].wfs_hg, loaded_wfs.wfs_hg) - def test_create_from_events_list(self) : - waveformsMaker = WaveformsMaker(run_number = TestWaveformsMaker.run_number, - max_events = TestWaveformsMaker.max_events) + def test_create_from_events_list(self): + waveformsMaker = WaveformsMaker( + run_number=TestWaveformsMaker.run_number, + max_events=TestWaveformsMaker.max_events, + ) events_list = [] - for i,event in enumerate(waveformsMaker._reader) : + for i, event in enumerate(waveformsMaker._reader): events_list.append(event) - waveformsContainer = WaveformsMaker.create_from_events_list(events_list, - waveformsMaker.run_number, - waveformsMaker.npixels, - waveformsMaker.nsamples, - waveformsMaker.subarray, - waveformsMaker.pixels_id) - assert isinstance(waveformsContainer,WaveformsContainer) + waveformsContainer = WaveformsMaker.create_from_events_list( + events_list, + waveformsMaker.run_number, + waveformsMaker.npixels, + waveformsMaker.nsamples, + waveformsMaker.subarray, + waveformsMaker.pixels_id, + ) + assert isinstance(waveformsContainer, WaveformsContainer) -if __name__ == '__main__' : +if __name__ == "__main__": import logging - logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level = logging.DEBUG) + + logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.DEBUG + ) log = logging.getLogger(__name__) - log.handlers = logging.getLogger('__main__').handlers \ No newline at end of file + log.handlers = logging.getLogger("__main__").handlers diff --git a/src/nectarchain/makers/waveformsMakers.py b/src/nectarchain/makers/waveformsMakers.py index f9ff48e4..013f8c2f 100644 --- a/src/nectarchain/makers/waveformsMakers.py +++ b/src/nectarchain/makers/waveformsMakers.py @@ -1,30 +1,32 @@ import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") log = logging.getLogger(__name__) -log.handlers = logging.getLogger('__main__').handlers +log.handlers = logging.getLogger("__main__").handlers -from argparse import ArgumentError -import numpy as np -from tqdm import tqdm import copy +from argparse import ArgumentError +import numpy as np +from ctapipe.containers import EventType from ctapipe.instrument import SubarrayDescription from ctapipe_io_nectarcam import constants -from ctapipe.containers import EventType from ctapipe_io_nectarcam.containers import NectarCAMDataContainer - +from tqdm import tqdm from ..data.container import WaveformsContainer from .core import ArrayDataMaker __all__ = ["WaveformsMaker"] -class WaveformsMaker(ArrayDataMaker) : - """class use to make the waveform extraction from event read from r0 data - """ -#constructors - def __init__(self,run_number : int,max_events : int = None,run_file = None,*args,**kwargs): +class WaveformsMaker(ArrayDataMaker): + """class use to make the waveform extraction from event read from r0 data""" + + # constructors + def __init__( + self, run_number: int, max_events: int = None, run_file=None, *args, **kwargs + ): """construtor Args: @@ -33,22 +35,23 @@ def __init__(self,run_number : int,max_events : int = None,run_file = None,*args nevents (int, optional) : number of events in run if known (parameter used to save computing time) run_file (optional) : if provided, will load this run file """ - super().__init__(run_number,max_events,run_file,*args,**kwargs) + super().__init__(run_number, max_events, run_file, *args, **kwargs) self.__geometry = self._reader.subarray.tel[__class__.TEL_ID].camera - self.__subarray = self._reader.subarray - + self.__subarray = self._reader.subarray + self.__wfs_hg = {} self.__wfs_lg = {} @staticmethod - def create_from_events_list(events_list : list, - run_number : int, - npixels : int, - nsamples :int, - subarray : SubarrayDescription, - pixels_id : int, - ) : + def create_from_events_list( + events_list: list, + run_number: int, + npixels: int, + nsamples: int, + subarray: SubarrayDescription, + pixels_id: int, + ): """Create a container for the extracted waveforms from a list of events. Args: @@ -63,12 +66,12 @@ def create_from_events_list(events_list : list, WaveformsContainer: A container object that contains the extracted waveforms and other relevant information. """ container = WaveformsContainer( - run_number = run_number, - npixels = npixels, - nsamples = nsamples, - subarray = subarray, - camera = __class__.CAMERA_NAME, - pixels_id = pixels_id, + run_number=run_number, + npixels=npixels, + nsamples=nsamples, + subarray=subarray, + camera=__class__.CAMERA_NAME, + pixels_id=pixels_id, ) ucts_timestamp = [] @@ -80,54 +83,63 @@ def create_from_events_list(events_list : list, wfs_hg = [] wfs_lg = [] - for event in tqdm(events_list) : - ucts_timestamp.append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_timestamp) - ucts_busy_counter.append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_busy_counter) - ucts_event_counter.append(event.nectarcam.tel[__class__.TEL_ID].evt.ucts_event_counter) + for event in tqdm(events_list): + ucts_timestamp.append( + event.nectarcam.tel[__class__.TEL_ID].evt.ucts_timestamp + ) + ucts_busy_counter.append( + event.nectarcam.tel[__class__.TEL_ID].evt.ucts_busy_counter + ) + ucts_event_counter.append( + event.nectarcam.tel[__class__.TEL_ID].evt.ucts_event_counter + ) event_type.append(event.trigger.event_type.value) event_id.append(event.index.event_id) - trig_pattern_all.append(event.nectarcam.tel[__class__.TEL_ID].evt.trigger_pattern.T) + trig_pattern_all.append( + event.nectarcam.tel[__class__.TEL_ID].evt.trigger_pattern.T + ) wfs_hg.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][pixels_id]) wfs_lg.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][pixels_id]) - container.wfs_hg = np.array(wfs_hg,dtype = np.uint16) - container.wfs_lg = np.array(wfs_lg,dtype = np.uint16) - - container.ucts_timestamp = np.array(ucts_timestamp,dtype = np.uint64) - container.ucts_busy_counter = np.array(ucts_busy_counter,dtype = np.uint32) - container.ucts_event_counter = np.array(ucts_event_counter,dtype = np.uint32) - container.event_type = np.array(event_type,dtype = np.uint8) - container.event_id = np.array(event_id,dtype = np.uint32) - container.trig_pattern_all = np.array(trig_pattern_all,dtype =bool ) - container.trig_pattern = container.trig_pattern_all.any(axis = 2) - container.multiplicity = np.uint16(np.count_nonzero(container.trig_pattern,axis = 1)) + container.wfs_hg = np.array(wfs_hg, dtype=np.uint16) + container.wfs_lg = np.array(wfs_lg, dtype=np.uint16) + + container.ucts_timestamp = np.array(ucts_timestamp, dtype=np.uint64) + container.ucts_busy_counter = np.array(ucts_busy_counter, dtype=np.uint32) + container.ucts_event_counter = np.array(ucts_event_counter, dtype=np.uint32) + container.event_type = np.array(event_type, dtype=np.uint8) + container.event_id = np.array(event_id, dtype=np.uint32) + container.trig_pattern_all = np.array(trig_pattern_all, dtype=bool) + container.trig_pattern = container.trig_pattern_all.any(axis=2) + container.multiplicity = np.uint16( + np.count_nonzero(container.trig_pattern, axis=1) + ) - broken_pixels = __class__._compute_broken_pixels(container.wfs_hg,container.wfs_lg) - container.broken_pixels_hg = broken_pixels[0] + broken_pixels = __class__._compute_broken_pixels( + container.wfs_hg, container.wfs_lg + ) + container.broken_pixels_hg = broken_pixels[0] container.broken_pixels_lg = broken_pixels[1] return container - - def _init_trigger_type(self,trigger_type : EventType,**kwargs) : + def _init_trigger_type(self, trigger_type: EventType, **kwargs): """Initialize the waveformsMaker following the trigger type. Args: trigger_type: The type of trigger. """ - super()._init_trigger_type(trigger_type,**kwargs) + super()._init_trigger_type(trigger_type, **kwargs) name = __class__._get_name_trigger(trigger_type) - log.info(f"initialization of the waveformsMaker following trigger type : {name}") + log.info( + f"initialization of the waveformsMaker following trigger type : {name}" + ) self.__wfs_hg[f"{name}"] = [] self.__wfs_lg[f"{name}"] = [] - - - def _make_event(self, - event : NectarCAMDataContainer, - trigger : EventType, - *args, - **kwargs - ) : + + def _make_event( + self, event: NectarCAMDataContainer, trigger: EventType, *args, **kwargs + ): """Process an event and extract waveforms. Args: @@ -135,23 +147,24 @@ def _make_event(self, trigger (EventType): The type of trigger for the event. """ - wfs_hg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) - wfs_lg_tmp=np.zeros((self.npixels,self.nsamples),dtype = np.uint16) + wfs_hg_tmp = np.zeros((self.npixels, self.nsamples), dtype=np.uint16) + wfs_lg_tmp = np.zeros((self.npixels, self.nsamples), dtype=np.uint16) - wfs_hg_tmp,wfs_lg_tmp = super()._make_event(event = event, - trigger = trigger, - return_wfs = True, - *args,**kwargs) + wfs_hg_tmp, wfs_lg_tmp = super()._make_event( + event=event, trigger=trigger, return_wfs=True, *args, **kwargs + ) name = __class__._get_name_trigger(trigger) - self.__wfs_hg[f'{name}'].append(wfs_hg_tmp.tolist()) - self.__wfs_lg[f'{name}'].append(wfs_lg_tmp.tolist()) + self.__wfs_hg[f"{name}"].append(wfs_hg_tmp.tolist()) + self.__wfs_lg[f"{name}"].append(wfs_lg_tmp.tolist()) - broken_pixels_hg,broken_pixels_lg = __class__._compute_broken_pixels_event(event,self._pixels_id) - self._broken_pixels_hg[f'{name}'].append(broken_pixels_hg.tolist()) - self._broken_pixels_lg[f'{name}'].append(broken_pixels_lg.tolist()) + broken_pixels_hg, broken_pixels_lg = __class__._compute_broken_pixels_event( + event, self._pixels_id + ) + self._broken_pixels_hg[f"{name}"].append(broken_pixels_hg.tolist()) + self._broken_pixels_lg[f"{name}"].append(broken_pixels_lg.tolist()) - def _make_output_container(self,trigger_type : EventType,*args,**kwargs) : + def _make_output_container(self, trigger_type: EventType, *args, **kwargs): """Make the output container for the selected trigger types. Args: @@ -161,33 +174,33 @@ def _make_output_container(self,trigger_type : EventType,*args,**kwargs) : list[WaveformsContainer]: A list of output containers for the selected trigger types. """ output = [] - for trigger in trigger_type : + for trigger in trigger_type: waveformsContainer = WaveformsContainer( - run_number = self._run_number, - npixels = self._npixels, - nsamples = self._nsamples, - subarray = self._subarray, - camera = self.CAMERA_NAME, - pixels_id = self._pixels_id, - nevents = self.nevents(trigger), - wfs_hg = self.wfs_hg(trigger), - wfs_lg = self.wfs_lg(trigger), - broken_pixels_hg = self.broken_pixels_hg(trigger), - broken_pixels_lg = self.broken_pixels_lg(trigger), - ucts_timestamp = self.ucts_timestamp(trigger), - ucts_busy_counter = self.ucts_busy_counter(trigger), - ucts_event_counter = self.ucts_event_counter(trigger), - event_type = self.event_type(trigger), - event_id = self.event_id(trigger), - trig_pattern_all = self.trig_pattern_all(trigger), - trig_pattern = self.trig_pattern(trigger), - multiplicity = self.multiplicity(trigger) + run_number=self._run_number, + npixels=self._npixels, + nsamples=self._nsamples, + subarray=self._subarray, + camera=self.CAMERA_NAME, + pixels_id=self._pixels_id, + nevents=self.nevents(trigger), + wfs_hg=self.wfs_hg(trigger), + wfs_lg=self.wfs_lg(trigger), + broken_pixels_hg=self.broken_pixels_hg(trigger), + broken_pixels_lg=self.broken_pixels_lg(trigger), + ucts_timestamp=self.ucts_timestamp(trigger), + ucts_busy_counter=self.ucts_busy_counter(trigger), + ucts_event_counter=self.ucts_event_counter(trigger), + event_type=self.event_type(trigger), + event_id=self.event_id(trigger), + trig_pattern_all=self.trig_pattern_all(trigger), + trig_pattern=self.trig_pattern(trigger), + multiplicity=self.multiplicity(trigger), ) output.append(waveformsContainer) return output @staticmethod - def sort(waveformsContainer :WaveformsContainer, method : str = 'event_id') : + def sort(waveformsContainer: WaveformsContainer, method: str = "event_id"): """Sort the waveformsContainer based on a specified method. Args: @@ -198,25 +211,38 @@ def sort(waveformsContainer :WaveformsContainer, method : str = 'event_id') : WaveformsContainer: The sorted waveformsContainer. """ output = WaveformsContainer( - run_number = waveformsContainer.run_number, - npixels = waveformsContainer.npixels, - nsamples = waveformsContainer.nsamples, - subarray = waveformsContainer.subarray, - camera = waveformsContainer.camera, - pixels_id = waveformsContainer.pixels_id, - nevents = waveformsContainer.nevents + run_number=waveformsContainer.run_number, + npixels=waveformsContainer.npixels, + nsamples=waveformsContainer.nsamples, + subarray=waveformsContainer.subarray, + camera=waveformsContainer.camera, + pixels_id=waveformsContainer.pixels_id, + nevents=waveformsContainer.nevents, ) - if method == 'event_id' : + if method == "event_id": index = np.argsort(waveformsContainer.event_id) - for field in waveformsContainer.keys() : - if not(field in ["run_number","npixels","nsamples","subarray","camera","pixels_id","nevents"]) : + for field in waveformsContainer.keys(): + if not ( + field + in [ + "run_number", + "npixels", + "nsamples", + "subarray", + "camera", + "pixels_id", + "nevents", + ] + ): output[field] = waveformsContainer[field][index] - else : + else: raise ArgumentError(f"{method} is not a valid method for sorting") return output @staticmethod - def select_waveforms_hg(waveformsContainer:WaveformsContainer,pixel_id : np.ndarray) : + def select_waveforms_hg( + waveformsContainer: WaveformsContainer, pixel_id: np.ndarray + ): """Select HIGH GAIN waveforms from the container. Args: @@ -225,13 +251,17 @@ def select_waveforms_hg(waveformsContainer:WaveformsContainer,pixel_id : np.ndar Returns: np.ndarray: An array of selected waveforms from the container. - """ - res = __class__.select_container_array_field(container = waveformsContainer,pixel_id = pixel_id,field = 'wfs_lg') - res = res.transpose(1,0,2) + """ + res = __class__.select_container_array_field( + container=waveformsContainer, pixel_id=pixel_id, field="wfs_lg" + ) + res = res.transpose(1, 0, 2) return res @staticmethod - def select_waveforms_lg(waveformsContainer:WaveformsContainer,pixel_id : np.ndarray) : + def select_waveforms_lg( + waveformsContainer: WaveformsContainer, pixel_id: np.ndarray + ): """Select LOW GAIN waveforms from the container. Args: @@ -240,13 +270,13 @@ def select_waveforms_lg(waveformsContainer:WaveformsContainer,pixel_id : np.ndar Returns: np.ndarray: An array of selected waveforms from the container. - """ - res = __class__.select_container_array_field(container = waveformsContainer,pixel_id = pixel_id,field = 'wfs_hg') - res = res.transpose(1,0,2) + """ + res = __class__.select_container_array_field( + container=waveformsContainer, pixel_id=pixel_id, field="wfs_hg" + ) + res = res.transpose(1, 0, 2) return res - - @property def _geometry(self): """ @@ -264,6 +294,7 @@ def _subarray(self): :return: The value of the private __subarray attribute. """ return self.__subarray + @property def geometry(self): """ @@ -294,7 +325,9 @@ def wfs_hg(self, trigger: EventType): Returns: An array of waveform data for the specified trigger type. """ - return np.array(self.__wfs_hg[__class__._get_name_trigger(trigger)], dtype=np.uint16) + return np.array( + self.__wfs_hg[__class__._get_name_trigger(trigger)], dtype=np.uint16 + ) def wfs_lg(self, trigger: EventType): """ @@ -306,4 +339,6 @@ def wfs_lg(self, trigger: EventType): Returns: An array of waveform data for the specified trigger type in the low gain channel. """ - return np.array(self.__wfs_lg[__class__._get_name_trigger(trigger)], dtype=np.uint16) \ No newline at end of file + return np.array( + self.__wfs_lg[__class__._get_name_trigger(trigger)], dtype=np.uint16 + ) diff --git a/src/nectarchain/tests/test_version.py b/src/nectarchain/tests/test_version.py index b62c0c66..c0c1392d 100644 --- a/src/nectarchain/tests/test_version.py +++ b/src/nectarchain/tests/test_version.py @@ -1,4 +1,4 @@ def test_version(): from nectarchain import __version__ - assert __version__ != '0.0.0' \ No newline at end of file + assert __version__ != "0.0.0" diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py index 8ffff837..44bd47ec 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_PhotoStat_computation.py @@ -1,13 +1,14 @@ import logging -import sys import os +import sys from pathlib import Path + import numpy as np -os.makedirs(os.environ.get('NECTARCHAIN_LOG'),exist_ok = True) +os.makedirs(os.environ.get("NECTARCHAIN_LOG"), exist_ok=True) -#to quiet numba -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s') +# to quiet numba +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") logging.getLogger("numba").setLevel(logging.WARNING) import argparse @@ -16,110 +17,132 @@ from nectarchain.makers.calibration.gain.PhotoStatisticMakers import PhotoStatisticMaker - parser = argparse.ArgumentParser( - prog = 'gain_PhotoStat_computation.py', - description = 'compute high gain and low gain with Photo-statistic method, it need a pedestal run and a FF run with a SPE fit results (for resolution value needed in this method). Output data will be saved in $NECTARCAMDATA/../PhotoStat/data/PhotoStat-FF{FF_run_number}-ped{ped_run_number}-SPEres{SPE_fit_results_tag}-{chargeExtractorPath}/' - ) - -#run numbers -parser.add_argument('-p', '--ped_run_number', - help='ped run', - required=True, - type=int) -parser.add_argument('-f', '--FF_run_number', - help='FF run', - required=True, - type=int) -parser.add_argument('--SPE_fit_results', - help='SPE fit results path for accessing SPE resolution', - type=str, - required=True - ) - -#tag for SPE fit results propagation -parser.add_argument('--SPE_fit_results_tag', - help='SPE fit results tag for propagate the SPE result to output, this tag will be used to setup the path where output data will be saved, see help for description', - type=str, - default='' - ) - -parser.add_argument('--overwrite', - action='store_true', - default=False, - help='to force overwrite files on disk' - ) - -#for plotting correlation -parser.add_argument('--correlation', - action='store_true', - default=True, - help='to plot correlation between SPE gain computation and Photo-statistic gain resluts' - ) - -#extractor arguments -parser.add_argument('--chargeExtractorPath', - help='charge extractor path where charges are saved', - type=str - ) - -parser.add_argument('--FFchargeExtractorWindowLength', - help='charge extractor window length in ns', - type=int - ) - -#verbosity argument -parser.add_argument('-v',"--verbosity", - help='set the verbosity level of logger', - default="info", - choices=["fatal","debug","info","warning"], - type=str) - - -def main(args) : - figpath = os.environ.get('NECTARCHAIN_FIGURES') - + prog="gain_PhotoStat_computation.py", + description="compute high gain and low gain with Photo-statistic method, it need a pedestal run and a FF run with a SPE fit results (for resolution value needed in this method). Output data will be saved in $NECTARCAMDATA/../PhotoStat/data/PhotoStat-FF{FF_run_number}-ped{ped_run_number}-SPEres{SPE_fit_results_tag}-{chargeExtractorPath}/", +) + +# run numbers +parser.add_argument("-p", "--ped_run_number", help="ped run", required=True, type=int) +parser.add_argument("-f", "--FF_run_number", help="FF run", required=True, type=int) +parser.add_argument( + "--SPE_fit_results", + help="SPE fit results path for accessing SPE resolution", + type=str, + required=True, +) + +# tag for SPE fit results propagation +parser.add_argument( + "--SPE_fit_results_tag", + help="SPE fit results tag for propagate the SPE result to output, this tag will be used to setup the path where output data will be saved, see help for description", + type=str, + default="", +) + +parser.add_argument( + "--overwrite", + action="store_true", + default=False, + help="to force overwrite files on disk", +) + +# for plotting correlation +parser.add_argument( + "--correlation", + action="store_true", + default=True, + help="to plot correlation between SPE gain computation and Photo-statistic gain resluts", +) + +# extractor arguments +parser.add_argument( + "--chargeExtractorPath", + help="charge extractor path where charges are saved", + type=str, +) + +parser.add_argument( + "--FFchargeExtractorWindowLength", + help="charge extractor window length in ns", + type=int, +) + +# verbosity argument +parser.add_argument( + "-v", + "--verbosity", + help="set the verbosity level of logger", + default="info", + choices=["fatal", "debug", "info", "warning"], + type=str, +) + + +def main(args): + figpath = os.environ.get("NECTARCHAIN_FIGURES") photoStat_FFandPed = PhotoStatisticMaker.create_from_run_numbers( - FFrun = args.FF_run_number, - Pedrun = args.ped_run_number, - SPE_resolution = args.SPE_fit_results, - method = args.chargeExtractorPath, - FFchargeExtractorWindowLength = args.FFchargeExtractorWindowLength - ) + FFrun=args.FF_run_number, + Pedrun=args.ped_run_number, + SPE_resolution=args.SPE_fit_results, + method=args.chargeExtractorPath, + FFchargeExtractorWindowLength=args.FFchargeExtractorWindowLength, + ) photoStat_FFandPed.make() - photoStat_FFandPed.save(f"{os.environ.get('NECTARCAMDATA')}/../PhotoStat/data/PhotoStat-FF{args.FF_run_number}-ped{args.ped_run_number}-SPEres{args.SPE_fit_results_tag}-{args.chargeExtractorPath}/",overwrite = args.overwrite) + photoStat_FFandPed.save( + f"{os.environ.get('NECTARCAMDATA')}/../PhotoStat/data/PhotoStat-FF{args.FF_run_number}-ped{args.ped_run_number}-SPEres{args.SPE_fit_results_tag}-{args.chargeExtractorPath}/", + overwrite=args.overwrite, + ) log.info(f"BF^2 HG : {np.power(np.mean(photoStat_FFandPed.BHG),2)}") log.info(f"BF^2 LG : {np.power(np.mean(photoStat_FFandPed.BLG),2)}") - if args.correlation : - table = QTable.read(args.SPE_fit_results,format = 'ascii.ecsv') - table.sort('pixels_id') - mask = np.array([pix in photoStat_FFandPed.pixels_id for pix in table['pixels_id'].value],dtype = bool) - fig = PhotoStatisticMaker.plot_correlation(photoStat_FFandPed.results['high_gain'],table['high_gain'][mask]) - os.makedirs(f"{figpath}/PhotoStat-FF{args.FF_run_number}-ped{args.ped_run_number}-{args.chargeExtractorPath}/",exist_ok=True) - fig.savefig(f"{figpath}/PhotoStat-FF{args.FF_run_number}-ped{args.ped_run_number}-{args.chargeExtractorPath}/correlation_PhotoStat_SPE{args.SPE_fit_results_tag}.pdf") + if args.correlation: + table = QTable.read(args.SPE_fit_results, format="ascii.ecsv") + table.sort("pixels_id") + mask = np.array( + [pix in photoStat_FFandPed.pixels_id for pix in table["pixels_id"].value], + dtype=bool, + ) + fig = PhotoStatisticMaker.plot_correlation( + photoStat_FFandPed.results["high_gain"], table["high_gain"][mask] + ) + os.makedirs( + f"{figpath}/PhotoStat-FF{args.FF_run_number}-ped{args.ped_run_number}-{args.chargeExtractorPath}/", + exist_ok=True, + ) + fig.savefig( + f"{figpath}/PhotoStat-FF{args.FF_run_number}-ped{args.ped_run_number}-{args.chargeExtractorPath}/correlation_PhotoStat_SPE{args.SPE_fit_results_tag}.pdf" + ) + if __name__ == "__main__": args = parser.parse_args() logginglevel = logging.FATAL - if args.verbosity == "warning" : + if args.verbosity == "warning": logginglevel = logging.WARNING - elif args.verbosity == "info" : + elif args.verbosity == "info": logginglevel = logging.INFO - elif args.verbosity == "debug" : + elif args.verbosity == "debug": logginglevel = logging.DEBUG os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures") - logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',force = True, level=logginglevel,filename = f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/{Path(__file__).stem}_{os.getpid()}.log") + logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", + force=True, + level=logginglevel, + filename=f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/{Path(__file__).stem}_{os.getpid()}.log", + ) log = logging.getLogger(__name__) log.setLevel(logginglevel) ##tips to add message to stdout handler = logging.StreamHandler(sys.stdout) handler.setLevel(logginglevel) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) handler.setFormatter(formatter) log.addHandler(handler) - main(args) \ No newline at end of file + main(args) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py index 25d8386f..80b90c47 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_combined_computation.py @@ -1,155 +1,176 @@ import logging -import sys import os -from pathlib import Path +import sys import time +from pathlib import Path -os.makedirs(os.environ.get('NECTARCHAIN_LOG'),exist_ok = True) +os.makedirs(os.environ.get("NECTARCHAIN_LOG"), exist_ok=True) -#to quiet numba +# to quiet numba logging.getLogger("numba").setLevel(logging.WARNING) import argparse -#import seaborn as sns +# import seaborn as sns from nectarchain.data.container import ChargeContainer -from nectarchain.makers.calibration.gain.FlatFieldSPEMakers import FlatFieldSingleNominalSPEMaker +from nectarchain.makers.calibration.gain.FlatFieldSPEMakers import ( + FlatFieldSingleNominalSPEMaker, +) parser = argparse.ArgumentParser( - prog = 'gain_SPEfit_combined_computation.py', - description = 'compute high gain with SPE combined fit for one run at nominal voltage. Output data will be saved in $NECTARCAMDATA/../SPEfit/data/{multipath}nominal-prefitCombinedSPE{args.SPE_fit_results_tag}-SPEStd-{args.run_number}-{args.chargeExtractorPath}/' - ) - -#run numbers -parser.add_argument('-r', '--run_number', - help='spe run', - type=int) - -parser.add_argument('--overwrite', - action='store_true', - default=False, - help='to force overwrite files on disk' - ) - -#output figures and path extension -parser.add_argument('--display', - action='store_true', - default=False, - help='whether to save plot or not' - ) -parser.add_argument('--output_fig_tag', - type = str, - default='', - help='tag to set output figure path' - ) - -#pixels selected -parser.add_argument('-p','--pixels', - nargs="+", - default=None, - help='pixels selected', - type=int) - - -#multiprocessing args -parser.add_argument('--multiproc', - action='store_true', - default=False, - help='to use multiprocessing' - ) -parser.add_argument('--nproc', - help='nproc used for multiprocessing', - type=int) -parser.add_argument('--chunksize', - help='chunksize used for multiprocessing, with multiproccesing, create one process per pixels is not optimal, we rather prefer to group quite a few pixels in same process, chunksize is used to set the number of pixels we use for one process, for example if you want to perform the gain computation of the whole camera with 1855 pixels on 6 CPU, a chunksize of 20 seems to be quite optimal ', - type=int) - - -#extractor arguments -parser.add_argument('--chargeExtractorPath', - help='charge extractor path where charges are saved', - type=str - ) - -#for VVH combined fit -parser.add_argument('--combined', - action='store_true', - default=False, - help='if True : perform a combined fit of VVH and nominal data, if False : perform a nominal fit with SPE resoltion fixed from VVH fitted data' - ) -parser.add_argument('--VVH_fitted_results', - help='previoulsy fitted VVH data path for nominal SPE fit by fixing some shared parameters', - type=str - ) -#tag for SPE fit results propagation -parser.add_argument('--SPE_fit_results_tag', - help='SPE fit results tag for propagate the SPE result to output, this tag will be used to setup the path where output data will be saved, see help for description', - type=str, - default='' - ) -parser.add_argument('--same_luminosity', - action='store_true', - default=False, - help='if luminosity for VVH and nominal data is the same' - ) - -#verbosity argument -parser.add_argument('-v',"--verbosity", - help='set the verbosity level of logger', - default="info", - choices=["fatal","debug","info","warning"], - type=str) - - -def main(args) : + prog="gain_SPEfit_combined_computation.py", + description="compute high gain with SPE combined fit for one run at nominal voltage. Output data will be saved in $NECTARCAMDATA/../SPEfit/data/{multipath}nominal-prefitCombinedSPE{args.SPE_fit_results_tag}-SPEStd-{args.run_number}-{args.chargeExtractorPath}/", +) + +# run numbers +parser.add_argument("-r", "--run_number", help="spe run", type=int) + +parser.add_argument( + "--overwrite", + action="store_true", + default=False, + help="to force overwrite files on disk", +) + +# output figures and path extension +parser.add_argument( + "--display", action="store_true", default=False, help="whether to save plot or not" +) +parser.add_argument( + "--output_fig_tag", type=str, default="", help="tag to set output figure path" +) + +# pixels selected +parser.add_argument( + "-p", "--pixels", nargs="+", default=None, help="pixels selected", type=int +) + + +# multiprocessing args +parser.add_argument( + "--multiproc", action="store_true", default=False, help="to use multiprocessing" +) +parser.add_argument("--nproc", help="nproc used for multiprocessing", type=int) +parser.add_argument( + "--chunksize", + help="chunksize used for multiprocessing, with multiproccesing, create one process per pixels is not optimal, we rather prefer to group quite a few pixels in same process, chunksize is used to set the number of pixels we use for one process, for example if you want to perform the gain computation of the whole camera with 1855 pixels on 6 CPU, a chunksize of 20 seems to be quite optimal ", + type=int, +) + + +# extractor arguments +parser.add_argument( + "--chargeExtractorPath", + help="charge extractor path where charges are saved", + type=str, +) + +# for VVH combined fit +parser.add_argument( + "--combined", + action="store_true", + default=False, + help="if True : perform a combined fit of VVH and nominal data, if False : perform a nominal fit with SPE resoltion fixed from VVH fitted data", +) +parser.add_argument( + "--VVH_fitted_results", + help="previoulsy fitted VVH data path for nominal SPE fit by fixing some shared parameters", + type=str, +) +# tag for SPE fit results propagation +parser.add_argument( + "--SPE_fit_results_tag", + help="SPE fit results tag for propagate the SPE result to output, this tag will be used to setup the path where output data will be saved, see help for description", + type=str, + default="", +) +parser.add_argument( + "--same_luminosity", + action="store_true", + default=False, + help="if luminosity for VVH and nominal data is the same", +) + +# verbosity argument +parser.add_argument( + "-v", + "--verbosity", + help="set the verbosity level of logger", + default="info", + choices=["fatal", "debug", "info", "warning"], + type=str, +) + + +def main(args): figpath = f"{os.environ.get('NECTARCHAIN_FIGURES',f'/tmp/nectarchain_log/{os.getpid()}/figure')}/" figpath_ext = "" if args.output_fig_tag == "" else f"-{args.output_fig_tag}" - multipath = "MULTI-" if args.multiproc else "" - charge_run = ChargeContainer.from_file(f"{os.environ.get('NECTARCAMDATA')}/charges/{args.chargeExtractorPath}/",args.run_number) + charge_run = ChargeContainer.from_file( + f"{os.environ.get('NECTARCAMDATA')}/charges/{args.chargeExtractorPath}/", + args.run_number, + ) - if args.combined : + if args.combined: raise NotImplementedError("combined fit not implemented yet") - else : - gain_Std = FlatFieldSingleNominalSPEMaker.create_from_chargeContainer(signal = charge_run, - nectarGainSPEresult=args.VVH_fitted_results, - same_luminosity=args.same_luminosity - ) + else: + gain_Std = FlatFieldSingleNominalSPEMaker.create_from_chargeContainer( + signal=charge_run, + nectarGainSPEresult=args.VVH_fitted_results, + same_luminosity=args.same_luminosity, + ) t = time.time() - gain_Std.make(pixels_id = args.pixels, - multiproc = args.multiproc, - display = args.display, - nproc = args.nproc, - chunksize = args.chunksize, - figpath = figpath+f"/{multipath}nominal-prefitCombinedSPE{args.SPE_fit_results_tag}-SPEStd-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}" - ) - + gain_Std.make( + pixels_id=args.pixels, + multiproc=args.multiproc, + display=args.display, + nproc=args.nproc, + chunksize=args.chunksize, + figpath=figpath + + f"/{multipath}nominal-prefitCombinedSPE{args.SPE_fit_results_tag}-SPEStd-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}", + ) + log.info(f"fit time = {time.time() - t } sec") - gain_Std.save(f"{os.environ.get('NECTARCAMDATA')}/../SPEfit/data/{multipath}nominal-prefitCombinedSPE{args.SPE_fit_results_tag}-SPEStd-{args.run_number}-{args.chargeExtractorPath}/",overwrite = args.overwrite) - log.info(f"convergence rate : {len(gain_Std._results[gain_Std._results['is_valid']])/gain_Std.npixels}") + gain_Std.save( + f"{os.environ.get('NECTARCAMDATA')}/../SPEfit/data/{multipath}nominal-prefitCombinedSPE{args.SPE_fit_results_tag}-SPEStd-{args.run_number}-{args.chargeExtractorPath}/", + overwrite=args.overwrite, + ) + log.info( + f"convergence rate : {len(gain_Std._results[gain_Std._results['is_valid']])/gain_Std.npixels}" + ) + if __name__ == "__main__": args = parser.parse_args() logginglevel = logging.FATAL - if args.verbosity == "warning" : + if args.verbosity == "warning": logginglevel = logging.WARNING - elif args.verbosity == "info" : + elif args.verbosity == "info": logginglevel = logging.INFO - elif args.verbosity == "debug" : + elif args.verbosity == "debug": logginglevel = logging.DEBUG - os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG','/tmp/nectarchain_log')}/{os.getpid()}/figures") - logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',force = True, level=logginglevel,filename = f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/{Path(__file__).stem}_{os.getpid()}.log") - + os.makedirs( + f"{os.environ.get('NECTARCHAIN_LOG','/tmp/nectarchain_log')}/{os.getpid()}/figures" + ) + logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", + force=True, + level=logginglevel, + filename=f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/{Path(__file__).stem}_{os.getpid()}.log", + ) + log = logging.getLogger(__name__) log.setLevel(logginglevel) ##tips to add message to stdout handler = logging.StreamHandler(sys.stdout) handler.setLevel(logginglevel) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) handler.setFormatter(formatter) log.addHandler(handler) - main(args) \ No newline at end of file + main(args) diff --git a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py index 02604694..514a4758 100644 --- a/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py +++ b/src/nectarchain/user_scripts/ggrolleron/gain_SPEfit_computation.py @@ -1,145 +1,162 @@ import logging -import sys import os -from pathlib import Path +import sys import time +from pathlib import Path -os.makedirs(os.environ.get('NECTARCHAIN_LOG'),exist_ok = True) +os.makedirs(os.environ.get("NECTARCHAIN_LOG"), exist_ok=True) -#to quiet numba +# to quiet numba logging.getLogger("numba").setLevel(logging.WARNING) import argparse -#import seaborn as sns +# import seaborn as sns from nectarchain.data.container import ChargeContainer -from nectarchain.makers.calibration.gain.FlatFieldSPEMakers import FlatFieldSingleHHVSPEMaker,FlatFieldSingleHHVStdSPEMaker +from nectarchain.makers.calibration.gain.FlatFieldSPEMakers import ( + FlatFieldSingleHHVSPEMaker, + FlatFieldSingleHHVStdSPEMaker, +) parser = argparse.ArgumentParser( - prog = 'gain_SPEfit_computation.py', - description = 'compute high gain with SPE fit for one run at very very high voltage (~1400V) or at nominal voltage (it can often fail). Output data are saved in $NECTARCAMDATA/../SPEfit/data/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}/' - ) - -#run numbers -parser.add_argument('-r', '--run_number', - help='spe run', - type=int) - -parser.add_argument('--overwrite', - action='store_true', - default=False, - help='to force overwrite files on disk' - ) - -parser.add_argument('--voltage_tag', - type = str, - default='', - help='tag for voltage specifcication (1400V or nominal), used to setup the output path. See help for more details' - ) - -#output figures and path extension -parser.add_argument('--display', - action='store_true', - default=False, - help='whether to save plot or not' - ) -parser.add_argument('--output_fig_tag', - type = str, - default='', - help='tag to set output figure path' - ) - -#pixels selected -parser.add_argument('-p','--pixels', - nargs="+", - default=None, - help='pixels selected', - type=int) - -#for let free pp and n : -parser.add_argument('--free_pp_n', - action='store_true', - default=False, - help='to let free pp and n' - ) - -#multiprocessing args -parser.add_argument('--multiproc', - action='store_true', - default=False, - help='to use multiprocessing' - ) -parser.add_argument('--nproc', - help='nproc used for multiprocessing', - type=int) -parser.add_argument('--chunksize', - help='chunksize used for multiprocessing', - type=int) - -#extractor arguments -parser.add_argument('--chargeExtractorPath', - help='charge extractor path where charges are saved', - type=str - ) - -#verbosity argument -parser.add_argument('-v',"--verbosity", - help='set the verbosity level of logger', - default="info", - choices=["fatal","debug","info","warning"], - type=str) - - -def main(args) : + prog="gain_SPEfit_computation.py", + description="compute high gain with SPE fit for one run at very very high voltage (~1400V) or at nominal voltage (it can often fail). Output data are saved in $NECTARCAMDATA/../SPEfit/data/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}/", +) + +# run numbers +parser.add_argument("-r", "--run_number", help="spe run", type=int) + +parser.add_argument( + "--overwrite", + action="store_true", + default=False, + help="to force overwrite files on disk", +) + +parser.add_argument( + "--voltage_tag", + type=str, + default="", + help="tag for voltage specifcication (1400V or nominal), used to setup the output path. See help for more details", +) + +# output figures and path extension +parser.add_argument( + "--display", action="store_true", default=False, help="whether to save plot or not" +) +parser.add_argument( + "--output_fig_tag", type=str, default="", help="tag to set output figure path" +) + +# pixels selected +parser.add_argument( + "-p", "--pixels", nargs="+", default=None, help="pixels selected", type=int +) + +# for let free pp and n : +parser.add_argument( + "--free_pp_n", action="store_true", default=False, help="to let free pp and n" +) + +# multiprocessing args +parser.add_argument( + "--multiproc", action="store_true", default=False, help="to use multiprocessing" +) +parser.add_argument("--nproc", help="nproc used for multiprocessing", type=int) +parser.add_argument("--chunksize", help="chunksize used for multiprocessing", type=int) + +# extractor arguments +parser.add_argument( + "--chargeExtractorPath", + help="charge extractor path where charges are saved", + type=str, +) + +# verbosity argument +parser.add_argument( + "-v", + "--verbosity", + help="set the verbosity level of logger", + default="info", + choices=["fatal", "debug", "info", "warning"], + type=str, +) + + +def main(args): figpath = f"{os.environ.get('NECTARCHAIN_FIGURES')}/" figpath_ext = "" if args.output_fig_tag == "" else f"-{args.output_fig_tag}" multipath = "MULTI-" if args.multiproc else "" SPEpath = "SPE" if args.free_pp_n else "SPEStd" - charge_run_1400V = ChargeContainer.from_file(f"{os.environ.get('NECTARCAMDATA')}/charges/{args.chargeExtractorPath}/",args.run_number) + charge_run_1400V = ChargeContainer.from_file( + f"{os.environ.get('NECTARCAMDATA')}/charges/{args.chargeExtractorPath}/", + args.run_number, + ) - if args.free_pp_n : - gain_Std = FlatFieldSingleHHVSPEMaker.create_from_chargeContainer(signal = charge_run_1400V) + if args.free_pp_n: + gain_Std = FlatFieldSingleHHVSPEMaker.create_from_chargeContainer( + signal=charge_run_1400V + ) - else : - gain_Std = FlatFieldSingleHHVStdSPEMaker.create_from_chargeContainer(signal = charge_run_1400V) + else: + gain_Std = FlatFieldSingleHHVStdSPEMaker.create_from_chargeContainer( + signal=charge_run_1400V + ) t = time.time() - gain_Std.make(pixels_id = args.pixels, - multiproc = args.multiproc, - display = args.display, - nproc = args.nproc, - chunksize = args.chunksize, - figpath = figpath+f"/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}" - ) - + gain_Std.make( + pixels_id=args.pixels, + multiproc=args.multiproc, + display=args.display, + nproc=args.nproc, + chunksize=args.chunksize, + figpath=figpath + + f"/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}{figpath_ext}", + ) + log.info(f"fit time = {time.time() - t } sec") - gain_Std.save(f"{os.environ.get('NECTARCAMDATA')}/../SPEfit/data/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}/",overwrite = args.overwrite) - conv_rate = len(gain_Std._results[gain_Std._results['is_valid']])/gain_Std.npixels if args.pixels is None else len(gain_Std._results[gain_Std._results['is_valid']])/len(args.pixels) + gain_Std.save( + f"{os.environ.get('NECTARCAMDATA')}/../SPEfit/data/{multipath}{args.voltage_tag}-{SPEpath}-{args.run_number}-{args.chargeExtractorPath}/", + overwrite=args.overwrite, + ) + conv_rate = ( + len(gain_Std._results[gain_Std._results["is_valid"]]) / gain_Std.npixels + if args.pixels is None + else len(gain_Std._results[gain_Std._results["is_valid"]]) / len(args.pixels) + ) log.info(f"convergence rate : {conv_rate}") + if __name__ == "__main__": args = parser.parse_args() logginglevel = logging.FATAL - if args.verbosity == "warning" : + if args.verbosity == "warning": logginglevel = logging.WARNING - elif args.verbosity == "info" : + elif args.verbosity == "info": logginglevel = logging.INFO - elif args.verbosity == "debug" : + elif args.verbosity == "debug": logginglevel = logging.DEBUG os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures") - logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',force = True, level=logginglevel,filename = f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/{Path(__file__).stem}_{os.getpid()}.log") + logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", + force=True, + level=logginglevel, + filename=f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/{Path(__file__).stem}_{os.getpid()}.log", + ) log = logging.getLogger(__name__) log.setLevel(logginglevel) ##tips to add message to stdout handler = logging.StreamHandler(sys.stdout) handler.setLevel(logginglevel) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) handler.setFormatter(formatter) log.addHandler(handler) - main(args) \ No newline at end of file + main(args) diff --git a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py index 7a2ae177..24249618 100644 --- a/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py +++ b/src/nectarchain/user_scripts/ggrolleron/load_wfs_compute_charge.py @@ -1,134 +1,160 @@ -from pathlib import Path -import sys -import os import argparse -import json import glob - +import json import logging +import os +import sys +from pathlib import Path + logging.getLogger("numba").setLevel(logging.WARNING) -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level=logging.DEBUG,filename = f"{os.environ.get('NECTARCHAIN_LOG')}/{Path(__file__).stem}_{os.getpid()}.log") +logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", + level=logging.DEBUG, + filename=f"{os.environ.get('NECTARCHAIN_LOG')}/{Path(__file__).stem}_{os.getpid()}.log", +) log = logging.getLogger(__name__) -from nectarchain.data.container import WaveformsContainer,WaveformsContainers,ChargeContainer,ChargeContainers +from nectarchain.data.container import ( + ChargeContainer, + ChargeContainers, + WaveformsContainer, + WaveformsContainers, +) parser = argparse.ArgumentParser( - prog = 'load_wfs_compute_charge', - description = 'This program load waveforms from fits.fz run files and compute charge') - -#run numbers -parser.add_argument('-s', '--spe_run_number', - nargs="+", - default=[], - help='spe run list', - type=int) -parser.add_argument('-p', '--ped_run_number', - nargs="+", - default=[], - help='ped run list', - type=int) -parser.add_argument('-f', '--ff_run_number', - nargs="+", - default=[], - help='FF run list', - type=int) - -#max events to be loaded -parser.add_argument('--spe_max_events', - nargs="+", - #default=[], - help='spe max events to be load', - type=int) -parser.add_argument('--ped_max_events', - nargs="+", - #default=[], - help='ped max events to be load', - type=int) -parser.add_argument('--ff_max_events', - nargs="+", - #default=[], - help='FF max events to be load', - type=int) - -#n_events in runs -parser.add_argument('--spe_nevents', - nargs="+", - #default=[], - help='spe n events to be load', - type=int) -parser.add_argument('--ped_nevents', - nargs="+", - #default=[], - help='ped n events to be load', - type=int) -parser.add_argument('--ff_nevents', - nargs="+", - #default=[], - help='FF n events to be load', - type=int) - -#boolean arguments -parser.add_argument('--reload_wfs', - action='store_true', - default=False, - help='to force re-computation of waveforms from fits.fz files' - ) -parser.add_argument('--overwrite', - action='store_true', - default=False, - help='to force overwrite files on disk' - ) - -parser.add_argument('--split', - action='store_true', - default=False, - help='split waveforms extraction with 1 file per fits.fz raw data file' - ) - -#extractor arguments -parser.add_argument('--extractorMethod', - choices=["FullWaveformSum", - "FixedWindowSum", - "GlobalPeakWindowSum", - "LocalPeakWindowSum", - "SlidingWindowMaxSum", - "TwoPassWindowSum"], - default="LocalPeakWindowSum", - help='charge extractor method', - type=str - ) -parser.add_argument('--extractor_kwargs', - default={'window_width' : 16, 'window_shift' : 4}, - help='charge extractor kwargs', - type=json.loads - ) - -#verbosity argument -parser.add_argument('-v',"--verbosity", - help='set the verbosity level of logger', - default="info", - choices=["fatal","debug","info","warning"], - type=str) + prog="load_wfs_compute_charge", + description="This program load waveforms from fits.fz run files and compute charge", +) + +# run numbers +parser.add_argument( + "-s", "--spe_run_number", nargs="+", default=[], help="spe run list", type=int +) +parser.add_argument( + "-p", "--ped_run_number", nargs="+", default=[], help="ped run list", type=int +) +parser.add_argument( + "-f", "--ff_run_number", nargs="+", default=[], help="FF run list", type=int +) + +# max events to be loaded +parser.add_argument( + "--spe_max_events", + nargs="+", + # default=[], + help="spe max events to be load", + type=int, +) +parser.add_argument( + "--ped_max_events", + nargs="+", + # default=[], + help="ped max events to be load", + type=int, +) +parser.add_argument( + "--ff_max_events", + nargs="+", + # default=[], + help="FF max events to be load", + type=int, +) + +# n_events in runs +parser.add_argument( + "--spe_nevents", + nargs="+", + # default=[], + help="spe n events to be load", + type=int, +) +parser.add_argument( + "--ped_nevents", + nargs="+", + # default=[], + help="ped n events to be load", + type=int, +) +parser.add_argument( + "--ff_nevents", + nargs="+", + # default=[], + help="FF n events to be load", + type=int, +) + +# boolean arguments +parser.add_argument( + "--reload_wfs", + action="store_true", + default=False, + help="to force re-computation of waveforms from fits.fz files", +) +parser.add_argument( + "--overwrite", + action="store_true", + default=False, + help="to force overwrite files on disk", +) + +parser.add_argument( + "--split", + action="store_true", + default=False, + help="split waveforms extraction with 1 file per fits.fz raw data file", +) + +# extractor arguments +parser.add_argument( + "--extractorMethod", + choices=[ + "FullWaveformSum", + "FixedWindowSum", + "GlobalPeakWindowSum", + "LocalPeakWindowSum", + "SlidingWindowMaxSum", + "TwoPassWindowSum", + ], + default="LocalPeakWindowSum", + help="charge extractor method", + type=str, +) +parser.add_argument( + "--extractor_kwargs", + default={"window_width": 16, "window_shift": 4}, + help="charge extractor kwargs", + type=json.loads, +) + +# verbosity argument +parser.add_argument( + "-v", + "--verbosity", + help="set the verbosity level of logger", + default="info", + choices=["fatal", "debug", "info", "warning"], + type=str, +) args = parser.parse_args() -#control shape of arguments lists -for arg in ['spe','ff','ped'] : +# control shape of arguments lists +for arg in ["spe", "ff", "ped"]: run_number = eval(f"args.{arg}_run_number") max_events = eval(f"args.{arg}_max_events") nevents = eval(f"args.{arg}_nevents") - if not(max_events is None) and len(max_events) != len(run_number) : - e = Exception(f'{arg}_run_number and {arg}_max_events must have same length') - log.error(e,exc_info=True) + if not (max_events is None) and len(max_events) != len(run_number): + e = Exception(f"{arg}_run_number and {arg}_max_events must have same length") + log.error(e, exc_info=True) raise e - if not(nevents is None) and len(nevents) != len(run_number) : - e = Exception(f'{arg}_run_number and {arg}_nevents must have same length') - log.error(e,exc_info=True) + if not (nevents is None) and len(nevents) != len(run_number): + e = Exception(f"{arg}_run_number and {arg}_nevents must have same length") + log.error(e, exc_info=True) raise e - -def load_wfs_no_split(i,runs_list,max_events,nevents,overwrite) : + +def load_wfs_no_split(i, runs_list, max_events, nevents, overwrite): """method to load waveforms without splitting Args: @@ -136,18 +162,21 @@ def load_wfs_no_split(i,runs_list,max_events,nevents,overwrite) : runs_list (list): the run number list max_events (list): max_events list nevents (list): nevents list - overwrite (bool): to overwrite + overwrite (bool): to overwrite Returns: WaveformsContainer: the output waveformsContainer """ log.info("loading wfs not splitted") - wfs = WaveformsContainer(runs_list[i],max_events = max_events[i],nevents = nevents[i]) + wfs = WaveformsContainer(runs_list[i], max_events=max_events[i], nevents=nevents[i]) wfs.load_wfs() - wfs.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",overwrite = overwrite) + wfs.write(f"{os.environ['NECTARCAMDATA']}/waveforms/", overwrite=overwrite) return wfs -def load_wfs_charge_split(i,runs_list,max_events,overwrite,charge_childpath,extractor_kwargs) : + +def load_wfs_charge_split( + i, runs_list, max_events, overwrite, charge_childpath, extractor_kwargs +): """_summary_ Args: @@ -155,37 +184,41 @@ def load_wfs_charge_split(i,runs_list,max_events,overwrite,charge_childpath,extr runs_list (list): the run number list max_events (list): max_events list nevents (list): nevents list - overwrite (bool): to overwrite + overwrite (bool): to overwrite charge_childpath (str): the extraction method extractor_kwargs (dict): the charge extractor kwargs Returns: WaveformsContainers,ChargeContainers: the output WaveformsContainers and ChargeContainers """ - + log.info("splitting wafevorms extraction with raw data list files") log.debug(f"creation of the WaveformsContainers") - wfs = WaveformsContainers(runs_list[i],max_events = max_events[i],init_arrays = False) + wfs = WaveformsContainers(runs_list[i], max_events=max_events[i], init_arrays=False) log.info(f"computation of charge with {charge_childpath}") log.info("splitting charge computation with raw data list files") charge = ChargeContainers() - for j in range(wfs.nWaveformsContainer) : + for j in range(wfs.nWaveformsContainer): log.debug(f"reader events for file {j}") - wfs.load_wfs(index = j) - wfs.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",index = j, overwrite = overwrite) + wfs.load_wfs(index=j) + wfs.write( + f"{os.environ['NECTARCAMDATA']}/waveforms/", index=j, overwrite=overwrite + ) log.debug(f"computation of charge for file {j}") - charge.append(ChargeContainer.from_waveforms(wfs.waveformsContainer[j], - method = charge_childpath, - **extractor_kwargs)) + charge.append( + ChargeContainer.from_waveforms( + wfs.waveformsContainer[j], method=charge_childpath, **extractor_kwargs + ) + ) log.debug(f"deleting waveformsContainer at index {j} to free RAM") wfs.waveformsContainer[j] = WaveformsContainer.__new__(WaveformsContainer) - - + log.info("merging charge") charge = charge.merge() - return wfs,charge + return wfs, charge -def load_wfs_charge_split_from_wfsFiles(wfsFiles,charge_childpath,extractor_kwargs) : + +def load_wfs_charge_split_from_wfsFiles(wfsFiles, charge_childpath, extractor_kwargs): """_summary_ Args: @@ -197,13 +230,15 @@ def load_wfs_charge_split_from_wfsFiles(wfsFiles,charge_childpath,extractor_kwar None,ChargeContainers: the output ChargeContainers (return tuple with None to keep same structure as load_wfs_charge_split) """ charge = ChargeContainers() - for j,file in enumerate(wfsFiles): + for j, file in enumerate(wfsFiles): log.debug(f"loading wfs from file {file}") wfs = WaveformsContainer.load(file) log.debug(f"computation of charge for file {file}") - charge.append(ChargeContainer.from_waveforms(wfs, - method = charge_childpath, - **extractor_kwargs)) + charge.append( + ChargeContainer.from_waveforms( + wfs, method=charge_childpath, **extractor_kwargs + ) + ) log.debug(f"deleting waveformsContainer from {file} to free RAM") del wfs.wfs_hg del wfs.wfs_lg @@ -214,17 +249,20 @@ def load_wfs_charge_split_from_wfsFiles(wfsFiles,charge_childpath,extractor_kwar del wfs.event_id del wfs.trig_pattern_all del wfs - #gc.collect() - + # gc.collect() + log.info("merging charge") charge = charge.merge() - return None,charge + return None, charge -def load_wfs_compute_charge(runs_list : list, - reload_wfs : bool = False, - overwrite : bool= False, - charge_extraction_method : str = "FullWaveformSum", - **kwargs) -> None : + +def load_wfs_compute_charge( + runs_list: list, + reload_wfs: bool = False, + overwrite: bool = False, + charge_extraction_method: str = "FullWaveformSum", + **kwargs, +) -> None: """this method is used to load waveforms from zfits files and compute charge with an user specified method Args: @@ -237,119 +275,159 @@ def load_wfs_compute_charge(runs_list : list, e : an error occurred during zfits loading from ctapipe EventSource """ - #print(runs_list) - #print(charge_extraction_method) - #print(overwrite) - #print(reload_wfs) - #print(kwargs) - - - max_events = kwargs.get("max_events",[None for i in range(len(runs_list))]) - nevents = kwargs.get("nevents",[-1 for i in range(len(runs_list))]) + # print(runs_list) + # print(charge_extraction_method) + # print(overwrite) + # print(reload_wfs) + # print(kwargs) + + max_events = kwargs.get("max_events", [None for i in range(len(runs_list))]) + nevents = kwargs.get("nevents", [-1 for i in range(len(runs_list))]) - charge_childpath = kwargs.get("charge_childpath",charge_extraction_method) - extractor_kwargs = kwargs.get("extractor_kwargs",{}) + charge_childpath = kwargs.get("charge_childpath", charge_extraction_method) + extractor_kwargs = kwargs.get("extractor_kwargs", {}) - split = kwargs.get("split",False) - + split = kwargs.get("split", False) - for i in range(len(runs_list)) : + for i in range(len(runs_list)): log.info(f"treating run {runs_list[i]}") log.info("waveform computation") - if not(reload_wfs): - log.info(f"trying to load waveforms from {os.environ['NECTARCAMDATA']}/waveforms/") - try : - if split : - files = glob.glob(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{runs_list[i]}_*.fits") - if len(files) == 0 : + if not (reload_wfs): + log.info( + f"trying to load waveforms from {os.environ['NECTARCAMDATA']}/waveforms/" + ) + try: + if split: + files = glob.glob( + f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{runs_list[i]}_*.fits" + ) + if len(files) == 0: raise FileNotFoundError(f"no splitted waveforms found") - else : - wfs,charge = load_wfs_charge_split_from_wfsFiles(files,charge_childpath,extractor_kwargs) - - else : - wfs = WaveformsContainer.load(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{runs_list[i]}.fits") - except FileNotFoundError as e : - log.warning(f"argument said to not reload waveforms from zfits files but computed waveforms not found at {os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{runs_list[i]}.fits") + else: + wfs, charge = load_wfs_charge_split_from_wfsFiles( + files, charge_childpath, extractor_kwargs + ) + + else: + wfs = WaveformsContainer.load( + f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{runs_list[i]}.fits" + ) + except FileNotFoundError as e: + log.warning( + f"argument said to not reload waveforms from zfits files but computed waveforms not found at {os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{runs_list[i]}.fits" + ) log.warning(f"reloading from zfits files") - if split : - wfs,charge = load_wfs_charge_split(i,runs_list,max_events,overwrite,charge_childpath,extractor_kwargs) - else : - wfs = load_wfs_no_split(i,runs_list,max_events,nevents,overwrite) - except Exception as e : - log.error(e,exc_info = True) + if split: + wfs, charge = load_wfs_charge_split( + i, + runs_list, + max_events, + overwrite, + charge_childpath, + extractor_kwargs, + ) + else: + wfs = load_wfs_no_split( + i, runs_list, max_events, nevents, overwrite + ) + except Exception as e: + log.error(e, exc_info=True) raise e - else : - if split : - wfs,charge = load_wfs_charge_split(i,runs_list,max_events,overwrite,charge_childpath,extractor_kwargs) - else : - wfs = load_wfs_no_split(i,runs_list,max_events,nevents,overwrite) - - - if not(split) : + else: + if split: + wfs, charge = load_wfs_charge_split( + i, + runs_list, + max_events, + overwrite, + charge_childpath, + extractor_kwargs, + ) + else: + wfs = load_wfs_no_split(i, runs_list, max_events, nevents, overwrite) + + if not (split): log.info(f"computation of charge with {charge_childpath}") - charge = ChargeContainer.from_waveforms(wfs,method = charge_childpath,**extractor_kwargs) + charge = ChargeContainer.from_waveforms( + wfs, method=charge_childpath, **extractor_kwargs + ) del wfs - charge.write(f"{os.environ['NECTARCAMDATA']}/charges/{path}/",overwrite = overwrite) + charge.write( + f"{os.environ['NECTARCAMDATA']}/charges/{path}/", overwrite=overwrite + ) del charge - - -def main(spe_run_number : list = [], - ff_run_number : list = [], - ped_run_number: list = [], - **kwargs): - - #print(kwargs) - spe_nevents = kwargs.pop('spe_nevents',[-1 for i in range(len(spe_run_number))]) - ff_nevents = kwargs.pop('ff_nevents',[-1 for i in range(len(ff_run_number))]) - ped_nevents = kwargs.pop('ped_nevents',[-1 for i in range(len(ped_run_number))]) - spe_max_events = kwargs.pop('spe_max_events',[None for i in range(len(spe_run_number))]) - ff_max_events = kwargs.pop('ff_max_events',[None for i in range(len(ff_run_number))]) - ped_max_events = kwargs.pop('ped_max_events',[None for i in range(len(ped_run_number))]) +def main( + spe_run_number: list = [], + ff_run_number: list = [], + ped_run_number: list = [], + **kwargs, +): + # print(kwargs) + + spe_nevents = kwargs.pop("spe_nevents", [-1 for i in range(len(spe_run_number))]) + ff_nevents = kwargs.pop("ff_nevents", [-1 for i in range(len(ff_run_number))]) + ped_nevents = kwargs.pop("ped_nevents", [-1 for i in range(len(ped_run_number))]) + + spe_max_events = kwargs.pop( + "spe_max_events", [None for i in range(len(spe_run_number))] + ) + ff_max_events = kwargs.pop( + "ff_max_events", [None for i in range(len(ff_run_number))] + ) + ped_max_events = kwargs.pop( + "ped_max_events", [None for i in range(len(ped_run_number))] + ) runs_list = spe_run_number + ff_run_number + ped_run_number nevents = spe_nevents + ff_nevents + ped_nevents max_events = spe_max_events + ff_max_events + ped_max_events - charge_extraction_method = kwargs.get('extractorMethod',"FullWaveformSum") - - + charge_extraction_method = kwargs.get("extractorMethod", "FullWaveformSum") - load_wfs_compute_charge(runs_list = runs_list, - charge_extraction_method = charge_extraction_method, - nevents = nevents, - max_events = max_events, - **kwargs) + load_wfs_compute_charge( + runs_list=runs_list, + charge_extraction_method=charge_extraction_method, + nevents=nevents, + max_events=max_events, + **kwargs, + ) -if __name__ == '__main__': - - #run of interest - #spe_run_number = [2633,2634,3784] - #ff_run_number = [2608] - #ped_run_number = [2609] - #spe_nevents = [49227,49148,-1] +if __name__ == "__main__": + # run of interest + # spe_run_number = [2633,2634,3784] + # ff_run_number = [2608] + # ped_run_number = [2609] + # spe_nevents = [49227,49148,-1] args = parser.parse_args() logginglevel = logging.FATAL - if args.verbosity == "warning" : + if args.verbosity == "warning": logginglevel = logging.WARNING - elif args.verbosity == "info" : + elif args.verbosity == "info": logginglevel = logging.INFO - elif args.verbosity == "debug" : + elif args.verbosity == "debug": logginglevel = logging.DEBUG os.makedirs(f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/figures") - logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',force = True, level=logginglevel,filename = f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/{Path(__file__).stem}_{os.getpid()}.log") + logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", + force=True, + level=logginglevel, + filename=f"{os.environ.get('NECTARCHAIN_LOG')}/{os.getpid()}/{Path(__file__).stem}_{os.getpid()}.log", + ) log = logging.getLogger(__name__) log.setLevel(logginglevel) ##tips to add message to stdout handler = logging.StreamHandler(sys.stdout) handler.setLevel(logginglevel) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) handler.setFormatter(formatter) log.addHandler(handler) @@ -357,23 +435,23 @@ def main(spe_run_number : list = [], log.info(f"arguments are : {arg}") key_to_pop = [] - for key in arg.keys() : - if arg[key] is None : + for key in arg.keys(): + if arg[key] is None: key_to_pop.append(key) - for key in key_to_pop : + for key in key_to_pop: arg.pop(key) log.info(f"arguments passed to main are : {arg}") - - path= args.extractorMethod - if args.extractorMethod in ["GlobalPeakWindowSum", "LocalPeakWindowSum"] : - path +=f"_{args.extractor_kwargs['window_shift']}-{args.extractor_kwargs['window_width']-args.extractor_kwargs['window_shift']}" - elif args.extractorMethod in ["SlidingWindowMaxSum"] : - path +=f"_{args.extractor_kwargs['window_width']}" - elif args.extractorMethod in ["FixedWindowSum"] : - path +=f"_{args.extractor_kwargs['peak_index']}_{args.extractor_kwargs['window_shift']}-{args.extractor_kwargs['window_width']-args.extractor_kwargs['window_shift']}" - - arg['path'] = path - + + path = args.extractorMethod + if args.extractorMethod in ["GlobalPeakWindowSum", "LocalPeakWindowSum"]: + path += f"_{args.extractor_kwargs['window_shift']}-{args.extractor_kwargs['window_width']-args.extractor_kwargs['window_shift']}" + elif args.extractorMethod in ["SlidingWindowMaxSum"]: + path += f"_{args.extractor_kwargs['window_width']}" + elif args.extractorMethod in ["FixedWindowSum"]: + path += f"_{args.extractor_kwargs['peak_index']}_{args.extractor_kwargs['window_shift']}-{args.extractor_kwargs['window_width']-args.extractor_kwargs['window_shift']}" + + arg["path"] = path + main(**arg) diff --git a/src/nectarchain/user_scripts/ggrolleron/test.py b/src/nectarchain/user_scripts/ggrolleron/test.py index 637ae3d3..031c2841 100644 --- a/src/nectarchain/user_scripts/ggrolleron/test.py +++ b/src/nectarchain/user_scripts/ggrolleron/test.py @@ -1,122 +1,139 @@ -import numpy as np -#import pandas as pd +import logging +import os +import sys +import time +from pathlib import Path + +# import pandas as pd import matplotlib as mpl -import matplotlib.pyplot as plt import matplotlib.cm as cm -from pathlib import Path -import sys,os -import time -import logging -logging.basicConfig(format='%(asctime)s %(name)s %(levelname)s %(message)s',level=logging.INFO) +import matplotlib.pyplot as plt +import numpy as np + +logging.basicConfig( + format="%(asctime)s %(name)s %(levelname)s %(message)s", level=logging.INFO +) log = logging.getLogger(__name__) ##tips to add message to stdout handler = logging.StreamHandler(sys.stdout) handler.setLevel(logging.INFO) -formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) log.addHandler(handler) import glob - - -from nectarchain.data.container import ChargeContainer,WaveformsContainer,ChargeContainers,WaveformsContainers +from nectarchain.data.container import ( + ChargeContainer, + ChargeContainers, + WaveformsContainer, + WaveformsContainers, +) from nectarchain.data.container.utils import DataManagement -def test_check_wfs() : +def test_check_wfs(): run_number = [3731] - files = glob.glob(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}_*.fits") - if len(files) == 0 : + files = glob.glob( + f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}_*.fits" + ) + if len(files) == 0: raise FileNotFoundError(f"no splitted waveforms found") - else : + else: charge_container = ChargeContainers() - for j,file in enumerate(files): + for j, file in enumerate(files): log.debug(f"loading wfs from file {file}") wfs = WaveformsContainer.load(file) print(f"min : {wfs.wfs_hg.min()}") - #fig,ax = wfs.plot_waveform_hg(0) - #for i in range(wfs.nevents) : + # fig,ax = wfs.plot_waveform_hg(0) + # for i in range(wfs.nevents) : # wfs.plot_waveform_hg(i,figure = fig,ax = ax) log.debug(f"computation of charge for file {file}") - charge = ChargeContainer.from_waveforms(wfs, - method = "LocalPeakWindowSum", - window_width = 16, - window_shift = 4) + charge = ChargeContainer.from_waveforms( + wfs, method="LocalPeakWindowSum", window_width=16, window_shift=4 + ) hist = charge.histo_hg() charge_container.append(charge) - -def test_extractor() : +def test_extractor(): run_number = [2633] - - wfs = WaveformsContainer(run_number[0],max_events = 200) + wfs = WaveformsContainer(run_number[0], max_events=200) # wfs.load_wfs() # - #spe_run_1000V.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",overwrite = True) + # spe_run_1000V.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",overwrite = True) - #wfs = WaveformsContainer.load(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}.fits") + # wfs = WaveformsContainer.load(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}.fits") - #charge = ChargeContainer.compute_charge(spe_run_1000V,1,method = "gradient_extractor",) + # charge = ChargeContainer.compute_charge(spe_run_1000V,1,method = "gradient_extractor",) t = time.time() - charge = ChargeContainer.from_waveforms(wfs, method = "LocalPeakWindowSum", window_width = 16, window_shift = 4) + charge = ChargeContainer.from_waveforms( + wfs, method="LocalPeakWindowSum", window_width=16, window_shift=4 + ) log.info(f"LocalPeakWindowSum duration : {time.time() - t} seconds") - charge = ChargeContainer.from_waveforms(wfs, method = "GlobalPeakWindowSum", window_width = 16, window_shift = 4) + charge = ChargeContainer.from_waveforms( + wfs, method="GlobalPeakWindowSum", window_width=16, window_shift=4 + ) log.info(f"GlobalPeakWindowSum duration : {time.time() - t} seconds") - charge = ChargeContainer.from_waveforms(wfs, method = "FullWaveformSum", window_width = 16, window_shift = 4) + charge = ChargeContainer.from_waveforms( + wfs, method="FullWaveformSum", window_width=16, window_shift=4 + ) log.info(f"FullWaveformSum duration : {time.time() - t} seconds") - charge = ChargeContainer.from_waveforms(wfs, method = "FixedWindowSum", window_width = 16, window_shift = 4, peak_index = 30) + charge = ChargeContainer.from_waveforms( + wfs, method="FixedWindowSum", window_width=16, window_shift=4, peak_index=30 + ) log.info(f"FullWindowSum duration : {time.time() - t} seconds") - charge = ChargeContainer.from_waveforms(wfs, method = "SlidingWindowMaxSum", window_width = 16, window_shift = 4) + charge = ChargeContainer.from_waveforms( + wfs, method="SlidingWindowMaxSum", window_width=16, window_shift=4 + ) log.info(f"SlidingWindowMaxSum duration : {time.time() - t} seconds") - #charge = ChargeContainer.from_waveforms(wfs, method = "NeighborPeakWindowSum", window_width = 16, window_shift = 4) - #log.info(f"NeighborPeakWindowSum duration : {time.time() - t} seconds") + # charge = ChargeContainer.from_waveforms(wfs, method = "NeighborPeakWindowSum", window_width = 16, window_shift = 4) + # log.info(f"NeighborPeakWindowSum duration : {time.time() - t} seconds") - #charge = ChargeContainer.from_waveforms(wfs, method = "BaselineSubtractedNeighborPeakWindowSum", baseline_start = 2, baseline_end = 12, window_width = 16, window_shift = 4) - #log.info(f"BaselineSubtractedNeighborPeakWindowSum duration : {time.time() - t} seconds") + # charge = ChargeContainer.from_waveforms(wfs, method = "BaselineSubtractedNeighborPeakWindowSum", baseline_start = 2, baseline_end = 12, window_width = 16, window_shift = 4) + # log.info(f"BaselineSubtractedNeighborPeakWindowSum duration : {time.time() - t} seconds") - charge = ChargeContainer.from_waveforms(wfs, method = "TwoPassWindowSum", window_width = 16, window_shift = 4) + charge = ChargeContainer.from_waveforms( + wfs, method="TwoPassWindowSum", window_width=16, window_shift=4 + ) log.info(f"TwoPassWindowSum duration : {time.time() - t} seconds") - #charge.write(f"{os.environ['NECTARCAMDATA']}/charges/std/",overwrite = False) - - + # charge.write(f"{os.environ['NECTARCAMDATA']}/charges/std/",overwrite = False) -def test_simplecontainer() : +def test_simplecontainer(): run_number = [2633] - spe_run_1000V = WaveformsContainer(run_number[0],max_events = 1000) + spe_run_1000V = WaveformsContainer(run_number[0], max_events=1000) spe_run_1000V.load_wfs() - #spe_run_1000V.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",overwrite = True) - - #spe_run_1000V.load(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}.fits") - - charge = ChargeContainer.compute_charge(spe_run_1000V,1,method = "LocalPeakWindowSum",extractor_kwargs = {'window_width' : 16, 'window_shift' : 4}) + # spe_run_1000V.write(f"{os.environ['NECTARCAMDATA']}/waveforms/",overwrite = True) + # spe_run_1000V.load(f"{os.environ['NECTARCAMDATA']}/waveforms/waveforms_run{run_number[0]}.fits") + charge = ChargeContainer.compute_charge( + spe_run_1000V, + 1, + method="LocalPeakWindowSum", + extractor_kwargs={"window_width": 16, "window_shift": 4}, + ) charge = ChargeContainer.from_waveforms(spe_run_1000V) - - #charge.write(f"{os.environ['NECTARCAMDATA']}/charges/std/",overwrite = True) - - + # charge.write(f"{os.environ['NECTARCAMDATA']}/charges/std/",overwrite = True) -if __name__ == "__main__" : +if __name__ == "__main__": test_check_wfs() print("work completed") From d0a196361e418cbfb922fcc29b737cd3f866c128 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Sun, 17 Sep 2023 15:38:04 +0200 Subject: [PATCH 57/62] set Dirac import into methods which need it --- src/nectarchain/data/management.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py index 6ac3f6f0..bc8dc171 100644 --- a/src/nectarchain/data/management.py +++ b/src/nectarchain/data/management.py @@ -12,7 +12,6 @@ import browser_cookie3 import mechanize import requests -from DIRAC.Interfaces.API.Dirac import Dirac __all__ = ["DataManagement"] @@ -68,6 +67,7 @@ def getRunFromDIRAC(lfns: list): Args: lfns (list): list of lfns path """ + from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() for lfn in lfns: @@ -96,14 +96,14 @@ def get_GRID_location( url = "http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?cmd=Find" - url_run = f"http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?mode=full&reverse=0&reverse=1&npp=20&subtext=%23{run_number}" + # url_run = f"http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?mode=full&reverse=0&reverse=1&npp=20&subtext=%23{run_number}" if not (username is None or password is None): log.debug("log to Elog with username and password") # log to Elog br = mechanize.Browser() br.open(url) - form = br.select_form("form1") + # form = br.select_form("form1") for i in range(4): log.debug(br.form.find_control(nr=i).name) br.form["uname"] = username @@ -148,6 +148,8 @@ def get_GRID_location( lfns = [] try: # Dirac + from DIRAC.Interfaces.API.Dirac import Dirac + dirac = Dirac() loc = f"/vo.cta.in2p3.fr/nectarcam/{url_data.split('/')[-2]}/{url_data.split('/')[-1]}" log.debug(f"searching in Dirac filecatalog at {loc}") From 272b0c53e423d726da0512467dc6d33a5372df0e Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 16 Oct 2023 21:05:08 +0200 Subject: [PATCH 58/62] first step to reimplement following ctapipe Tool and Components framework --- src/nectarchain/data/management.py | 7 +- src/nectarchain/makers/__init__.py | 2 +- .../makers/calibration/__init__.py | 6 +- src/nectarchain/makers/core.py | 305 ++++++++++-------- src/nectarchain/makers/waveformsMakers.py | 28 +- 5 files changed, 178 insertions(+), 170 deletions(-) diff --git a/src/nectarchain/data/management.py b/src/nectarchain/data/management.py index bc8dc171..b524bc40 100644 --- a/src/nectarchain/data/management.py +++ b/src/nectarchain/data/management.py @@ -9,10 +9,6 @@ from pathlib import Path from typing import List, Tuple -import browser_cookie3 -import mechanize -import requests - __all__ = ["DataManagement"] @@ -93,6 +89,9 @@ def get_GRID_location( Returns: _type_: _description_ """ + import browser_cookie3 + import mechanize + import requests url = "http://nectarcam.in2p3.fr/elog/nectarcam-data-qm/?cmd=Find" diff --git a/src/nectarchain/makers/__init__.py b/src/nectarchain/makers/__init__.py index fa4ef370..a59e6de1 100644 --- a/src/nectarchain/makers/__init__.py +++ b/src/nectarchain/makers/__init__.py @@ -1,3 +1,3 @@ -from .chargesMakers import * +# from .chargesMakers import * from .core import * from .waveformsMakers import * diff --git a/src/nectarchain/makers/calibration/__init__.py b/src/nectarchain/makers/calibration/__init__.py index 0ded83f5..81f96f56 100644 --- a/src/nectarchain/makers/calibration/__init__.py +++ b/src/nectarchain/makers/calibration/__init__.py @@ -1,3 +1,3 @@ -from .flatfieldMakers import * -from .gain import * -from .pedestalMakers import * +# from .flatfieldMakers import * +# from .gain import * +# from .pedestalMakers import * diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 30a9260f..7fc3a4d3 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -5,44 +5,48 @@ log.handlers = logging.getLogger("__main__").handlers import copy +import pathlib from abc import ABC, abstractmethod import numpy as np from ctapipe.containers import EventType +from ctapipe.core import Tool +from ctapipe.core.traits import Bool, Integer, Path, classes_with_traits, flag from ctapipe.instrument import CameraGeometry +from ctapipe.io import HDF5TableWriter +from ctapipe.io.datawriter import DATA_MODEL_VERSION from ctapipe_io_nectarcam import NectarCAMEventSource, constants from ctapipe_io_nectarcam.containers import NectarCAMDataContainer +from tqdm.auto import tqdm from ..data import DataManagement from ..data.container.core import ArrayDataContainer -__all__ = ["ArrayDataMaker"] +__all__ = ["ArrayDataNectarCAMCalibrationTool"] """The code snippet is a part of a class hierarchy for data processing. It includes the `BaseMaker` abstract class, the `EventsLoopMaker` and `ArrayDataMaker` subclasses. These classes are used to perform computations on data from a specific run.""" -class BaseMaker(ABC): +class BaseNectarCAMCalibrationTool(Tool): """Mother class for all the makers, the role of makers is to do computation on the data.""" - @abstractmethod - def make(self, *args, **kwargs): - """ - Abstract method that needs to be implemented by subclasses. - This method is the main one, which computes and does the work. - """ - pass + name = "BaseNectarCAMCalibration" + + progress_bar = Bool( + help="show progress bar during processing", default_value=False + ).tag(config=True) @staticmethod def load_run( - run_number: int, max_events: int = None, run_file=None + run_number: int, max_events: int = None, run_file: str = None ) -> NectarCAMEventSource: """Static method to load from $NECTARCAMDATA directory data for specified run with max_events Args:self.__run_number = run_number run_number (int): run_id - maxevents (int, optional): max of events to be loaded. Defaults to 0, to load everythings. + maxevents (int, optional): max of events to be loaded. Defaults to -1, to load everythings. run_file (optional) : if provided, will load this run file Returns: List[ctapipe_io_nectarcam.NectarCAMEventSource]: List of EventSource for each run files @@ -62,7 +66,7 @@ def load_run( return eventsource -class EventsLoopMaker(BaseMaker): +class EventsLoopNectarCAMCalibrationTool(BaseNectarCAMCalibrationTool): """ A class for data processing and computation on events from a specific run. @@ -74,57 +78,97 @@ class EventsLoopMaker(BaseMaker): Example Usage: maker = EventsLoopMaker(run_number=1234, max_events=1000) maker.make(n_events=500) - """ - def __init__( - self, run_number: int, max_events: int = None, run_file=None, *args, **kwargs - ): - """ - Constructor method that initializes the EventsLoopMaker object. - Args: - run_number (int): The ID of the run to be loaded. - max_events (int, optional): The maximum number of events to be loaded. Defaults to None. - run_file (optional): The specific run file to be loaded. - """ - super().__init__(*args, **kwargs) - - self.__run_number = run_number - self.__run_file = run_file - self.__max_events = max_events - - self.__reader = __class__.load_run(run_number, max_events, run_file=run_file) + """ - # from reader members - self.__npixels = self.__reader.camera_config.num_pixels - self.__pixels_id = self.__reader.camera_config.expected_pixels_id + name = "EventsLoopNectarCAMCalibration" + + description = ( + __doc__ + f" This currently uses data model version {DATA_MODEL_VERSION}" + ) + examples = """To be implemented""" + + aliases = { + ("i", "input"): "EventsLoopNectarCAMCalibrationTool.run_file", + ("r", "run-number"): "EventsLoopNectarCAMCalibrationTool.run_number", + ("m", "max-events"): "EventsLoopNectarCAMCalibrationTool.max_events", + } + + flags = { + "overwrite": ( + {"HDF5TableWriter": {"overwrite": True}}, + "Overwrite output file if it exists", + ), + **flag( + "progress", + "ProcessorTool.progress_bar", + "show a progress bar during event processing", + "don't show a progress bar during event processing", + ), + } + + classes = classes_with_traits(NectarCAMEventSource) + + run_number = Integer(help="run number to be treated", default_value=-1).tag( + config=True + ) + + max_events = Integer( + help="maximum number of events to be loaded", + default_value=None, + allow_none=True, + ).tag(config=True) + + run_file = Path( + help="file name to be loaded", + default_value=None, + allow_none=True, + ).tag(config=True) + + def _load_eventsource(self): + self.event_source = self.enter_context( + self.load_run(self.run_number, self.max_events, run_file=self.run_file) + ) - log.info(f"N pixels : {self.npixels}") + def setup(self, *args, **kwargs): + if self.run_number == -1: + raise Exception("run_number need to be set up") + self._load_eventsource() + self.__npixels = self._event_source.camera_config.num_pixels + self.__pixels_id = self._event_source.camera_config.expected_pixels_id - def make( - self, n_events=np.inf, restart_from_beginning: bool = False, *args, **kwargs - ): - """ - Method to iterate over the events and perform computations on each event. + # self.comp = MyComponent(parent=self) + # self.comp2 = SecondaryMyComponent(parent=self) + # self.comp3 = TelescopeWiseComponent(parent=self, subarray=subarray) + # self.advanced = AdvancedComponent(parent=self) - Args: - n_events (int, optional): The number of events to process. Defaults to np.inf. - restart_from_beginning (bool, optional): Whether to restart from the beginning of the run. Defaults to False. - """ + """ + def start(self, n_events=np.inf, restart_from_beginning: bool = False,*args,**kwargs): if restart_from_beginning: - log.debug("restart from beginning : creation of the EventSource reader") - self.__reader = __class__.load_run( - self.__run_number, self.__max_events, run_file=self.__run_file - ) + self.log.debug("restart from beginning : creation of the EventSource reader") + self._load_eventsource() + + #self.event_source.subarray.info(printer=self.log.info) n_traited_events = 0 - for i, event in enumerate(self.__reader): + for i,event in enumerate(tqdm( + self._event_source, + desc=self._event_source.__class__.__name__, + total=min(self._event_source.max_events,n_events), + unit="ev", + disable=not self.progress_bar, + )): if i % 100 == 0: - log.info(f"reading event number {i}") + self.log.info(f"reading event number {i}") self._make_event(event, *args, **kwargs) n_traited_events += 1 if n_traited_events >= n_events: break + """ + + def finish(self): + self.log.warning("Shutting down.") @abstractmethod def _make_event(self, event: NectarCAMDataContainer): @@ -138,28 +182,14 @@ def _make_event(self, event: NectarCAMDataContainer): pass @property - def _run_file(self): - """ - Getter method for the _run_file attribute. - """ - return self.__run_file - - @property - def _max_events(self): - """ - Getter method for the _max_events attribute. - """ - return self.__max_events - - @property - def _reader(self): + def event_source(self): """ - Getter method for the _reader attribute. + Getter method for the _event_source attribute. """ - return self.__reader + return copy.deepcopy(self._event_source) - @_reader.setter - def _reader(self, value): + @event_source.setter + def event_source(self, value): """ Setter method to set a new NectarCAMEventSource to the _reader attribute. @@ -167,7 +197,7 @@ def _reader(self, value): value: a NectarCAMEventSource instance. """ if isinstance(value, NectarCAMEventSource): - self.__reader = value + self._event_source = value else: raise TypeError("The reader must be a NectarCAMEventSource") @@ -185,20 +215,6 @@ def _pixels_id(self): """ return self.__pixels_id - @property - def _run_number(self): - """ - Getter method for the _run_number attribute. - """ - return self.__run_number - - @property - def reader(self): - """ - Getter method for the reader attribute. - """ - return copy.deepcopy(self.__reader) - @property def npixels(self): """ @@ -213,62 +229,51 @@ def pixels_id(self): """ return copy.deepcopy(self.__pixels_id) - @property - def run_number(self): - """ - Getter method for the run_number attribute. - """ - return copy.deepcopy(self.__run_number) +def main(): + """run the tool""" + tool = EventsLoopNectarCAMCalibrationTool() + tool.run() -class ArrayDataMaker(EventsLoopMaker): - """ - Class used to loop over the events of a run and to extract informations that are stored in arrays. - Example Usage: - - Create an instance of the ArrayDataMaker class - maker = ArrayDataMaker(run_number=1234, max_events=1000) - - - Perform data processing on the specified run - maker.make(n_events=500, trigger_type=[EventType.SKY]) - - - Access the computed data - ucts_timestamp = maker.ucts_timestamp(EventType.SKY) - event_type = maker.event_type(EventType.SKY) - - Inputs: - - run_number (int): The ID of the run to be processed. - - max_events (int, optional): The maximum number of events to be loaded. Defaults to None, which loads all events. - - run_file (optional): The specific run file to be loaded. - - Flow: - 1. The ArrayDataMaker class is initialized with the run number, maximum events, or a run file. - 2. The make method is called to perform data processing on the specified run. - 3. The _make_event method is called for each event in the run to extract and store relevant data. - 4. The computed data is stored in instance variables. - 5. The computed data can be accessed using getter methods. - - Outputs: - - Computed data such as UCTS timestamps, event types, and event IDs can be accessed using getter methods provided by the ArrayDataMaker class. - """ - TEL_ID = 0 - CAMERA_NAME = "NectarCam-003" - CAMERA = CameraGeometry.from_name(CAMERA_NAME) +if __name__ == "__main__": + main() - def __init__( - self, run_number: int, max_events: int = None, run_file=None, *args, **kwargs - ): - """construtor - Args: - run_number (int): id of the run to be loaded - maxevents (int, optional): max of events to be loaded. Defaults to 0, to load everythings. - nevents (int, optional) : number of events in run if known (parameter used to save computing time) - run_file (optional) : if provided, will load this run file - """ - super().__init__(run_number, max_events, run_file, *args, **kwargs) - self.__nsamples = self._reader.camera_config.num_samples +class ArrayDataNectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): + name = "ArrayDataNectarCAMCalibration" + + # description = ( + # __doc__ + f" This currently uses data model version {DATA_MODEL_VERSION}" + # ) + examples = """To be implemented""" + + aliases = super( + EventsLoopNectarCAMCalibrationTool, EventsLoopNectarCAMCalibrationTool + ).aliases + aliases.update( + { + ("o", "output"): "HDF5TableWriter.output_path", + } + ) + + classes = super( + EventsLoopNectarCAMCalibrationTool, EventsLoopNectarCAMCalibrationTool + ).classes + ( + [ + HDF5TableWriter, + ] + ) + + TEL_ID = int(0) + + CAMERA_NAME = str("NectarCam-003") + + CAMERA = CameraGeometry.from_name(CAMERA_NAME) + def setup(self, *args, **kwargs): + super().setup(*args, **kwargs) + self.__nsamples = self._event_source.camera_config.num_samples # data we want to compute self.__ucts_timestamp = {} self.__ucts_busy_counter = {} @@ -348,7 +353,7 @@ def _get_name_trigger(trigger: EventType): name = trigger.name return name - def make( + def start( self, n_events=np.inf, trigger_type: list = None, @@ -370,7 +375,7 @@ def make( The output container created by the _make_output_container method. """ if ~np.isfinite(n_events): - log.warning( + self.log.warning( "no needed events number specified, it may cause a memory error" ) if isinstance(trigger_type, EventType) or trigger_type is None: @@ -379,15 +384,21 @@ def make( self._init_trigger_type(_trigger_type) if restart_from_begining: - log.debug("restart from begining : creation of the EventSource reader") - self._reader = __class__.load_run( - self._run_number, self._max_events, run_file=self._run_file - ) + self.log.debug("restart from begining : creation of the EventSource reader") + self._load_eventsource() n_traited_events = 0 - for i, event in enumerate(self._reader): + for i, event in enumerate( + tqdm( + self._event_source, + desc=self._event_source.__class__.__name__, + total=min(self._event_source.max_events, n_events), + unit="ev", + disable=not self.progress_bar, + ) + ): if i % 100 == 0: - log.info(f"reading event number {i}") + self.log.info(f"reading event number {i}") for trigger in trigger_type: if (trigger is None) or (trigger == event.trigger.event_type): self._make_event(event, trigger, *args, **kwargs) @@ -395,8 +406,6 @@ def make( if n_traited_events >= n_events: break - return self._make_output_container(trigger_type, *args, **kwargs) - def _make_event( self, event: NectarCAMDataContainer, trigger: EventType, *args, **kwargs ): @@ -433,6 +442,16 @@ def _make_event( get_wfs_lg = event.r0.tel[0].waveform[constants.LOW_GAIN][self.pixels_id] return get_wfs_hg, get_wfs_lg + def finish(self, trigger_type: list = None, *args, **kwargs): + # self.write = self.enter_context( + # HDF5TableWriter(filename=filename, parent=self) + # ) + if isinstance(trigger_type, EventType) or trigger_type is None: + trigger_type = [trigger_type] + output_container = self._make_output_container(trigger_type, *args, **kwargs) + self.log.info(f"{output_container}") + super().finish() + @abstractmethod def _make_output_container(self): pass diff --git a/src/nectarchain/makers/waveformsMakers.py b/src/nectarchain/makers/waveformsMakers.py index 013f8c2f..4a3bc474 100644 --- a/src/nectarchain/makers/waveformsMakers.py +++ b/src/nectarchain/makers/waveformsMakers.py @@ -15,30 +15,20 @@ from tqdm import tqdm from ..data.container import WaveformsContainer -from .core import ArrayDataMaker +from .core import ArrayDataNectarCAMCalibrationTool -__all__ = ["WaveformsMaker"] +__all__ = ["WaveformsNectarCAMCalibrationTool"] -class WaveformsMaker(ArrayDataMaker): +class WaveformsNectarCAMCalibrationTool(ArrayDataNectarCAMCalibrationTool): """class use to make the waveform extraction from event read from r0 data""" # constructors - def __init__( - self, run_number: int, max_events: int = None, run_file=None, *args, **kwargs - ): - """construtor + def setup(self, *args, **kwargs): + super().setup(*args, **kwargs) - Args: - run_number (int): id of the run to be loaded - maxevents (int, optional): max of events to be loaded. Defaults to 0, to load everythings. - nevents (int, optional) : number of events in run if known (parameter used to save computing time) - run_file (optional) : if provided, will load this run file - """ - super().__init__(run_number, max_events, run_file, *args, **kwargs) - - self.__geometry = self._reader.subarray.tel[__class__.TEL_ID].camera - self.__subarray = self._reader.subarray + self.__geometry = self._event_source.subarray.tel[__class__.TEL_ID].camera + self.__subarray = self._event_source.subarray self.__wfs_hg = {} self.__wfs_lg = {} @@ -51,7 +41,7 @@ def create_from_events_list( nsamples: int, subarray: SubarrayDescription, pixels_id: int, - ): + ) -> WaveformsContainer: """Create a container for the extracted waveforms from a list of events. Args: @@ -176,7 +166,7 @@ def _make_output_container(self, trigger_type: EventType, *args, **kwargs): output = [] for trigger in trigger_type: waveformsContainer = WaveformsContainer( - run_number=self._run_number, + run_number=self.run_number, npixels=self._npixels, nsamples=self._nsamples, subarray=self._subarray, From 878bb96902350d36323fdc2ce37dcc78e96860e4 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 23 Oct 2023 14:32:11 +0200 Subject: [PATCH 59/62] -removed the I/O method for containers -> now we will use ctapipe HDF5TableWriter and Reader -creation of recursive containers --- .../data/container/chargesContainer.py | 202 +--------------- src/nectarchain/data/container/core.py | 36 +-- .../data/container/waveformsContainer.py | 224 +----------------- 3 files changed, 41 insertions(+), 421 deletions(-) diff --git a/src/nectarchain/data/container/chargesContainer.py b/src/nectarchain/data/container/chargesContainer.py index 1d985850..42fdad88 100644 --- a/src/nectarchain/data/container/chargesContainer.py +++ b/src/nectarchain/data/container/chargesContainer.py @@ -10,10 +10,9 @@ import numpy as np from astropy.io import fits -from ctapipe.containers import Field - -from .core import ArrayDataContainer +from ctapipe.containers import Field,partial,Map +from .core import ArrayDataContainer,TriggerMapContainer class ChargesContainer(ArrayDataContainer): """ @@ -27,194 +26,13 @@ class ChargesContainer(ArrayDataContainer): method (str): The charge extraction method used. """ - charges_hg = Field(type=np.ndarray, description="The high gain charges") - charges_lg = Field(type=np.ndarray, description="The low gain charges") - peak_hg = Field(type=np.ndarray, description="The high gain peak time") - peak_lg = Field(type=np.ndarray, description="The low gain peak time") + charges_hg = Field(type=np.ndarray, dtype = np.uint16, ndim = 2, description="The high gain charges") + charges_lg = Field(type=np.ndarray, dtype = np.uint16, ndim = 2, description="The low gain charges") + peak_hg = Field(type=np.ndarray, dtype = np.uint16, ndim = 2, description="The high gain peak time") + peak_lg = Field(type=np.ndarray, dtype = np.uint16, ndim = 2, description="The low gain peak time") method = Field(type=str, description="The charge extraction method used") - -class ChargesContainerIO(ABC): - """ - The `ChargesContainerIO` class provides methods for writing and loading `ChargesContainer` instances to/from FITS files. - - Example Usage: - # Writing a ChargesContainer instance to a FITS file - chargesContainer = ChargesContainer() - ChargesContainerIO.write(path, chargesContainer) - - # Loading a ChargesContainer instance from a FITS file - chargesContainer = ChargesContainerIO.load(path, run_number) - - Main functionalities: - - Writing a `ChargesContainer` instance to a FITS file. - - Loading a `ChargesContainer` instance from a FITS file. - - Methods: - - `write(path: Path, container: ChargesContainer, **kwargs) -> None`: Writes a `ChargesContainer` instance to a FITS file. The method takes a file path and the `ChargesContainer` instance as input. Additional keyword arguments can be provided to customize the file name and overwrite behavior. - - `load(path: Path, run_number: int, **kwargs) -> ChargesContainer`: Loads a `ChargesContainer` instance from a FITS file. The method takes a file path and the run number as input. Additional keyword arguments can be provided to specify an explicit file name. - - Fields: - The `ChargesContainerIO` class does not have any fields. - """ - - @staticmethod - def write(path: Path, container: ChargesContainer, **kwargs) -> None: - """Write a ChargesContainer instance to a FITS file. - Args: - path (str): The directory where the FITS file will be saved. - container (ChargesContainer): The ChargesContainer instance to be written to the FITS file. - **kwargs: Additional keyword arguments for customization. - Keyword Args: - suffix (str): A suffix to be added to the file name (default: ""). - overwrite (bool): Whether to overwrite the file if it already exists (default: False). - Returns: - None: This method does not return any value. - Raises: - OSError: If there is an error while writing the FITS file. - Exception: If there is any other exception during the writing process. - Example: - chargesContainer = ChargesContainer() - ChargesContainerIO.write(path, chargesContainer, suffix="v1", overwrite=True) - """ - suffix = kwargs.get("suffix", "") - if suffix != "": - suffix = f"_{suffix}" - log.info(f"saving in {path}") - os.makedirs(path, exist_ok=True) - hdr = fits.Header() - hdr["RUN"] = container.run_number - hdr["NEVENTS"] = container.nevents - hdr["NPIXELS"] = container.npixels - hdr["METHOD"] = container.method - hdr["CAMERA"] = container.camera - hdr[ - "COMMENT" - ] = f"The charge containeur for run {container.run_number} with {container.method} method : primary is the pixels id, then you can find HG charge, LG charge, HG peak and LG peak, 2 last HDU are composed of event properties and trigger patern" - - primary_hdu = fits.PrimaryHDU(container.pixels_id, header=hdr) - charge_hg_hdu = fits.ImageHDU(container.charges_hg, name="HG charge") - charge_lg_hdu = fits.ImageHDU(container.charges_lg, name="LG charge") - peak_hg_hdu = fits.ImageHDU(container.peak_hg, name="HG peak time") - peak_lg_hdu = fits.ImageHDU(container.peak_lg, name="LG peak time") - - col1 = fits.Column( - array=container.broken_pixels_hg, - name="HG broken pixels", - format=f"{container.broken_pixels_hg.shape[1]}L", - ) - col2 = fits.Column( - array=container.broken_pixels_lg, - name="LG broken pixels", - format=f"{container.broken_pixels_lg.shape[1]}L", - ) - coldefs = fits.ColDefs([col1, col2]) - broken_pixels = fits.BinTableHDU.from_columns(coldefs, name="trigger patern") - - col1 = fits.Column(array=container.event_id, name="event_id", format="1I") - col2 = fits.Column(array=container.event_type, name="event_type", format="1I") - col3 = fits.Column( - array=container.ucts_timestamp, name="ucts_timestamp", format="1K" - ) - col4 = fits.Column( - array=container.ucts_busy_counter, name="ucts_busy_counter", format="1I" - ) - col5 = fits.Column( - array=container.ucts_event_counter, name="ucts_event_counter", format="1I" - ) - col6 = fits.Column( - array=container.multiplicity, name="multiplicity", format="1I" - ) - coldefs = fits.ColDefs([col1, col2, col3, col4, col5, col6]) - event_properties = fits.BinTableHDU.from_columns( - coldefs, name="event properties" - ) - - col1 = fits.Column( - array=container.trig_pattern_all, - name="trig_pattern_all", - format=f"{4 * container.trig_pattern_all.shape[1]}L", - dim=f"({ container.trig_pattern_all.shape[1]},4)", - ) - col2 = fits.Column( - array=container.trig_pattern, - name="trig_pattern", - format=f"{container.trig_pattern_all.shape[1]}L", - ) - coldefs = fits.ColDefs([col1, col2]) - trigger_patern = fits.BinTableHDU.from_columns(coldefs, name="trigger patern") - - hdul = fits.HDUList( - [ - primary_hdu, - charge_hg_hdu, - charge_lg_hdu, - peak_hg_hdu, - peak_lg_hdu, - broken_pixels, - event_properties, - trigger_patern, - ] - ) - try: - hdul.writeto( - Path(path) / f"charge_run{container.run_number}{suffix}.fits", - overwrite=kwargs.get("overwrite", False), - ) - log.info( - f"charge saved in {Path(path)}/charge_run{container.run_number}{suffix}.fits" - ) - except OSError as e: - log.warning(e) - except Exception as e: - log.error(e, exc_info=True) - raise e - - @staticmethod - def load(path: str, run_number: int, **kwargs) -> ChargesContainer: - """Load a ChargesContainer instance from a FITS file. - This method opens a FITS file and retrieves the necessary data to create a ChargesContainer instance. - The FITS file should have been previously written using the write method of the ChargesContainerIO class. - Args: - path (str): The path of the FITS file. - run_number (int): The run number. - **kwargs: Additional keyword arguments. - explicit_filename (str): If provided, the explicit filename to load. - Returns: - ChargesContainer: The loaded ChargesContainer instance. - Example: - chargesContainer = ChargesContainerIO.load(path, run_number) - """ - if kwargs.get("explicit_filename", False): - filename = kwargs.get("explicit_filename") - log.info(f"Loading {filename}") - else: - log.info(f"Loading in {path} run number {run_number}") - filename = Path(path) / f"charge_run{run_number}.fits" - - with fits.open(filename) as hdul: - container = ChargesContainer() - container.run_number = hdul[0].header["RUN"] - container.nevents = hdul[0].header["NEVENTS"] - container.npixels = hdul[0].header["NPIXELS"] - container.method = hdul[0].header["METHOD"] - container.camera = hdul[0].header["CAMERA"] - container.pixels_id = hdul[0].data - container.charges_hg = hdul[1].data - container.charges_lg = hdul[2].data - container.peak_hg = hdul[3].data - container.peak_lg = hdul[4].data - broken_pixels = hdul[5].data - container.broken_pixels_hg = broken_pixels["HG broken pixels"] - container.broken_pixels_lg = broken_pixels["LG broken pixels"] - table_prop = hdul[6].data - container.event_id = table_prop["event_id"] - container.event_type = table_prop["event_type"] - container.ucts_timestamp = table_prop["ucts_timestamp"] - container.ucts_busy_counter = table_prop["ucts_busy_counter"] - container.ucts_event_counter = table_prop["ucts_event_counter"] - container.multiplicity = table_prop["multiplicity"] - table_trigger = hdul[7].data - container.trig_pattern_all = table_trigger["trig_pattern_all"] - container.trig_pattern = table_trigger["trig_pattern"] - return container +class ChargesContainers(TriggerMapContainer): + containers = Field(default_factory=partial(Map, ChargesContainer), + description = "trigger mapping of ChargesContainer" + ) \ No newline at end of file diff --git a/src/nectarchain/data/container/core.py b/src/nectarchain/data/container/core.py index d034246e..ca6dffd4 100644 --- a/src/nectarchain/data/container/core.py +++ b/src/nectarchain/data/container/core.py @@ -6,12 +6,15 @@ log.handlers = logging.getLogger("__main__").handlers import numpy as np -from ctapipe.containers import Container, Field +from ctapipe.containers import Field,Container,partial,Map -__all__ = ["ArrayDataContainer"] +__all__ = ["ArrayDataContainer","TriggerMapContainer"] -class ArrayDataContainer(Container): +class NectarCAMContainer(Container) : + """base class for the NectarCAM containers. This contaner cannot berecursive, to be directly written with a HDF5TableWriter """ + +class ArrayDataContainer(NectarCAMContainer): """ A container that holds information about waveforms from a specific run. @@ -45,18 +48,23 @@ class ArrayDataContainer(Container): type=int, description="number of effective pixels", ) - pixels_id = Field(type=np.ndarray, description="pixel ids") - broken_pixels_hg = Field(type=np.ndarray, description="high gain broken pixels") - broken_pixels_lg = Field(type=np.ndarray, description="low gain broken pixels") + pixels_id = Field(type=np.ndarray, dtype = np.uint16, ndim = 1,description="pixel ids") + broken_pixels_hg = Field(type=np.ndarray, dtype = bool, ndim = 2, description="high gain broken pixels") + broken_pixels_lg = Field(type=np.ndarray, dtype = bool, ndim = 2, description="low gain broken pixels") camera = Field( type=str, description="camera name", ) - ucts_timestamp = Field(type=np.ndarray, description="events ucts timestamp") - ucts_busy_counter = Field(type=np.ndarray, description="ucts busy counter") - ucts_event_counter = Field(type=np.ndarray, description="ucts event counter") - event_type = Field(type=np.ndarray, description="trigger event type") - event_id = Field(type=np.ndarray, description="event ids") - trig_pattern_all = Field(type=np.ndarray, description="trigger pattern") - trig_pattern = Field(type=np.ndarray, description="reduced trigger pattern") - multiplicity = Field(type=np.ndarray, description="events multiplicity") + ucts_timestamp = Field(type=np.ndarray, dtype = np.uint64, ndim = 1, description="events ucts timestamp") + ucts_busy_counter = Field(type=np.ndarray, dtype = np.uint32, ndim = 1, description="ucts busy counter") + ucts_event_counter = Field(type=np.ndarray, dtype = np.uint32, ndim = 1, description="ucts event counter") + event_type = Field(type=np.ndarray, dtype = np.uint8, ndim = 1, description="trigger event type") + event_id = Field(type=np.ndarray, dtype = np.uint32, ndim = 1, description="event ids") + trig_pattern_all = Field(type=np.ndarray, dtype = bool, ndim = 3, description="trigger pattern") + trig_pattern = Field(type=np.ndarray, dtype = bool, ndim = 2, description="reduced trigger pattern") + multiplicity = Field(type=np.ndarray, dtype = np.uint16, ndim = 1, description="events multiplicity") + +class TriggerMapContainer(Container) : + containers = Field(default_factory=partial(Map, Container), + description = "trigger mapping of Container" + ) diff --git a/src/nectarchain/data/container/waveformsContainer.py b/src/nectarchain/data/container/waveformsContainer.py index 13037ea3..0eda692c 100644 --- a/src/nectarchain/data/container/waveformsContainer.py +++ b/src/nectarchain/data/container/waveformsContainer.py @@ -14,11 +14,11 @@ import numpy as np from astropy.io import fits from astropy.table import Column, QTable, Table -from ctapipe.containers import Field +from ctapipe.containers import Field,Container,partial,Map from ctapipe.instrument.subarray import SubarrayDescription from tqdm import tqdm -from .core import ArrayDataContainer +from .core import ArrayDataContainer,TriggerMapContainer class WaveformsContainer(ArrayDataContainer): @@ -36,219 +36,13 @@ class WaveformsContainer(ArrayDataContainer): type=int, description="number of samples in the waveforms", ) - subarray = Field(type=SubarrayDescription, description="The subarray description") - wfs_hg = Field(type=np.ndarray, description="high gain waveforms") - wfs_lg = Field(type=np.ndarray, description="low gain waveforms") + #subarray = Field(type=SubarrayDescription, description="The subarray description") + wfs_hg = Field(type=np.ndarray, dtype = np.uint16, ndim = 3, description="high gain waveforms") + wfs_lg = Field(type=np.ndarray, dtype = np.uint16, ndim = 3, description="low gain waveforms") -class WaveformsContainerIO(ABC): - """ - The `WaveformsContainerIO` class provides methods for writing and loading `WaveformsContainer` instances to/from FITS files. It also includes a method for writing the subarray configuration to an HDF5 file. - - Example Usage: - # Writing a WaveformsContainer instance to a FITS file - container = WaveformsContainer() - # ... populate the container with data ... - WaveformsContainerIO.write('/path/to/output', container) - - # Loading a WaveformsContainer instance from a FITS file - container = WaveformsContainerIO.load('/path/to/input.fits') - - Main functionalities: - - Writing a `WaveformsContainer` instance to a FITS file, including the waveforms data and metadata. - - Loading a `WaveformsContainer` instance from a FITS file, including the waveforms data and metadata. - - Writing the subarray configuration to an HDF5 file. - - Methods: - - `write(path: str, containers: WaveformsContainer, **kwargs) -> None`: Writes a `WaveformsContainer` instance to a FITS file. The method also creates an HDF5 file representing the subarray configuration. - - `load(path: str) -> WaveformsContainer`: Loads a `WaveformsContainer` instance from a FITS file. The method also loads the subarray configuration from the corresponding HDF5 file. - - Fields: - None. - """ - - @staticmethod - def write(path: str, containers: WaveformsContainer, **kwargs) -> None: - """Write the WaveformsContainer data to an output FITS file. - - This method creates two files: one FITS file representing the waveform data and one HDF5 file representing the subarray configuration. - - Args: - path (str): The directory where you want to save the data. - containers (WaveformsContainer): The WaveformsContainer instance containing the data to be saved. - **kwargs: Additional keyword arguments for customization (optional). - - Keyword Args: - suffix (str, optional): A suffix to append to the output file names (default is ''). - overwrite (bool, optional): If True, overwrite the output files if they already exist (default is False). - Returns: - None: This method does not return any value. - Raises: - OSError: If there is an error while writing the FITS file. - Exception: If there is any other exception during the writing process. - Example: - waveformsContainer = WaveformsContainer() - WaveformsContainerIO.write(path, waveformsContainer, suffix="v1", overwrite=True) - """ - suffix = kwargs.get("suffix", "") - if suffix != "": - suffix = f"_{suffix}" - - log.info(f"saving in {path}") - os.makedirs(path, exist_ok=True) - - hdr = fits.Header() - hdr["RUN"] = containers.run_number - hdr["NEVENTS"] = containers.nevents - hdr["NPIXELS"] = containers.npixels - hdr["NSAMPLES"] = containers.nsamples - hdr["SUBARRAY"] = containers.subarray.name - hdr["CAMERA"] = containers.camera - - containers.subarray.to_hdf( - f"{Path(path)}/subarray_run{containers.run_number}{suffix}.hdf5", - overwrite=kwargs.get("overwrite", False), - ) - - hdr[ - "COMMENT" - ] = f"The waveforms containeur for run {containers.run_number} : primary is the pixels id, 2nd HDU : high gain waveforms, 3rd HDU : low gain waveforms, 4th HDU : event properties and 5th HDU trigger paterns." - - primary_hdu = fits.PrimaryHDU(containers.pixels_id, header=hdr) - - wfs_hg_hdu = fits.ImageHDU(containers.wfs_hg, name="HG Waveforms") - wfs_lg_hdu = fits.ImageHDU(containers.wfs_lg, name="LG Waveforms") - - col1 = fits.Column( - array=containers.broken_pixels_hg, - name="HG broken pixels", - format=f"{containers.broken_pixels_hg.shape[1]}L", - ) - col2 = fits.Column( - array=containers.broken_pixels_lg, - name="LG broken pixels", - format=f"{containers.broken_pixels_lg.shape[1]}L", - ) - coldefs = fits.ColDefs([col1, col2]) - broken_pixels = fits.BinTableHDU.from_columns(coldefs, name="trigger patern") - - col1 = fits.Column(array=containers.event_id, name="event_id", format="1J") - col2 = fits.Column(array=containers.event_type, name="event_type", format="1I") - col3 = fits.Column( - array=containers.ucts_timestamp, name="ucts_timestamp", format="1K" - ) - col4 = fits.Column( - array=containers.ucts_busy_counter, name="ucts_busy_counter", format="1J" - ) - col5 = fits.Column( - array=containers.ucts_event_counter, name="ucts_event_counter", format="1J" - ) - col6 = fits.Column( - array=containers.multiplicity, name="multiplicity", format="1I" - ) - - coldefs = fits.ColDefs([col1, col2, col3, col4, col5, col6]) - event_properties = fits.BinTableHDU.from_columns( - coldefs, name="event properties" - ) - - col1 = fits.Column( - array=containers.trig_pattern_all, - name="trig_pattern_all", - format=f"{4 * containers.trig_pattern_all.shape[1]}L", - dim=f"({containers.trig_pattern_all.shape[1]},4)", - ) - col2 = fits.Column( - array=containers.trig_pattern, - name="trig_pattern", - format=f"{containers.trig_pattern_all.shape[1]}L", - ) - coldefs = fits.ColDefs([col1, col2]) - trigger_patern = fits.BinTableHDU.from_columns(coldefs, name="trigger patern") - - hdul = fits.HDUList( - [ - primary_hdu, - wfs_hg_hdu, - wfs_lg_hdu, - broken_pixels, - event_properties, - trigger_patern, - ] - ) - try: - hdul.writeto( - Path(path) / f"waveforms_run{containers.run_number}{suffix}.fits", - overwrite=kwargs.get("overwrite", False), - ) - log.info( - f"run saved in {Path(path)}/waveforms_run{containers.run_number}{suffix}.fits" - ) - except OSError as e: - log.warning(e) - except Exception as e: - log.error(e, exc_info=True) - raise e - - @staticmethod - def load(path: str, run_number: int, **kwargs) -> WaveformsContainer: - """Load a WaveformsContainer from a FITS file previously written with WaveformsContainerIO.write() method. - - Note: Two files are loaded—the FITS file representing the waveform data and an HDF5 file representing the subarray configuration. - The HDF5 file should be located next to the FITS file. - - Args: - path (str): The path to the FITS file containing the waveform data. - **kwargs: Additional keyword arguments. - explicit_filename (str): If provided, the explicit filename to load. - Returns: - WaveformsContainer: A WaveformsContainer instance loaded from the specified file. - Example: - waveformsContainer = WaveformsContainerIO.load(path, run_number) - """ - if kwargs.get("explicit_filename", False): - filename = kwargs.get("explicit_filename") - log.info(f"Loading {filename}") - else: - log.info(f"Loading in {path} run number {run_number}") - filename = Path(path) / f"waveforms_run{run_number}.fits" - - log.info(f"loading from {path}") - with fits.open(filename) as hdul: - containers = WaveformsContainer() - - containers.run_number = hdul[0].header["RUN"] - containers.nevents = hdul[0].header["NEVENTS"] - containers.npixels = hdul[0].header["NPIXELS"] - containers.nsamples = hdul[0].header["NSAMPLES"] - containers.camera = hdul[0].header["CAMERA"] - - containers.subarray = SubarrayDescription.from_hdf( - Path( - filename._str.replace("waveforms_", "subarray_").replace( - "fits", "hdf5" - ) - ) - ) - - containers.pixels_id = hdul[0].data - containers.wfs_hg = hdul[1].data - containers.wfs_lg = hdul[2].data - - broken_pixels = hdul[3].data - containers.broken_pixels_hg = broken_pixels["HG broken pixels"] - containers.broken_pixels_lg = broken_pixels["LG broken pixels"] - - table_prop = hdul[4].data - containers.event_id = table_prop["event_id"] - containers.event_type = table_prop["event_type"] - containers.ucts_timestamp = table_prop["ucts_timestamp"] - containers.ucts_busy_counter = table_prop["ucts_busy_counter"] - containers.ucts_event_counter = table_prop["ucts_event_counter"] - containers.multiplicity = table_prop["multiplicity"] - - table_trigger = hdul[5].data - containers.trig_pattern_all = table_trigger["trig_pattern_all"] - containers.trig_pattern = table_trigger["trig_pattern"] +class WaveformsContainers(TriggerMapContainer): + containers = Field(default_factory=partial(Map, WaveformsContainer), + description = "trigger mapping of WaveformContainer" + ) - return containers From ae0edf5ec149147092dfb0f2cf64eb58599b2764 Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 23 Oct 2023 14:33:22 +0200 Subject: [PATCH 60/62] -adapt charges extraction makers and waveform extraction makers with the ctapipe.Tool framework --- src/nectarchain/makers/__init__.py | 1 + src/nectarchain/makers/chargesMakers.py | 683 +--------------------- src/nectarchain/makers/core.py | 636 ++++---------------- src/nectarchain/makers/waveformsMakers.py | 322 +--------- 4 files changed, 158 insertions(+), 1484 deletions(-) diff --git a/src/nectarchain/makers/__init__.py b/src/nectarchain/makers/__init__.py index a59e6de1..f059c4fa 100644 --- a/src/nectarchain/makers/__init__.py +++ b/src/nectarchain/makers/__init__.py @@ -1,3 +1,4 @@ # from .chargesMakers import * from .core import * from .waveformsMakers import * +from .chargesMakers import * diff --git a/src/nectarchain/makers/chargesMakers.py b/src/nectarchain/makers/chargesMakers.py index 2742781c..f6d82d4c 100644 --- a/src/nectarchain/makers/chargesMakers.py +++ b/src/nectarchain/makers/chargesMakers.py @@ -26,676 +26,31 @@ from numba import bool_, float64, guvectorize, int64 from ..data.container import ChargesContainer, WaveformsContainer -from .core import ArrayDataMaker from .extractor.utils import CtapipeExtractor -__all__ = ["ChargesMaker"] - -list_ctapipe_charge_extractor = [ - "FullWaveformSum", - "FixedWindowSum", - "GlobalPeakWindowSum", - "LocalPeakWindowSum", - "SlidingWindowMaxSum", - "NeighborPeakWindowSum", - "BaselineSubtractedNeighborPeakWindowSum", - "TwoPassWindowSum", -] - - -list_nectarchain_charge_extractor = ["gradient_extractor"] - - -@guvectorize( - [ - (int64[:], float64[:], bool_, bool_[:], int64[:]), - ], - "(s),(n),()->(n),(n)", - nopython=True, - cache=True, -) -def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): - """compute histogram of charge with numba - - Args: - charge (np.ndarray(pixels,nevents)): charge - all_range (np.ndarray(nbins)): charge range - mask_broken_pix (np.ndarray(pixels)): mask on broxen pixels - _mask (np.ndarray(pixels,nbins)): mask - hist_ma_data (np.ndarray(pixels,nbins)): histogram - """ - # print(f"charge.shape = {charge.shape[0]}") - # print(f"_mask.shape = {_mask.shape[0]}") - # print(f"_mask[0] = {_mask[0]}") - # print(f"hist_ma_data[0] = {hist_ma_data[0]}") - # print(f"mask_broken_pix = {mask_broken_pix}") - - if not (mask_broken_pix): - # print("this pixel is not broken, let's continue computation") - hist, _charge = np.histogram( - charge, - bins=np.arange( - np.uint16(np.min(charge)) - 1, np.uint16(np.max(charge)) + 2, 1 - ), - ) - # print(f"hist.shape[0] = {hist.shape[0]}") - # print(f"charge.shape[0] = {_charge.shape[0]}") - charge_edges = np.array( - [np.mean(_charge[i : i + 2]) for i in range(_charge.shape[0] - 1)] - ) - # print(f"charge_edges.shape[0] = {charge_edges.shape[0]}") - mask = (all_range >= charge_edges[0]) * (all_range <= charge_edges[-1]) - # print(f"all_range = {int(all_range[0])}-{int(all_range[-1])}") - # print(f"charge_edges[0] = {int(charge_edges[0])}") - # print(f"charge_edges[-1] = {int(charge_edges[-1])}") - # print(f"mask[0] = {mask[0]}") - # print(f"mask[-1] = {mask[-1]}") - - # MASK THE DATA - # print(f"mask.shape = {mask.shape[0]}") - _mask[:] = ~mask - # print(f"_mask[0] = {_mask[0]}") - # print(f"_mask[-1] = {_mask[-1]}") - # FILL THE DATA - hist_ma_data[mask] = hist - # print("work done") - else: - # print("this pixel is broken, skipped") - pass - - -class ChargesMaker(ArrayDataMaker): - """ - The `ChargesMaker` class is a subclass of `ArrayDataMaker` and is responsible for computing charges and peaks from waveforms. It provides methods for initializing the class, making charges and peaks for events, creating output containers, sorting charges containers, selecting charges for specific pixels, computing histograms of charges, and more. - - Example Usage: - # Create an instance of ChargesMaker - charges_maker = ChargesMaker(run_number=1234, max_events=1000, run_file='data.fits') - - # Initialize the charges maker for a specific trigger type - charges_maker._init_trigger_type(trigger_type='TypeA') - - # Make charges and peaks for events - charges_maker.make(n_events=100, trigger_type=['TypeA', 'TypeB'], method='FullWaveformSum') - - # Get the charges and peaks for a specific trigger type - charges_hg = charges_maker.charges_hg(trigger='TypeA') - charges_lg = charges_maker.charges_lg(trigger='TypeA') - peaks_hg = charges_maker.peak_hg(trigger='TypeA') - peaks_lg = charges_maker.peak_lg(trigger='TypeA') - - # Create a charges container from waveforms - waveforms_container = WaveformsContainer() - charges_container = ChargesMaker.create_from_waveforms(waveforms_container, method='FullWaveformSum') - - # Compute histograms of charges - histo_hg = ChargesMaker.histo_hg(charges_container, n_bins=1000, autoscale=True) - histo_lg = ChargesMaker.histo_lg(charges_container, n_bins=1000, autoscale=True) - - Methods: - - __init__(self, run_number: int, max_events: int = None, run_file=None, *args, **kwargs): Constructor method to initialize the class. - - _init_trigger_type(self, trigger_type : EventType, **kwargs): Method to initialize the charges for a specific trigger type. - - make(self, n_events=np.inf, trigger_type: list = None, restart_from_beginning=False, method: str = "FullWaveformSum", *args, **kwargs): Method to make charges and peaks for events. - - _make_event(self, event : NectarCAMDataContainer, trigger: EventType, method: str = "FullWaveformSum", *args, **kwargs): Method to make charges and peaks for a single event. - - _make_output_container(self, trigger_type : EventType, method: str, *args, **kwargs): Method to create output containers for charges and peaks. - - sort(chargesContainer: ChargesContainer, method='event_id'): Static method to sort charges containers based on a specified method. - - select_charges_hg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): Static method to select charges for specific pixels in the high-gain channel. - - select_charges_lg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): Static method to select charges for specific pixels in the low-gain channel. - - charges_hg(self, trigger : EventType): Method to get the charges in the high-gain channel for a specific trigger type. - - charges_lg(self, trigger : EventType): Method to get the charges in the low-gain channel for a specific trigger type. - - peak_hg(self, trigger : EventType): Method to get the peaks in the high-gain channel for a specific trigger type. - - peak_lg(self, trigger : EventType): Method to get the peaks in the low-gain channel for a specific trigger type. - - create_from_waveforms(waveformsContainer: WaveformsContainer, method: str = "FullWaveformSum", **kwargs) -> ChargesContainer: Static method to create a charges container from waveforms. - - compute_charge(waveformContainer: WaveformsContainer, channel: int, method: str = "FullWaveformSum", **kwargs): Static method to compute charges and peaks from waveforms. - - histo_hg(chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True) -> ma.masked_array: Static method to compute a histogram of charges in the high-gain channel. - - histo_lg(chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True) -> ma.masked_array: Static method to compute a histogram of charges in the low-gain channel. - - Fields: - - __charges_hg: Dictionary to store the charges in the high-gain channel for each trigger type. - - __charges_lg: Dictionary to store the charges in the low-gain channel for each trigger type. - - __peak_hg: Dictionary to store the peaks in the high-gain channel for each trigger type. - - __peak_lg: Dictionary to store the peaks in the low-gain channel for each trigger type. - """ - - # constructors - def __init__( - self, - run_number: int, - max_events: int = None, - run_file: str = None, - *args, - **kwargs, - ): - """construtor - - Args: - run_number (int): id of the run to be loaded - maxevents (int, optional): max of events to be loaded. Defaults to 0, to load everythings. - nevents (int, optional) : number of events in run if known (parameter used to save computing time) - run_file (optional) : if provided, will load this run file - """ - super().__init__(run_number, max_events, run_file, *args, **kwargs) - self.__charges_hg = {} - self.__charges_lg = {} - self.__peak_hg = {} - self.__peak_lg = {} - - def _init_trigger_type(self, trigger_type: EventType, **kwargs): - """ - Initializes the ChargesMaker based on the trigger type. - - Args: - trigger_type (EventType): The type of trigger. - **kwargs: Additional keyword arguments. - - Returns: - None - """ - super()._init_trigger_type(trigger_type, **kwargs) - name = __class__._get_name_trigger(trigger_type) - log.info(f"initialization of the ChargesMaker following trigger type : {name}") - self.__charges_hg[f"{name}"] = [] - self.__charges_lg[f"{name}"] = [] - self.__peak_hg[f"{name}"] = [] - self.__peak_lg[f"{name}"] = [] - - def make( - self, - n_events=np.inf, - trigger_type: list = None, - restart_from_beginning: bool = False, - method: str = "FullWaveformSum", - *args, - **kwargs, - ): - """ - Makes charges based on the specified arguments. - - Args: - n_events (int): The number of events to process (default is infinity). - trigger_type (list): The type of trigger (default is None). - restart_from_beginning (bool): Whether to restart the processing from the beginning (default is False). - method (str): The method to use for making charges (default is "FullWaveformSum"). - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. - - Returns: - The result of the parent class's make method. - """ - kwargs["method"] = method - return super().make( - n_events=n_events, - trigger_type=trigger_type, - restart_from_beginning=restart_from_beginning, - *args, - **kwargs, - ) - - def _make_event( - self, - event: NectarCAMDataContainer, - trigger: EventType, - method: str = "FullWaveformSum", - *args, - **kwargs, - ): - """ - Make charges and peaks for a single event. - - Args: - event (NectarCAMDataContainer): The event container that contains the data for a single event. - trigger (EventType): The type of trigger for the event. - method (str, optional): The method to use for computing charges and peaks. Defaults to "FullWaveformSum". - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. - - Returns: - None - - Summary: - This method extracts waveforms from the event, computes broken pixels, and then uses an image extractor to calculate charges and peaks for the high-gain and low-gain channels. The results are stored in dictionaries for each trigger type. - ``` - """ - wfs_hg_tmp, wfs_lg_tmp = super()._make_event( - event=event, trigger=trigger, return_wfs=True, *args, **kwargs - ) - name = __class__._get_name_trigger(trigger) - - broken_pixels_hg, broken_pixels_lg = __class__._compute_broken_pixels_event( - event, self._pixels_id - ) - self._broken_pixels_hg[f"{name}"].append(broken_pixels_hg.tolist()) - self._broken_pixels_lg[f"{name}"].append(broken_pixels_lg.tolist()) - - imageExtractor = __class__._get_imageExtractor( - method, self._reader.subarray, **kwargs - ) - - __image = CtapipeExtractor.get_image_peak_time( - imageExtractor( - wfs_hg_tmp, __class__.TEL_ID, constants.HIGH_GAIN, broken_pixels_hg - ) - ) - self.__charges_hg[f"{name}"].append(__image[0].tolist()) - self.__peak_hg[f"{name}"].append(__image[1].tolist()) - - __image = CtapipeExtractor.get_image_peak_time( - imageExtractor( - wfs_lg_tmp, __class__.TEL_ID, constants.LOW_GAIN, broken_pixels_lg - ) - ) - self.__charges_lg[f"{name}"].append(__image[0].tolist()) - self.__peak_lg[f"{name}"].append(__image[1].tolist()) - - @staticmethod - def _get_imageExtractor(method: str, subarray: SubarrayDescription, **kwargs): - """ - Create an instance of a charge extraction method based on the provided method name and subarray description. - Args: - method (str): The name of the charge extraction method. - subarray (SubarrayDescription): The description of the subarray. - **kwargs (dict): Additional keyword arguments for the charge extraction method. - Returns: - imageExtractor: An instance of the charge extraction method specified by `method` with the provided subarray description and keyword arguments. - """ - if not ( - method in list_ctapipe_charge_extractor - or method in list_nectarchain_charge_extractor - ): - raise ArgumentError(f"method must be in {list_ctapipe_charge_extractor}") - extractor_kwargs = {} - for key in eval(method).class_own_traits().keys(): - if key in kwargs.keys(): - extractor_kwargs[key] = kwargs[key] - if ( - "apply_integration_correction" in eval(method).class_own_traits().keys() - ): # to change the default behavior of ctapipe extractor - extractor_kwargs["apply_integration_correction"] = kwargs.get( - "apply_integration_correction", False - ) - log.debug( - f"Extracting charges with method {method} and extractor_kwargs {extractor_kwargs}" - ) - imageExtractor = eval(method)(subarray, **extractor_kwargs) - return imageExtractor - - def _make_output_container( - self, trigger_type: EventType, method: str, *args, **kwargs - ): - """ - Create an output container for the specified trigger type and method. - Args: - trigger_type (EventType): The type of trigger. - method (str): The name of the charge extraction method. - *args: Additional positional arguments. - **kwargs: Additional keyword arguments. - Returns: - list: A list of ChargesContainer objects. - """ - output = [] - for trigger in trigger_type: - chargesContainer = ChargesContainer( - run_number=self._run_number, - npixels=self._npixels, - camera=self.CAMERA_NAME, - pixels_id=self._pixels_id, - method=method, - nevents=self.nevents(trigger), - charges_hg=self.charges_hg(trigger), - charges_lg=self.charges_lg(trigger), - peak_hg=self.peak_hg(trigger), - peak_lg=self.peak_lg(trigger), - broken_pixels_hg=self.broken_pixels_hg(trigger), - broken_pixels_lg=self.broken_pixels_lg(trigger), - ucts_timestamp=self.ucts_timestamp(trigger), - ucts_busy_counter=self.ucts_busy_counter(trigger), - ucts_event_counter=self.ucts_event_counter(trigger), - event_type=self.event_type(trigger), - event_id=self.event_id(trigger), - trig_pattern_all=self.trig_pattern_all(trigger), - trig_pattern=self.trig_pattern(trigger), - multiplicity=self.multiplicity(trigger), - ) - output.append(chargesContainer) - return output - - @staticmethod - def sort(chargesContainer: ChargesContainer, method: str = "event_id"): - """ - Sorts the charges in a ChargesContainer object based on the specified method. - Args: - chargesContainer (ChargesContainer): The ChargesContainer object to be sorted. - method (str, optional): The sorting method. Defaults to 'event_id'. - Returns: - ChargesContainer: A new ChargesContainer object with the charges sorted based on the specified method. - - Raises: - ArgumentError: If the specified method is not valid. - """ - output = ChargesContainer( - run_number=chargesContainer.run_number, - npixels=chargesContainer.npixels, - camera=chargesContainer.camera, - pixels_id=chargesContainer.pixels_id, - nevents=chargesContainer.nevents, - method=chargesContainer.method, - ) - if method == "event_id": - index = np.argsort(chargesContainer.event_id) - for field in chargesContainer.keys(): - if not ( - field - in [ - "run_number", - "npixels", - "camera", - "pixels_id", - "nevents", - "method", - ] - ): - output[field] = chargesContainer[field][index] - else: - raise ArgumentError(f"{method} is not a valid method for sorting") - return output - - @staticmethod - def select_charges_hg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): - """ - Selects the charges from the ChargesContainer object for the given pixel_id and returns the result transposed. - Args: - chargesContainer (ChargesContainer): The ChargesContainer object. - pixel_id (np.ndarray): An array of pixel IDs. - Returns: - np.ndarray: The selected charges from the ChargesContainer object for the given pixel_id, transposed. - """ - res = __class__.select_container_array_field( - container=chargesContainer, pixel_id=pixel_id, field="charges_hg" - ) - res = res.transpose(1, 0) - return res - - @staticmethod - def select_charges_lg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): - """ - Selects the charges from the ChargesContainer object for the given pixel_id and returns the result transposed. - Args: - chargesContainer (ChargesContainer): The ChargesContainer object. - pixel_id (np.ndarray): An array of pixel IDs. - Returns: - np.ndarray: The selected charges from the ChargesContainer object for the given pixel_id, transposed. - """ - res = __class__.select_container_array_field( - container=chargesContainer, pixel_id=pixel_id, field="charges_lg" - ) - res = res.transpose(1, 0) - return res - - def charges_hg(self, trigger: EventType): - """ - Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. - Args: - trigger (EventType): The specific trigger type. - Returns: - np.ndarray: The charges for the specific trigger type. - """ - return np.array( - self.__charges_hg[__class__._get_name_trigger(trigger)], dtype=np.uint16 - ) - - def charges_lg(self, trigger: EventType): - """ - Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. - Args: - trigger (EventType): The specific trigger type. - Returns: - np.ndarray: The charges for the specific trigger type. - """ - return np.array( - self.__charges_lg[__class__._get_name_trigger(trigger)], dtype=np.uint16 - ) - - def peak_hg(self, trigger: EventType): - """ - Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. - Args: - trigger (EventType): The specific trigger type. - Returns: - np.ndarray: The peak charges for the specific trigger type. - """ - return np.array( - self.__peak_hg[__class__._get_name_trigger(trigger)], dtype=np.uint16 - ) - - def peak_lg(self, trigger: EventType): - """ - Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. - Args: - trigger (EventType): The specific trigger type. - Returns: - np.ndarray: The peak charges for the specific trigger type. - """ - return np.array( - self.__peak_lg[__class__._get_name_trigger(trigger)], dtype=np.uint16 - ) - - @staticmethod - def create_from_waveforms( - waveformsContainer: WaveformsContainer, - method: str = "FullWaveformSum", - **kwargs, - ) -> ChargesContainer: - """ - Create a ChargesContainer object from waveforms using the specified charge extraction method. - Args: - waveformsContainer (WaveformsContainer): The waveforms container object. - method (str, optional): The charge extraction method to use (default is "FullWaveformSum"). - **kwargs: Additional keyword arguments to pass to the charge extraction method. - Returns: - ChargesContainer: The charges container object containing the computed charges and peak times. - """ - chargesContainer = ChargesContainer() - for field in waveformsContainer.keys(): - if not (field in ["subarray", "nsamples", "wfs_hg", "wfs_lg"]): - chargesContainer[field] = waveformsContainer[field] - log.info(f"computing hg charge with {method} method") - charges_hg, peak_hg = __class__.compute_charge( - waveformsContainer, constants.HIGH_GAIN, method, **kwargs - ) - charges_hg = np.array(charges_hg, dtype=np.uint16) - log.info(f"computing lg charge with {method} method") - charges_lg, peak_lg = __class__.compute_charge( - waveformsContainer, constants.LOW_GAIN, method, **kwargs - ) - charges_lg = np.array(charges_lg, dtype=np.uint16) - chargesContainer.charges_hg = charges_hg - chargesContainer.charges_lg = charges_lg - chargesContainer.peak_hg = peak_hg - chargesContainer.peak_lg = peak_lg - chargesContainer.method = method - return chargesContainer - - @staticmethod - def compute_charge( - waveformContainer: WaveformsContainer, - channel: int, - method: str = "FullWaveformSum", - **kwargs, - ): - """ - Compute charge from waveforms. - Args: - waveformContainer (WaveformsContainer): The waveforms container object. - channel (int): The channel to compute charges for. - method (str, optional): The charge extraction method to use (default is "FullWaveformSum"). - **kwargs: Additional keyword arguments to pass to the charge extraction method. - Raises: - ArgumentError: If the extraction method is unknown. - ArgumentError: If the channel is unknown. - Returns: - tuple: A tuple containing the computed charges and peak times. - """ - # import is here for fix issue with pytest (TypeError : inference is not possible with python <3.9 (Numba conflict bc there is no inference...)) - from .extractor.utils import CtapipeExtractor - - imageExtractor = __class__._get_imageExtractor( - method=method, subarray=waveformContainer.subarray, **kwargs - ) - if channel == constants.HIGH_GAIN: - out = np.array( - [ - CtapipeExtractor.get_image_peak_time( - imageExtractor( - waveformContainer.wfs_hg[i], - __class__.TEL_ID, - channel, - waveformContainer.broken_pixels_hg, - ) - ) - for i in range(len(waveformContainer.wfs_hg)) - ] - ).transpose(1, 0, 2) - return out[0], out[1] - elif channel == constants.LOW_GAIN: - out = np.array( - [ - CtapipeExtractor.get_image_peak_time( - imageExtractor( - waveformContainer.wfs_lg[i], - __class__.TEL_ID, - channel, - waveformContainer.broken_pixels_lg, - ) - ) - for i in range(len(waveformContainer.wfs_lg)) - ] - ).transpose(1, 0, 2) - return out[0], out[1] - else: - raise ArgumentError( - f"channel must be {constants.LOW_GAIN} or {constants.HIGH_GAIN}" - ) - - @staticmethod - def histo_hg( - chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True - ) -> ma.masked_array: - """ - Computes histogram of high gain charges from a ChargesContainer object. - - Args: - chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. - n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. - autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. - - Returns: - ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. - """ - return __class__._histo( - chargesContainer=chargesContainer, - field="charges_hg", - n_bins=n_bins, - autoscale=autoscale, - ) - - @staticmethod - def histo_lg( - chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True - ) -> ma.masked_array: - """ - Computes histogram of low gain charges from a ChargesContainer object. - - Args: - chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. - n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. - autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. - - Returns: - ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. - """ - return __class__._histo( - chargesContainer=chargesContainer, - field="charges_lg", - n_bins=n_bins, - autoscale=autoscale, - ) +import numpy as np - @staticmethod - def _histo( - chargesContainer: ChargesContainer, - field: str, - n_bins: int = 1000, - autoscale: bool = True, - ) -> ma.masked_array: - """ - Computes histogram of charges for a given field from a ChargesContainer object. - Numba is used to compute histograms in a vectorized way. +from ctapipe.containers import EventType +from ctapipe.instrument import SubarrayDescription +from ctapipe_io_nectarcam import constants +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer +from ctapipe.core.traits import ComponentNameList,Unicode +from tqdm import tqdm - Args: - chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. - field (str): The field name for which the histogram is computed. - n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. - autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. +from ..data.container import WaveformsContainer +from .core import EventsLoopNectarCAMCalibrationTool +from .component import NectarCAMComponent,ChargesComponent,get_specific_traits - Returns: - ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. - """ - mask_broken_pix = np.array( - (chargesContainer[field] == chargesContainer[field].mean(axis=0)).mean( - axis=0 - ), - dtype=bool, - ) - log.debug( - f"there are {mask_broken_pix.sum()} broken pixels (charge stays at same level for each events)" - ) +__all__ = ["ChargesNectarCAMCalibrationTool"] - if autoscale: - all_range = np.arange( - np.uint16(np.min(chargesContainer[field].T[~mask_broken_pix].T)) - 0.5, - np.uint16(np.max(chargesContainer[field].T[~mask_broken_pix].T)) + 1.5, - 1, - ) - charge_ma = ma.masked_array( - ( - all_range.reshape(all_range.shape[0], 1) - @ np.ones((1, chargesContainer[field].shape[1])) - ).T, - mask=np.zeros( - (chargesContainer[field].shape[1], all_range.shape[0]), dtype=bool - ), - ) - broxen_pixels_mask = np.array( - [mask_broken_pix for i in range(charge_ma.shape[1])] - ).T - start = time.time() - _mask, hist_ma_data = make_histo( - chargesContainer[field].T, all_range, mask_broken_pix - ) - charge_ma.mask = np.logical_or(_mask, broxen_pixels_mask) - hist_ma = ma.masked_array(hist_ma_data, mask=charge_ma.mask) - log.debug(f"histogram hg computation time : {time.time() - start} sec") - return ma.masked_array((hist_ma, charge_ma)) - else: - hist = np.array( - [ - np.histogram(chargesContainer[field].T[i], bins=n_bins)[0] - for i in range(chargesContainer[field].shape[1]) - ] - ) - charge = np.array( - [ - np.histogram(chargesContainer[field].T[i], bins=n_bins)[1] - for i in range(chargesContainer[field].shape[1]) - ] - ) - charge_edges = np.array( - [ - np.mean(charge.T[i : i + 2], axis=0) - for i in range(charge.shape[1] - 1) - ] - ).T - return np.array((hist, charge_edges)) +class ChargesNectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): + """class use to make the waveform extraction from event read from r0 data""" + componentsList = ComponentNameList( + NectarCAMComponent, + default_value = ["ChargesComponent"], + help="List of Component names to be apply, the order will be respected" + ).tag(config=True) + diff --git a/src/nectarchain/makers/core.py b/src/nectarchain/makers/core.py index 7fc3a4d3..62a53a25 100644 --- a/src/nectarchain/makers/core.py +++ b/src/nectarchain/makers/core.py @@ -10,19 +10,23 @@ import numpy as np from ctapipe.containers import EventType -from ctapipe.core import Tool -from ctapipe.core.traits import Bool, Integer, Path, classes_with_traits, flag +from ctapipe.core import Tool,Component +from ctapipe.core.traits import Bool, Integer, Path, classes_with_traits, flag, ComponentNameList from ctapipe.instrument import CameraGeometry from ctapipe.io import HDF5TableWriter from ctapipe.io.datawriter import DATA_MODEL_VERSION from ctapipe_io_nectarcam import NectarCAMEventSource, constants from ctapipe_io_nectarcam.containers import NectarCAMDataContainer + + + from tqdm.auto import tqdm from ..data import DataManagement -from ..data.container.core import ArrayDataContainer +from ..data.container.core import ArrayDataContainer,NectarCAMContainer,TriggerMapContainer +from .component import * -__all__ = ["ArrayDataNectarCAMCalibrationTool"] +__all__ = ["EventsLoopNectarCAMCalibrationTool"] """The code snippet is a part of a class hierarchy for data processing. It includes the `BaseMaker` abstract class, the `EventsLoopMaker` and `ArrayDataMaker` subclasses. @@ -89,10 +93,12 @@ class EventsLoopNectarCAMCalibrationTool(BaseNectarCAMCalibrationTool): ) examples = """To be implemented""" + aliases = { ("i", "input"): "EventsLoopNectarCAMCalibrationTool.run_file", ("r", "run-number"): "EventsLoopNectarCAMCalibrationTool.run_number", ("m", "max-events"): "EventsLoopNectarCAMCalibrationTool.max_events", + ("o", "output"): "EventsLoopNectarCAMCalibrationTool.output_path", } flags = { @@ -108,7 +114,19 @@ class EventsLoopNectarCAMCalibrationTool(BaseNectarCAMCalibrationTool): ), } - classes = classes_with_traits(NectarCAMEventSource) + classes = ( + [ + HDF5TableWriter, + ] + + classes_with_traits(NectarCAMEventSource) + + classes_with_traits(NectarCAMComponent) + + ) + + + output_path = Path( + help="output filename", default_value=pathlib.Path("/tmp/EventsLoopNectarCAMCalibrationTool.h5") + ).tag(config=True) run_number = Integer(help="run number to be treated", default_value=-1).tag( config=True @@ -126,11 +144,36 @@ class EventsLoopNectarCAMCalibrationTool(BaseNectarCAMCalibrationTool): allow_none=True, ).tag(config=True) + componentsList = ComponentNameList(NectarCAMComponent, + help="List of Component names to be apply, the order will be respected" + ).tag(config=True) + + def __new__(cls,*args,**kwargs) : + """This method is used to pass to the current instance of Tool the traits defined + in the components provided in the componentsList trait. + WARNING : This method is maybe not the best way to do it, need to discuss with ctapipe developpers. + """ + _cls = super(EventsLoopNectarCAMCalibrationTool,cls).__new__(cls,*args,**kwargs) + log.warning("the componentName in componentsList must be defined in the nectarchain.makers.component module, otherwise the import of the componentName will raise an error") + for componentName in _cls.componentsList : + configurable_traits = get_configurable_traits(eval(componentName)) + _cls.add_traits(**configurable_traits) + _cls.aliases.update({key : f"{componentName}.{key}" for key in configurable_traits.keys()}) + return _cls + def _load_eventsource(self): self.event_source = self.enter_context( self.load_run(self.run_number, self.max_events, run_file=self.run_file) ) + def _get_provided_component_kwargs(self,componentName : str) : + component_kwargs = get_configurable_traits(eval(componentName)) + output_component_kwargs = {} + for key in component_kwargs.keys() : + if hasattr(self,key) : + output_component_kwargs[key] = getattr(self,key) + return output_component_kwargs + def setup(self, *args, **kwargs): if self.run_number == -1: raise Exception("run_number need to be set up") @@ -138,225 +181,39 @@ def setup(self, *args, **kwargs): self.__npixels = self._event_source.camera_config.num_pixels self.__pixels_id = self._event_source.camera_config.expected_pixels_id + self.components = [] + for componentName in self.componentsList : + if componentName in get_valid_component(): + component_kwargs = self._get_provided_component_kwargs(componentName) + self.components.append( + # self.add_component( + Component.from_name( + componentName, + subarray = self.event_source.subarray, + parent=self, + **component_kwargs, + ) + # ) + ) + + self.writer = self.enter_context( + HDF5TableWriter( + filename = pathlib.Path(f"{self.output_path.parent}/{self.output_path.stem}_{self.run_number}{self.output_path.suffix}"), + parent = self, + group_name = "data" + ) + ) + # self.comp = MyComponent(parent=self) # self.comp2 = SecondaryMyComponent(parent=self) # self.comp3 = TelescopeWiseComponent(parent=self, subarray=subarray) # self.advanced = AdvancedComponent(parent=self) - """ - def start(self, n_events=np.inf, restart_from_beginning: bool = False,*args,**kwargs): - if restart_from_beginning: - self.log.debug("restart from beginning : creation of the EventSource reader") - self._load_eventsource() - - #self.event_source.subarray.info(printer=self.log.info) - - n_traited_events = 0 - for i,event in enumerate(tqdm( - self._event_source, - desc=self._event_source.__class__.__name__, - total=min(self._event_source.max_events,n_events), - unit="ev", - disable=not self.progress_bar, - )): - if i % 100 == 0: - self.log.info(f"reading event number {i}") - self._make_event(event, *args, **kwargs) - n_traited_events += 1 - if n_traited_events >= n_events: - break - """ - - def finish(self): - self.log.warning("Shutting down.") - - @abstractmethod - def _make_event(self, event: NectarCAMDataContainer): - """ - Abstract method that needs to be implemented by subclasses. - This method is called for each event in the run to perform computations on the event. - - Args: - event (NectarCAMDataContainer): The event to perform computations on. - """ - pass - - @property - def event_source(self): - """ - Getter method for the _event_source attribute. - """ - return copy.deepcopy(self._event_source) - - @event_source.setter - def event_source(self, value): - """ - Setter method to set a new NectarCAMEventSource to the _reader attribute. - - Args: - value: a NectarCAMEventSource instance. - """ - if isinstance(value, NectarCAMEventSource): - self._event_source = value - else: - raise TypeError("The reader must be a NectarCAMEventSource") - - @property - def _npixels(self): - """ - Getter method for the _npixels attribute. - """ - return self.__npixels - - @property - def _pixels_id(self): - """ - Getter method for the _pixels_id attribute. - """ - return self.__pixels_id - - @property - def npixels(self): - """ - Getter method for the npixels attribute. - """ - return copy.deepcopy(self.__npixels) - - @property - def pixels_id(self): - """ - Getter method for the pixels_id attribute. - """ - return copy.deepcopy(self.__pixels_id) - - -def main(): - """run the tool""" - tool = EventsLoopNectarCAMCalibrationTool() - tool.run() - - -if __name__ == "__main__": - main() - - -class ArrayDataNectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): - name = "ArrayDataNectarCAMCalibration" - - # description = ( - # __doc__ + f" This currently uses data model version {DATA_MODEL_VERSION}" - # ) - examples = """To be implemented""" - - aliases = super( - EventsLoopNectarCAMCalibrationTool, EventsLoopNectarCAMCalibrationTool - ).aliases - aliases.update( - { - ("o", "output"): "HDF5TableWriter.output_path", - } - ) - - classes = super( - EventsLoopNectarCAMCalibrationTool, EventsLoopNectarCAMCalibrationTool - ).classes + ( - [ - HDF5TableWriter, - ] - ) - - TEL_ID = int(0) - - CAMERA_NAME = str("NectarCam-003") - - CAMERA = CameraGeometry.from_name(CAMERA_NAME) - - def setup(self, *args, **kwargs): - super().setup(*args, **kwargs) - self.__nsamples = self._event_source.camera_config.num_samples - # data we want to compute - self.__ucts_timestamp = {} - self.__ucts_busy_counter = {} - self.__ucts_event_counter = {} - self.__event_type = {} - self.__event_id = {} - self.__trig_patter_all = {} - self.__broken_pixels_hg = {} - self.__broken_pixels_lg = {} - - def _init_trigger_type(self, trigger: EventType, **kwargs): - """ - Initializes empty lists for different trigger types in the ArrayDataMaker class. - - Args: - trigger (EventType): The trigger type for which the lists are being initialized. - - Returns: - None. The method only initializes the empty lists for the trigger type. - """ - name = __class__._get_name_trigger(trigger) - self.__ucts_timestamp[f"{name}"] = [] - self.__ucts_busy_counter[f"{name}"] = [] - self.__ucts_event_counter[f"{name}"] = [] - self.__event_type[f"{name}"] = [] - self.__event_id[f"{name}"] = [] - self.__trig_patter_all[f"{name}"] = [] - self.__broken_pixels_hg[f"{name}"] = [] - self.__broken_pixels_lg[f"{name}"] = [] - - @staticmethod - def _compute_broken_pixels(wfs_hg, wfs_lg, **kwargs): - """ - Computes broken pixels for high and low gain waveforms. - Args: - wfs_hg (ndarray): High gain waveforms. - wfs_lg (ndarray): Low gain waveforms. - **kwargs: Additional keyword arguments. - Returns: - tuple: Two arrays of zeros with the same shape as `wfs_hg` (or `wfs_lg`) but without the last dimension. - """ - log.warning("computation of broken pixels is not yet implemented") - return np.zeros((wfs_hg.shape[:-1]), dtype=bool), np.zeros( - (wfs_hg.shape[:-1]), dtype=bool - ) - - @staticmethod - def _compute_broken_pixels_event( - event: NectarCAMDataContainer, pixels_id: np.ndarray, **kwargs - ): - """ - Computes broken pixels for a specific event and pixel IDs. - Args: - event (NectarCAMDataContainer): An event. - pixels_id (list or np.ndarray): IDs of pixels. - **kwargs: Additional keyword arguments. - Returns: - tuple: Two arrays of zeros with the length of `pixels_id`. - """ - log.warning("computation of broken pixels is not yet implemented") - return np.zeros((len(pixels_id)), dtype=bool), np.zeros( - (len(pixels_id)), dtype=bool - ) - - @staticmethod - def _get_name_trigger(trigger: EventType): - """ - Gets the name of a trigger event. - Args: - trigger (EventType): A trigger event. - Returns: - str: The name of the trigger event. - """ - if trigger is None: - name = "None" - else: - name = trigger.name - return name def start( self, n_events=np.inf, - trigger_type: list = None, + #trigger_type: list = None, restart_from_begining: bool = False, *args, **kwargs, @@ -366,7 +223,6 @@ def start( Args: n_events (int, optional): The maximum number of events to process. Default is np.inf. - trigger_type (list[EventType], optional): Only events with the specified trigger types will be used. Default is None. restart_from_begining (bool, optional): Whether to restart the event source reader. Default is False. *args: Additional arguments that can be passed to the method. **kwargs: Additional keyword arguments that can be passed to the method. @@ -378,10 +234,10 @@ def start( self.log.warning( "no needed events number specified, it may cause a memory error" ) - if isinstance(trigger_type, EventType) or trigger_type is None: - trigger_type = [trigger_type] - for _trigger_type in trigger_type: - self._init_trigger_type(_trigger_type) + #if isinstance(trigger_type, EventType) or trigger_type is None: + # trigger_type = [trigger_type] + #for _trigger_type in trigger_type: + # self._init_trigger_type(_trigger_type) if restart_from_begining: self.log.debug("restart from begining : creation of the EventSource reader") @@ -399,327 +255,91 @@ def start( ): if i % 100 == 0: self.log.info(f"reading event number {i}") - for trigger in trigger_type: - if (trigger is None) or (trigger == event.trigger.event_type): - self._make_event(event, trigger, *args, **kwargs) - n_traited_events += 1 + for component in self.components : + component(event,*args,**kwargs) + n_traited_events += 1 if n_traited_events >= n_events: break - def _make_event( - self, event: NectarCAMDataContainer, trigger: EventType, *args, **kwargs - ): - """ - Method to extract data from the event. - - Args: - event (NectarCAMDataContainer): The event object. - trigger (EventType): The trigger type. - *args: Additional arguments that can be passed to the method. - **kwargs: Additional keyword arguments that can be passed to the method. - - Returns: - If the return_wfs keyword argument is True, the method returns the high and low gain waveforms from the event. - """ - name = __class__._get_name_trigger(trigger) - self.__event_id[f"{name}"].append(np.uint16(event.index.event_id)) - self.__ucts_timestamp[f"{name}"].append( - event.nectarcam.tel[__class__.TEL_ID].evt.ucts_timestamp - ) - self.__event_type[f"{name}"].append(event.trigger.event_type.value) - self.__ucts_busy_counter[f"{name}"].append( - event.nectarcam.tel[__class__.TEL_ID].evt.ucts_busy_counter - ) - self.__ucts_event_counter[f"{name}"].append( - event.nectarcam.tel[__class__.TEL_ID].evt.ucts_event_counter - ) - self.__trig_patter_all[f"{name}"].append( - event.nectarcam.tel[__class__.TEL_ID].evt.trigger_pattern.T - ) - - if kwargs.get("return_wfs", False): - get_wfs_hg = event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id] - get_wfs_lg = event.r0.tel[0].waveform[constants.LOW_GAIN][self.pixels_id] - return get_wfs_hg, get_wfs_lg - - def finish(self, trigger_type: list = None, *args, **kwargs): + def finish(self, *args, **kwargs): # self.write = self.enter_context( # HDF5TableWriter(filename=filename, parent=self) # ) - if isinstance(trigger_type, EventType) or trigger_type is None: - trigger_type = [trigger_type] - output_container = self._make_output_container(trigger_type, *args, **kwargs) - self.log.info(f"{output_container}") - super().finish() - - @abstractmethod - def _make_output_container(self): - pass - - @staticmethod - def select_container_array_field( - container: ArrayDataContainer, pixel_id: np.ndarray, field: str - ) -> np.ndarray: - """ - Selects specific fields from an ArrayDataContainer object based on a given list of pixel IDs. - - Args: - container (ArrayDataContainer): An object of type ArrayDataContainer that contains the data. - pixel_id (ndarray): An array of pixel IDs for which the data needs to be selected. - field (str): The name of the field to be selected from the container. - - Returns: - ndarray: An array containing the selected data for the given pixel IDs. - """ - mask_contain_pixels_id = np.array( - [pixel in container.pixels_id for pixel in pixel_id], dtype=bool - ) - for pixel in pixel_id[~mask_contain_pixels_id]: - log.warning( - f"You asked for pixel_id {pixel} but it is not present in this container, skip this one" - ) - res = np.array( - [ - np.take( - container[field], - np.where(container.pixels_id == pixel)[0][0], - axis=1, + output = [] + for component in self.components : + output.append(component.finish(*args,**kwargs)) + log.info(output) + for _output in output : + if isinstance(_output,NectarCAMContainer) : + self.writer.write(table_name = str(_output.__class__.__name__), + containers = _output, ) - for pixel in pixel_id[mask_contain_pixels_id] - ] - ) - ####could be nice to return np.ma.masked_array(data = res, mask = container.broken_pixels_hg.transpose(res.shape[1],res.shape[0],res.shape[2])) - return res - - @staticmethod - def merge( - container_a: ArrayDataContainer, container_b: ArrayDataContainer - ) -> ArrayDataContainer: - """method to merge 2 ArrayDataContainer into one single ArrayDataContainer - - Returns: - ArrayDataContainer: the merged object - """ - if type(container_a) != type(container_b): - raise Exception("The containers have to be instnace of the same class") - - if np.array_equal(container_a.pixels_id, container_b.pixels_id): - raise Exception("The containers have not the same pixels ids") - - merged_container = container_a.__class__.__new__() - - for field in container_a.keys(): - if ~isinstance(container_a[field], np.ndarray): - if container_a[field] != container_b[field]: - raise Exception( - f"merge impossible because of {field} filed (values are {container_a[field]} and {container_b[field]}" + elif isinstance(_output,TriggerMapContainer) : + for i,key in enumerate(_output.containers.keys()) : + self.writer.write(table_name = f"{_output.containers[key].__class__.__name__}_{i}/{key.name}", + containers = _output.containers[key], ) + else : + raise TypeError("component output must be an instance of TriggerMapContainer or NectarCAMContainer") - for field in container_a.keys(): - if isinstance(container_a[field], np.ndarray): - merged_container[field] = np.concatenate( - container_a[field], container_a[field], axis=0 - ) - else: - merged_container[field] = container_a[field] - - return merged_container - - @property - def nsamples(self): - """ - Returns a deep copy of the nsamples attribute. - - Returns: - np.ndarray: A deep copy of the nsamples attribute. - """ - return copy.deepcopy(self.__nsamples) + self.writer.close() + super().finish() + self.log.warning("Shutting down.") @property - def _nsamples(self): + def event_source(self): """ - Returns the nsamples attribute. - - Returns: - np.ndarray: The nsamples attribute. + Getter method for the _event_source attribute. """ - return self.__nsamples + return copy.copy(self._event_source) - def nevents(self, trigger: EventType): + @event_source.setter + def event_source(self, value): """ - Returns the number of events for the specified trigger type. + Setter method to set a new NectarCAMEventSource to the _reader attribute. Args: - trigger (EventType): The trigger type for which the number of events is requested. - - Returns: - int: The number of events for the specified trigger type. + value: a NectarCAMEventSource instance. """ - return len(self.__event_id[__class__._get_name_trigger(trigger)]) + if isinstance(value, NectarCAMEventSource): + self._event_source = value + else: + raise TypeError("The reader must be a NectarCAMEventSource") @property - def _broken_pixels_hg(self): - """ - Returns the broken_pixels_hg attribute. - - Returns: - np.ndarray: The broken_pixels_hg attribute. - """ - return self.__broken_pixels_hg - - def broken_pixels_hg(self, trigger: EventType): + def _npixels(self): """ - Returns an array of broken pixels for high gain for the specified trigger type. - - Args: - trigger (EventType): The trigger type for which the broken pixels for high gain are requested. - - Returns: - np.ndarray: An array of broken pixels for high gain for the specified trigger type. + Getter method for the _npixels attribute. """ - return np.array( - self.__broken_pixels_hg[__class__._get_name_trigger(trigger)], dtype=bool - ) + return self.__npixels @property - def _broken_pixels_lg(self): - """ - Returns the broken_pixels_lg attribute. - - Returns: - np.ndarray: The broken_pixels_lg attribute. - """ - return self.__broken_pixels_lg - - def broken_pixels_lg(self, trigger: EventType): - """ - Returns an array of broken pixels for low gain for the specified trigger type. - - Args: - trigger (EventType): The trigger type for which the broken pixels for low gain are requested. - - Returns: - np.ndarray: An array of broken pixels for low gain for the specified trigger type. - """ - return np.array( - self.__broken_pixels_lg[__class__._get_name_trigger(trigger)], dtype=bool - ) - - def ucts_timestamp(self, trigger: EventType): - """ - Returns an array of UCTS timestamps for the specified trigger type. - - Args: - trigger (EventType): The trigger type for which the UCTS timestamps are requested. - - Returns: - np.ndarray: An array of UCTS timestamps for the specified trigger type. - """ - return np.array( - self.__ucts_timestamp[__class__._get_name_trigger(trigger)], dtype=np.uint64 - ) - - def ucts_busy_counter(self, trigger: EventType): - """ - Returns an array of UCTS busy counters for the specified trigger type. - - Args: - trigger (EventType): The trigger type for which the UCTS busy counters are requested. - - Returns: - np.ndarray: An array of UCTS busy counters for the specified trigger type. - """ - return np.array( - self.__ucts_busy_counter[__class__._get_name_trigger(trigger)], - dtype=np.uint32, - ) - - def ucts_event_counter(self, trigger: EventType): - """ - Returns an array of UCTS event counters for the specified trigger type. - - Args: - trigger (EventType): The trigger type for which the UCTS event counters are requested. - - Returns: - np.ndarray: An array of UCTS event counters for the specified trigger type. - """ - return np.array( - self.__ucts_event_counter[__class__._get_name_trigger(trigger)], - dtype=np.uint32, - ) - - def event_type(self, trigger: EventType): - """ - Returns an array of event types for the specified trigger type. - - Args: - trigger (EventType): The trigger type for which the event types are requested. - - Returns: - np.ndarray: An array of event types for the specified trigger type. + def _pixels_id(self): """ - return np.array( - self.__event_type[__class__._get_name_trigger(trigger)], dtype=np.uint8 - ) - - def event_id(self, trigger: EventType): + Getter method for the _pixels_id attribute. """ - Returns an array of event IDs for the specified trigger type. - - Args: - trigger (EventType): The trigger type for which the event IDs are requested. + return self.__pixels_id - Returns: - np.ndarray: An array of event IDs for the specified trigger type. + @property + def npixels(self): """ - return np.array( - self.__event_id[__class__._get_name_trigger(trigger)], dtype=np.uint32 - ) - - def multiplicity(self, trigger: EventType): + Getter method for the npixels attribute. """ - Returns an array of multiplicities for the specified trigger type. - - Args: - trigger (EventType): The trigger type for which the multiplicities are requested. + return copy.deepcopy(self.__npixels) - Returns: - np.ndarray: An array of multiplicities for the specified trigger type. + @property + def pixels_id(self): """ - tmp = self.trig_pattern(trigger) - if len(tmp) == 0: - return np.array([]) - else: - return np.uint16(np.count_nonzero(tmp, axis=1)) - - def trig_pattern(self, trigger: EventType): + Getter method for the pixels_id attribute. """ - Returns an array of trigger patterns for the specified trigger type. + return copy.deepcopy(self.__pixels_id) - Args: - trigger (EventType): The trigger type for which the trigger patterns are requested. - Returns: - np.ndarray: An array of trigger patterns for the specified trigger type. - """ - tmp = self.trig_pattern_all(trigger) - if len(tmp) == 0: - return np.array([]) - else: - return tmp.any(axis=2) - - def trig_pattern_all(self, trigger: EventType): - """ - Returns an array of trigger patterns for all events for the specified trigger type. +def main(): + """run the tool""" + tool = EventsLoopNectarCAMCalibrationTool() + tool.run() - Args: - trigger (EventType): The trigger type for which the trigger patterns for all events are requested. - Returns: - np.ndarray: An array of trigger patterns for all events for the specified trigger type. - """ - return np.array( - self.__trig_patter_all[f"{__class__._get_name_trigger(trigger)}"], - dtype=bool, - ) +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/nectarchain/makers/waveformsMakers.py b/src/nectarchain/makers/waveformsMakers.py index 4a3bc474..04105fb8 100644 --- a/src/nectarchain/makers/waveformsMakers.py +++ b/src/nectarchain/makers/waveformsMakers.py @@ -12,323 +12,21 @@ from ctapipe.instrument import SubarrayDescription from ctapipe_io_nectarcam import constants from ctapipe_io_nectarcam.containers import NectarCAMDataContainer +from ctapipe.core.traits import ComponentNameList from tqdm import tqdm from ..data.container import WaveformsContainer -from .core import ArrayDataNectarCAMCalibrationTool +from .core import EventsLoopNectarCAMCalibrationTool +from .component import NectarCAMComponent __all__ = ["WaveformsNectarCAMCalibrationTool"] -class WaveformsNectarCAMCalibrationTool(ArrayDataNectarCAMCalibrationTool): +class WaveformsNectarCAMCalibrationTool(EventsLoopNectarCAMCalibrationTool): """class use to make the waveform extraction from event read from r0 data""" - - # constructors - def setup(self, *args, **kwargs): - super().setup(*args, **kwargs) - - self.__geometry = self._event_source.subarray.tel[__class__.TEL_ID].camera - self.__subarray = self._event_source.subarray - - self.__wfs_hg = {} - self.__wfs_lg = {} - - @staticmethod - def create_from_events_list( - events_list: list, - run_number: int, - npixels: int, - nsamples: int, - subarray: SubarrayDescription, - pixels_id: int, - ) -> WaveformsContainer: - """Create a container for the extracted waveforms from a list of events. - - Args: - events_list (list[NectarCAMDataContainer]): A list of events to extract waveforms from. - run_number (int): The ID of the run to be loaded. - npixels (int): The number of pixels in the waveforms. - nsamples (int): The number of samples in the waveforms. - subarray (SubarrayDescription): The subarray description instance. - pixels_id (int): The ID of the pixels to extract waveforms from. - - Returns: - WaveformsContainer: A container object that contains the extracted waveforms and other relevant information. - """ - container = WaveformsContainer( - run_number=run_number, - npixels=npixels, - nsamples=nsamples, - subarray=subarray, - camera=__class__.CAMERA_NAME, - pixels_id=pixels_id, - ) - - ucts_timestamp = [] - ucts_busy_counter = [] - ucts_event_counter = [] - event_type = [] - event_id = [] - trig_pattern_all = [] - wfs_hg = [] - wfs_lg = [] - - for event in tqdm(events_list): - ucts_timestamp.append( - event.nectarcam.tel[__class__.TEL_ID].evt.ucts_timestamp - ) - ucts_busy_counter.append( - event.nectarcam.tel[__class__.TEL_ID].evt.ucts_busy_counter - ) - ucts_event_counter.append( - event.nectarcam.tel[__class__.TEL_ID].evt.ucts_event_counter - ) - event_type.append(event.trigger.event_type.value) - event_id.append(event.index.event_id) - trig_pattern_all.append( - event.nectarcam.tel[__class__.TEL_ID].evt.trigger_pattern.T - ) - wfs_hg.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][pixels_id]) - wfs_lg.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][pixels_id]) - - container.wfs_hg = np.array(wfs_hg, dtype=np.uint16) - container.wfs_lg = np.array(wfs_lg, dtype=np.uint16) - - container.ucts_timestamp = np.array(ucts_timestamp, dtype=np.uint64) - container.ucts_busy_counter = np.array(ucts_busy_counter, dtype=np.uint32) - container.ucts_event_counter = np.array(ucts_event_counter, dtype=np.uint32) - container.event_type = np.array(event_type, dtype=np.uint8) - container.event_id = np.array(event_id, dtype=np.uint32) - container.trig_pattern_all = np.array(trig_pattern_all, dtype=bool) - container.trig_pattern = container.trig_pattern_all.any(axis=2) - container.multiplicity = np.uint16( - np.count_nonzero(container.trig_pattern, axis=1) - ) - - broken_pixels = __class__._compute_broken_pixels( - container.wfs_hg, container.wfs_lg - ) - container.broken_pixels_hg = broken_pixels[0] - container.broken_pixels_lg = broken_pixels[1] - return container - - def _init_trigger_type(self, trigger_type: EventType, **kwargs): - """Initialize the waveformsMaker following the trigger type. - - Args: - trigger_type: The type of trigger. - - """ - super()._init_trigger_type(trigger_type, **kwargs) - name = __class__._get_name_trigger(trigger_type) - log.info( - f"initialization of the waveformsMaker following trigger type : {name}" - ) - self.__wfs_hg[f"{name}"] = [] - self.__wfs_lg[f"{name}"] = [] - - def _make_event( - self, event: NectarCAMDataContainer, trigger: EventType, *args, **kwargs - ): - """Process an event and extract waveforms. - - Args: - event (NectarCAMDataContainer): The event to process and extract waveforms from. - trigger (EventType): The type of trigger for the event. - - """ - wfs_hg_tmp = np.zeros((self.npixels, self.nsamples), dtype=np.uint16) - wfs_lg_tmp = np.zeros((self.npixels, self.nsamples), dtype=np.uint16) - - wfs_hg_tmp, wfs_lg_tmp = super()._make_event( - event=event, trigger=trigger, return_wfs=True, *args, **kwargs - ) - name = __class__._get_name_trigger(trigger) - - self.__wfs_hg[f"{name}"].append(wfs_hg_tmp.tolist()) - self.__wfs_lg[f"{name}"].append(wfs_lg_tmp.tolist()) - - broken_pixels_hg, broken_pixels_lg = __class__._compute_broken_pixels_event( - event, self._pixels_id - ) - self._broken_pixels_hg[f"{name}"].append(broken_pixels_hg.tolist()) - self._broken_pixels_lg[f"{name}"].append(broken_pixels_lg.tolist()) - - def _make_output_container(self, trigger_type: EventType, *args, **kwargs): - """Make the output container for the selected trigger types. - - Args: - trigger_type (EventType): The selected trigger types. - - Returns: - list[WaveformsContainer]: A list of output containers for the selected trigger types. - """ - output = [] - for trigger in trigger_type: - waveformsContainer = WaveformsContainer( - run_number=self.run_number, - npixels=self._npixels, - nsamples=self._nsamples, - subarray=self._subarray, - camera=self.CAMERA_NAME, - pixels_id=self._pixels_id, - nevents=self.nevents(trigger), - wfs_hg=self.wfs_hg(trigger), - wfs_lg=self.wfs_lg(trigger), - broken_pixels_hg=self.broken_pixels_hg(trigger), - broken_pixels_lg=self.broken_pixels_lg(trigger), - ucts_timestamp=self.ucts_timestamp(trigger), - ucts_busy_counter=self.ucts_busy_counter(trigger), - ucts_event_counter=self.ucts_event_counter(trigger), - event_type=self.event_type(trigger), - event_id=self.event_id(trigger), - trig_pattern_all=self.trig_pattern_all(trigger), - trig_pattern=self.trig_pattern(trigger), - multiplicity=self.multiplicity(trigger), - ) - output.append(waveformsContainer) - return output - - @staticmethod - def sort(waveformsContainer: WaveformsContainer, method: str = "event_id"): - """Sort the waveformsContainer based on a specified method. - - Args: - waveformsContainer (WaveformsContainer): The waveformsContainer to be sorted. - method (str, optional): The sorting method. Defaults to 'event_id'. - - Returns: - WaveformsContainer: The sorted waveformsContainer. - """ - output = WaveformsContainer( - run_number=waveformsContainer.run_number, - npixels=waveformsContainer.npixels, - nsamples=waveformsContainer.nsamples, - subarray=waveformsContainer.subarray, - camera=waveformsContainer.camera, - pixels_id=waveformsContainer.pixels_id, - nevents=waveformsContainer.nevents, - ) - if method == "event_id": - index = np.argsort(waveformsContainer.event_id) - for field in waveformsContainer.keys(): - if not ( - field - in [ - "run_number", - "npixels", - "nsamples", - "subarray", - "camera", - "pixels_id", - "nevents", - ] - ): - output[field] = waveformsContainer[field][index] - else: - raise ArgumentError(f"{method} is not a valid method for sorting") - return output - - @staticmethod - def select_waveforms_hg( - waveformsContainer: WaveformsContainer, pixel_id: np.ndarray - ): - """Select HIGH GAIN waveforms from the container. - - Args: - waveformsContainer (WaveformsContainer): The container object that contains the waveforms. - pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms from the container. - - Returns: - np.ndarray: An array of selected waveforms from the container. - """ - res = __class__.select_container_array_field( - container=waveformsContainer, pixel_id=pixel_id, field="wfs_lg" - ) - res = res.transpose(1, 0, 2) - return res - - @staticmethod - def select_waveforms_lg( - waveformsContainer: WaveformsContainer, pixel_id: np.ndarray - ): - """Select LOW GAIN waveforms from the container. - - Args: - waveformsContainer (WaveformsContainer): The container object that contains the waveforms. - pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms from the container. - - Returns: - np.ndarray: An array of selected waveforms from the container. - """ - res = __class__.select_container_array_field( - container=waveformsContainer, pixel_id=pixel_id, field="wfs_hg" - ) - res = res.transpose(1, 0, 2) - return res - - @property - def _geometry(self): - """ - Returns the private __geometry attribute of the WaveformsMaker class. - - :return: The value of the private __geometry attribute. - """ - return self.__geometry - - @property - def _subarray(self): - """ - Returns the private __subarray attribute of the WaveformsMaker class. - - :return: The value of the private __subarray attribute. - """ - return self.__subarray - - @property - def geometry(self): - """ - Returns a deep copy of the geometry attribute. - - Returns: - A deep copy of the geometry attribute. - """ - return copy.deepcopy(self.__geometry) - - @property - def subarray(self): - """ - Returns a deep copy of the subarray attribute. - - Returns: - A deep copy of the subarray attribute. - """ - return copy.deepcopy(self.__subarray) - - def wfs_hg(self, trigger: EventType): - """ - Returns the waveform data for the specified trigger type. - - Args: - trigger (EventType): The type of trigger for which the waveform data is requested. - - Returns: - An array of waveform data for the specified trigger type. - """ - return np.array( - self.__wfs_hg[__class__._get_name_trigger(trigger)], dtype=np.uint16 - ) - - def wfs_lg(self, trigger: EventType): - """ - Returns the waveform data for the specified trigger type in the low gain channel. - - Args: - trigger (EventType): The type of trigger for which the waveform data is requested. - - Returns: - An array of waveform data for the specified trigger type in the low gain channel. - """ - return np.array( - self.__wfs_lg[__class__._get_name_trigger(trigger)], dtype=np.uint16 - ) + componentsList = ComponentNameList( + NectarCAMComponent, + default_value = ["WaveformsComponent"], + help="List of Component names to be apply, the order will be respected" + ).tag(config=True) + \ No newline at end of file From eb6bf07ecacc8204d5d7e9d9e7b0a17f189bb42d Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 23 Oct 2023 14:34:54 +0200 Subject: [PATCH 61/62] -creation of the component module, which inherits from ctapipe.component -implementation of the NectarCAMComponent, the base class for nectrachain components, - implementation of the ChargesComponent and WaveformsComponent --- src/nectarchain/makers/component/__init__.py | 3 + .../makers/component/chargesComponent.py | 613 ++++++++++++++++++ src/nectarchain/makers/component/core.py | 505 +++++++++++++++ .../makers/component/waveformsComponent.py | 326 ++++++++++ 4 files changed, 1447 insertions(+) create mode 100644 src/nectarchain/makers/component/__init__.py create mode 100644 src/nectarchain/makers/component/chargesComponent.py create mode 100644 src/nectarchain/makers/component/core.py create mode 100644 src/nectarchain/makers/component/waveformsComponent.py diff --git a/src/nectarchain/makers/component/__init__.py b/src/nectarchain/makers/component/__init__.py new file mode 100644 index 00000000..dee30c9d --- /dev/null +++ b/src/nectarchain/makers/component/__init__.py @@ -0,0 +1,3 @@ +from .core import * +from .waveformsComponent import * +from .chargesComponent import * \ No newline at end of file diff --git a/src/nectarchain/makers/component/chargesComponent.py b/src/nectarchain/makers/component/chargesComponent.py new file mode 100644 index 00000000..7a5947e4 --- /dev/null +++ b/src/nectarchain/makers/component/chargesComponent.py @@ -0,0 +1,613 @@ +import logging + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + +import numpy as np +import numpy.ma as ma + +from abc import ABC, abstractmethod +import copy +import tqdm +from argparse import ArgumentError +import time +from numba import bool_, float64, guvectorize, int64 + +from ctapipe.core import Component,TelescopeComponent +from ctapipe.instrument import CameraGeometry +from ctapipe.containers import EventType +from ctapipe.core.traits import List,Unicode,Dict +from ctapipe.instrument import SubarrayDescription +from ctapipe.image.extractor import ( + BaselineSubtractedNeighborPeakWindowSum, + FixedWindowSum, + FullWaveformSum, + GlobalPeakWindowSum, + LocalPeakWindowSum, + NeighborPeakWindowSum, + SlidingWindowMaxSum, + TwoPassWindowSum, +) +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer +from ctapipe_io_nectarcam import NectarCAMEventSource, constants + +from .core import ArrayDataComponent +from ..extractor.utils import CtapipeExtractor +from ...data.container import ChargesContainer,ChargesContainers,WaveformsContainer + +__all__ = ["ChargesComponent"] + +list_ctapipe_charge_extractor = [ + "FullWaveformSum", + "FixedWindowSum", + "GlobalPeakWindowSum", + "LocalPeakWindowSum", + "SlidingWindowMaxSum", + "NeighborPeakWindowSum", + "BaselineSubtractedNeighborPeakWindowSum", + "TwoPassWindowSum", +] + + +list_nectarchain_charge_extractor = ["gradient_extractor"] + + +@guvectorize( + [ + (int64[:], float64[:], bool_, bool_[:], int64[:]), + ], + "(s),(n),()->(n),(n)", + nopython=True, + cache=True, +) +def make_histo(charge, all_range, mask_broken_pix, _mask, hist_ma_data): + """compute histogram of charge with numba + + Args: + charge (np.ndarray(pixels,nevents)): charge + all_range (np.ndarray(nbins)): charge range + mask_broken_pix (np.ndarray(pixels)): mask on broxen pixels + _mask (np.ndarray(pixels,nbins)): mask + hist_ma_data (np.ndarray(pixels,nbins)): histogram + """ + # print(f"charge.shape = {charge.shape[0]}") + # print(f"_mask.shape = {_mask.shape[0]}") + # print(f"_mask[0] = {_mask[0]}") + # print(f"hist_ma_data[0] = {hist_ma_data[0]}") + # print(f"mask_broken_pix = {mask_broken_pix}") + + if not (mask_broken_pix): + # print("this pixel is not broken, let's continue computation") + hist, _charge = np.histogram( + charge, + bins=np.arange( + np.uint16(np.min(charge)) - 1, np.uint16(np.max(charge)) + 2, 1 + ), + ) + # print(f"hist.shape[0] = {hist.shape[0]}") + # print(f"charge.shape[0] = {_charge.shape[0]}") + charge_edges = np.array( + [np.mean(_charge[i : i + 2]) for i in range(_charge.shape[0] - 1)] + ) + # print(f"charge_edges.shape[0] = {charge_edges.shape[0]}") + mask = (all_range >= charge_edges[0]) * (all_range <= charge_edges[-1]) + # print(f"all_range = {int(all_range[0])}-{int(all_range[-1])}") + # print(f"charge_edges[0] = {int(charge_edges[0])}") + # print(f"charge_edges[-1] = {int(charge_edges[-1])}") + # print(f"mask[0] = {mask[0]}") + # print(f"mask[-1] = {mask[-1]}") + + # MASK THE DATA + # print(f"mask.shape = {mask.shape[0]}") + _mask[:] = ~mask + # print(f"_mask[0] = {_mask[0]}") + # print(f"_mask[-1] = {_mask[-1]}") + # FILL THE DATA + hist_ma_data[mask] = hist + # print("work done") + else: + # print("this pixel is broken, skipped") + pass + + + + + + + +class ChargesComponent(ArrayDataComponent) : + + method = Unicode(default_value = "FullWaveformSum", + help = "the charge extraction method", + + ).tag(config = True) + + extractor_kwargs = Dict(default_value = {}, + help = "The kwargs to be pass to the charge extractor method", + ).tag(config = True) + + def __init__(self, subarray, config=None, parent=None,*args, **kwargs): + super().__init__(subarray = subarray, config = config, parent = parent, *args,**kwargs) + + self.__charges_hg = {} + self.__charges_lg = {} + self.__peak_hg = {} + self.__peak_lg = {} + + + def _init_trigger_type(self, trigger_type: EventType, **kwargs): + """ + Initializes the ChargesMaker based on the trigger type. + + Args: + trigger_type (EventType): The type of trigger. + **kwargs: Additional keyword arguments. + + Returns: + None + """ + super()._init_trigger_type(trigger_type, **kwargs) + name = __class__._get_name_trigger(trigger_type) + log.info(f"initialization of the ChargesMaker following trigger type : {name}") + self.__charges_hg[f"{name}"] = [] + self.__charges_lg[f"{name}"] = [] + self.__peak_hg[f"{name}"] = [] + self.__peak_lg[f"{name}"] = [] + + def __call__( + self, + event: NectarCAMDataContainer, + *args, + **kwargs, + ): + + wfs_hg_tmp, wfs_lg_tmp = super(ChargesComponent,self).__call__( + event=event, return_wfs=True, *args, **kwargs + ) + name = __class__._get_name_trigger(event.trigger.event_type) + + broken_pixels_hg, broken_pixels_lg = __class__._compute_broken_pixels_event( + event, self._pixels_id + ) + self._broken_pixels_hg[f"{name}"].append(broken_pixels_hg.tolist()) + self._broken_pixels_lg[f"{name}"].append(broken_pixels_lg.tolist()) + + imageExtractor = __class__._get_imageExtractor( + self.method, self.subarray, **self.extractor_kwargs + ) + + __image = CtapipeExtractor.get_image_peak_time( + imageExtractor( + wfs_hg_tmp, self.TEL_ID, constants.HIGH_GAIN, broken_pixels_hg + ) + ) + self.__charges_hg[f"{name}"].append(__image[0].tolist()) + self.__peak_hg[f"{name}"].append(__image[1].tolist()) + + __image = CtapipeExtractor.get_image_peak_time( + imageExtractor( + wfs_lg_tmp, self.TEL_ID, constants.LOW_GAIN, broken_pixels_lg + ) + ) + self.__charges_lg[f"{name}"].append(__image[0].tolist()) + self.__peak_lg[f"{name}"].append(__image[1].tolist()) + + @staticmethod + def _get_imageExtractor(method: str, subarray: SubarrayDescription, **kwargs): + """ + Create an instance of a charge extraction method based on the provided method name and subarray description. + Args: + method (str): The name of the charge extraction method. + subarray (SubarrayDescription): The description of the subarray. + **kwargs (dict): Additional keyword arguments for the charge extraction method. + Returns: + imageExtractor: An instance of the charge extraction method specified by `method` with the provided subarray description and keyword arguments. + """ + if not ( + method in list_ctapipe_charge_extractor + or method in list_nectarchain_charge_extractor + ): + raise ArgumentError(f"method must be in {list_ctapipe_charge_extractor} or {list_nectarchain_charge_extractor}") + extractor_kwargs = {} + for key in eval(method).class_own_traits().keys(): + if key in kwargs.keys(): + extractor_kwargs[key] = kwargs[key] + if ( + "apply_integration_correction" in eval(method).class_own_traits().keys() + ): # to change the default behavior of ctapipe extractor + extractor_kwargs["apply_integration_correction"] = kwargs.get( + "apply_integration_correction", False + ) + log.debug( + f"Extracting charges with method {method} and extractor_kwargs {extractor_kwargs}" + ) + imageExtractor = eval(method)(subarray, **extractor_kwargs) + return imageExtractor + + def finish( + self, *args, **kwargs + ): + """ + Create an output container for the specified trigger type and method. + Args: + trigger_type (EventType): The type of trigger. + method (str): The name of the charge extraction method. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + Returns: + list: A list of ChargesContainer objects. + """ + output = ChargesContainers() + for i,trigger in enumerate(self.trigger_list): + chargesContainer = ChargesContainer( + run_number=self._run_number, + npixels=self._npixels, + camera=self.CAMERA_NAME, + pixels_id=self._pixels_id, + method=self.method, + nevents=self.nevents(trigger), + charges_hg=self.charges_hg(trigger), + charges_lg=self.charges_lg(trigger), + peak_hg=self.peak_hg(trigger), + peak_lg=self.peak_lg(trigger), + broken_pixels_hg=self.broken_pixels_hg(trigger), + broken_pixels_lg=self.broken_pixels_lg(trigger), + ucts_timestamp=self.ucts_timestamp(trigger), + ucts_busy_counter=self.ucts_busy_counter(trigger), + ucts_event_counter=self.ucts_event_counter(trigger), + event_type=self.event_type(trigger), + event_id=self.event_id(trigger), + trig_pattern_all=self.trig_pattern_all(trigger), + trig_pattern=self.trig_pattern(trigger), + multiplicity=self.multiplicity(trigger), + ) + output.containers[trigger] = chargesContainer + return output + + @staticmethod + def sort(chargesContainer: ChargesContainer, method: str = "event_id"): + """ + Sorts the charges in a ChargesContainer object based on the specified method. + Args: + chargesContainer (ChargesContainer): The ChargesContainer object to be sorted. + method (str, optional): The sorting method. Defaults to 'event_id'. + Returns: + ChargesContainer: A new ChargesContainer object with the charges sorted based on the specified method. + + Raises: + ArgumentError: If the specified method is not valid. + """ + output = ChargesContainer( + run_number=chargesContainer.run_number, + npixels=chargesContainer.npixels, + camera=chargesContainer.camera, + pixels_id=chargesContainer.pixels_id, + nevents=chargesContainer.nevents, + method=chargesContainer.method, + ) + if method == "event_id": + index = np.argsort(chargesContainer.event_id) + for field in chargesContainer.keys(): + if not ( + field + in [ + "run_number", + "npixels", + "camera", + "pixels_id", + "nevents", + "method", + ] + ): + output[field] = chargesContainer[field][index] + else: + raise ArgumentError(f"{method} is not a valid method for sorting") + return output + + @staticmethod + def select_charges_hg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): + """ + Selects the charges from the ChargesContainer object for the given pixel_id and returns the result transposed. + Args: + chargesContainer (ChargesContainer): The ChargesContainer object. + pixel_id (np.ndarray): An array of pixel IDs. + Returns: + np.ndarray: The selected charges from the ChargesContainer object for the given pixel_id, transposed. + """ + res = __class__.select_container_array_field( + container=chargesContainer, pixel_id=pixel_id, field="charges_hg" + ) + res = res.transpose(1, 0) + return res + + @staticmethod + def select_charges_lg(chargesContainer: ChargesContainer, pixel_id: np.ndarray): + """ + Selects the charges from the ChargesContainer object for the given pixel_id and returns the result transposed. + Args: + chargesContainer (ChargesContainer): The ChargesContainer object. + pixel_id (np.ndarray): An array of pixel IDs. + Returns: + np.ndarray: The selected charges from the ChargesContainer object for the given pixel_id, transposed. + """ + res = __class__.select_container_array_field( + container=chargesContainer, pixel_id=pixel_id, field="charges_lg" + ) + res = res.transpose(1, 0) + return res + + def charges_hg(self, trigger: EventType): + """ + Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. + Args: + trigger (EventType): The specific trigger type. + Returns: + np.ndarray: The charges for the specific trigger type. + """ + return np.array( + self.__charges_hg[__class__._get_name_trigger(trigger)], dtype=np.uint16 + ) + + def charges_lg(self, trigger: EventType): + """ + Returns the charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. + Args: + trigger (EventType): The specific trigger type. + Returns: + np.ndarray: The charges for the specific trigger type. + """ + return np.array( + self.__charges_lg[__class__._get_name_trigger(trigger)], dtype=np.uint16 + ) + + def peak_hg(self, trigger: EventType): + """ + Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. + Args: + trigger (EventType): The specific trigger type. + Returns: + np.ndarray: The peak charges for the specific trigger type. + """ + return np.array( + self.__peak_hg[__class__._get_name_trigger(trigger)], dtype=np.uint16 + ) + + def peak_lg(self, trigger: EventType): + """ + Returns the peak charges for a specific trigger type as a NumPy array of unsigned 16-bit integers. + Args: + trigger (EventType): The specific trigger type. + Returns: + np.ndarray: The peak charges for the specific trigger type. + """ + return np.array( + self.__peak_lg[__class__._get_name_trigger(trigger)], dtype=np.uint16 + ) + + @staticmethod + def create_from_waveforms( + waveformsContainer: WaveformsContainer, + method: str = "FullWaveformSum", + **kwargs, + ) -> ChargesContainer: + """ + Create a ChargesContainer object from waveforms using the specified charge extraction method. + Args: + waveformsContainer (WaveformsContainer): The waveforms container object. + method (str, optional): The charge extraction method to use (default is "FullWaveformSum"). + **kwargs: Additional keyword arguments to pass to the charge extraction method. + Returns: + ChargesContainer: The charges container object containing the computed charges and peak times. + """ + chargesContainer = ChargesContainer() + for field in waveformsContainer.keys(): + if not (field in ["nsamples", "wfs_hg", "wfs_lg"]): + chargesContainer[field] = waveformsContainer[field] + log.info(f"computing hg charge with {method} method") + charges_hg, peak_hg = __class__.compute_charge( + waveformsContainer, constants.HIGH_GAIN, method, **kwargs + ) + charges_hg = np.array(charges_hg, dtype=np.uint16) + log.info(f"computing lg charge with {method} method") + charges_lg, peak_lg = __class__.compute_charge( + waveformsContainer, constants.LOW_GAIN, method, **kwargs + ) + charges_lg = np.array(charges_lg, dtype=np.uint16) + chargesContainer.charges_hg = charges_hg + chargesContainer.charges_lg = charges_lg + chargesContainer.peak_hg = peak_hg + chargesContainer.peak_lg = peak_lg + chargesContainer.method = method + return chargesContainer + + @staticmethod + def compute_charge( + waveformContainer: WaveformsContainer, + channel: int, + subarray : SubarrayDescription, + method: str = "FullWaveformSum", + tel_id : int = None, + **kwargs, + ): + """ + Compute charge from waveforms. + Args: + waveformContainer (WaveformsContainer): The waveforms container object. + channel (int): The channel to compute charges for. + method (str, optional): The charge extraction method to use (default is "FullWaveformSum"). + **kwargs: Additional keyword arguments to pass to the charge extraction method. + Raises: + ArgumentError: If the extraction method is unknown. + ArgumentError: If the channel is unknown. + Returns: + tuple: A tuple containing the computed charges and peak times. + """ + # import is here for fix issue with pytest (TypeError : inference is not possible with python <3.9 (Numba conflict bc there is no inference...)) + from ..extractor.utils import CtapipeExtractor + + if tel_id is None : + tel_id = __class__.TEL_ID.default_value + + imageExtractor = __class__._get_imageExtractor( + method=method, subarray=subarray, **kwargs + ) + if channel == constants.HIGH_GAIN: + out = np.array( + [ + CtapipeExtractor.get_image_peak_time( + imageExtractor( + waveformContainer.wfs_hg[i], + tel_id, + channel, + waveformContainer.broken_pixels_hg, + ) + ) + for i in range(len(waveformContainer.wfs_hg)) + ] + ).transpose(1, 0, 2) + return out[0], out[1] + elif channel == constants.LOW_GAIN: + out = np.array( + [ + CtapipeExtractor.get_image_peak_time( + imageExtractor( + waveformContainer.wfs_lg[i], + tel_id, + channel, + waveformContainer.broken_pixels_lg, + ) + ) + for i in range(len(waveformContainer.wfs_lg)) + ] + ).transpose(1, 0, 2) + return out[0], out[1] + else: + raise ArgumentError( + f"channel must be {constants.LOW_GAIN} or {constants.HIGH_GAIN}" + ) + + @staticmethod + def histo_hg( + chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True + ) -> ma.masked_array: + """ + Computes histogram of high gain charges from a ChargesContainer object. + + Args: + chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. + n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. + autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. + + Returns: + ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. + """ + return __class__._histo( + chargesContainer=chargesContainer, + field="charges_hg", + n_bins=n_bins, + autoscale=autoscale, + ) + + @staticmethod + def histo_lg( + chargesContainer: ChargesContainer, n_bins: int = 1000, autoscale: bool = True + ) -> ma.masked_array: + """ + Computes histogram of low gain charges from a ChargesContainer object. + + Args: + chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. + n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. + autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. + + Returns: + ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. + """ + return __class__._histo( + chargesContainer=chargesContainer, + field="charges_lg", + n_bins=n_bins, + autoscale=autoscale, + ) + + @staticmethod + def _histo( + chargesContainer: ChargesContainer, + field: str, + n_bins: int = 1000, + autoscale: bool = True, + ) -> ma.masked_array: + """ + Computes histogram of charges for a given field from a ChargesContainer object. + Numba is used to compute histograms in a vectorized way. + + Args: + chargesContainer (ChargesContainer): A ChargesContainer object that holds information about charges from a specific run. + field (str): The field name for which the histogram is computed. + n_bins (int, optional): The number of bins in the charge histogram. Defaults to 1000. + autoscale (bool, optional): Whether to automatically detect the number of bins based on the pixel data. Defaults to True. + + Returns: + ma.masked_array: A masked array representing the charge histogram, where each row corresponds to an event and each column corresponds to a bin in the histogram. + """ + mask_broken_pix = np.array( + (chargesContainer[field] == chargesContainer[field].mean(axis=0)).mean( + axis=0 + ), + dtype=bool, + ) + log.debug( + f"there are {mask_broken_pix.sum()} broken pixels (charge stays at same level for each events)" + ) + + if autoscale: + all_range = np.arange( + np.uint16(np.min(chargesContainer[field].T[~mask_broken_pix].T)) - 0.5, + np.uint16(np.max(chargesContainer[field].T[~mask_broken_pix].T)) + 1.5, + 1, + ) + charge_ma = ma.masked_array( + ( + all_range.reshape(all_range.shape[0], 1) + @ np.ones((1, chargesContainer[field].shape[1])) + ).T, + mask=np.zeros( + (chargesContainer[field].shape[1], all_range.shape[0]), dtype=bool + ), + ) + broxen_pixels_mask = np.array( + [mask_broken_pix for i in range(charge_ma.shape[1])] + ).T + start = time.time() + _mask, hist_ma_data = make_histo( + chargesContainer[field].T, all_range, mask_broken_pix + ) + charge_ma.mask = np.logical_or(_mask, broxen_pixels_mask) + hist_ma = ma.masked_array(hist_ma_data, mask=charge_ma.mask) + log.debug(f"histogram hg computation time : {time.time() - start} sec") + + return ma.masked_array((hist_ma, charge_ma)) + + else: + hist = np.array( + [ + np.histogram(chargesContainer[field].T[i], bins=n_bins)[0] + for i in range(chargesContainer[field].shape[1]) + ] + ) + charge = np.array( + [ + np.histogram(chargesContainer[field].T[i], bins=n_bins)[1] + for i in range(chargesContainer[field].shape[1]) + ] + ) + charge_edges = np.array( + [ + np.mean(charge.T[i : i + 2], axis=0) + for i in range(charge.shape[1] - 1) + ] + ).T + + return np.array((hist, charge_edges)) diff --git a/src/nectarchain/makers/component/core.py b/src/nectarchain/makers/component/core.py new file mode 100644 index 00000000..c6fb5eb2 --- /dev/null +++ b/src/nectarchain/makers/component/core.py @@ -0,0 +1,505 @@ +import logging + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + +import numpy as np +from abc import ABC, abstractmethod +import copy + +from ctapipe.core import Component,TelescopeComponent +from ctapipe.instrument import CameraGeometry +from ctapipe.containers import EventType +from ctapipe.core.traits import Unicode,Integer + + +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer +from ctapipe_io_nectarcam import NectarCAMEventSource, constants + +from ...data.container.core import ArrayDataContainer + + +__all__ = ["ArrayDataComponent", + "NectarCAMComponent", + "get_valid_component", + "get_specific_traits", + "get_configurable_traits", + ] + +def get_valid_component() : + return NectarCAMComponent.non_abstract_subclasses() + +def get_specific_traits(component : Component) : + traits_dict = component.class_traits() + traits_dict.pop("config",True) + traits_dict.pop("parent",True) + return traits_dict + +def get_configurable_traits(component : Component) : + traits_dict = get_specific_traits(component) + output_traits_dict = traits_dict.copy() + for key,item in traits_dict.items() : + if item.read_only : + output_traits_dict.pop(key) + return output_traits_dict + +class NectarCAMComponent(TelescopeComponent) : + """The base class for NectarCAM components""" + def __init__(self, subarray, config=None, parent=None,*args, **kwargs): + super().__init__(subarray = subarray, config = config, parent = parent, *args,**kwargs) + self.__pixels_id = parent._event_source.camera_config.expected_pixels_id + self.__run_number = parent.run_number + self.__npixels=parent.npixels + @abstractmethod + def __call__( + self, event: NectarCAMDataContainer, *args, **kwargs + ): + pass + + @property + def _pixels_id(self) : return self.__pixels_id + @property + def pixels_id(self) : return copy.deepcopy(self.__pixels_id) + @property + def _run_number(self) : + return self.__run_number + @property + def run_number(self) : + return copy.deepcopy(self.__run_number) + @property + def _npixels(self) : + return self.__npixels + @property + def npixels(self) : + return copy.deepcopy(self.__npixels) + + +#class NectarCAMTelescopeComponent(TelescopeComponent) : +# """The base class for NectarCAM telescope component""" +# pass + + +class ArrayDataComponent(NectarCAMComponent) : + TEL_ID = Integer(default_value = 0, + help = "The telescope ID", + read_only = True, + ).tag(config = True) + + CAMERA_NAME = Unicode(default_value = "NectarCam-003", + help = "The camera name", + read_only = True, + ).tag(config = True) + + CAMERA = CameraGeometry.from_name(CAMERA_NAME.default_value) + + #trigger_list = List( + # help="List of trigger(EventType) inside the instance", + # default_value=[], + #).tag(config=True) + + + + def __init__(self, subarray, config=None, parent=None,*args, **kwargs): + super().__init__(subarray = subarray,config = config, parent = parent,*args, **kwargs) + self.__nsamples = parent._event_source.camera_config.num_samples + + + self.trigger_list = [] + + # data we want to compute + self.__ucts_timestamp = {} + self.__ucts_busy_counter = {} + self.__ucts_event_counter = {} + self.__event_type = {} + self.__event_id = {} + self.__trig_patter_all = {} + self.__broken_pixels_hg = {} + self.__broken_pixels_lg = {} + + def _init_trigger_type(self, trigger: EventType, **kwargs): + """ + Initializes empty lists for different trigger types in the ArrayDataMaker class. + + Args: + trigger (EventType): The trigger type for which the lists are being initialized. + + Returns: + None. The method only initializes the empty lists for the trigger type. + """ + name = __class__._get_name_trigger(trigger) + self.__ucts_timestamp[f"{name}"] = [] + self.__ucts_busy_counter[f"{name}"] = [] + self.__ucts_event_counter[f"{name}"] = [] + self.__event_type[f"{name}"] = [] + self.__event_id[f"{name}"] = [] + self.__trig_patter_all[f"{name}"] = [] + self.__broken_pixels_hg[f"{name}"] = [] + self.__broken_pixels_lg[f"{name}"] = [] + self.trigger_list.append(trigger) + + @staticmethod + def _compute_broken_pixels(wfs_hg, wfs_lg, **kwargs): + """ + Computes broken pixels for high and low gain waveforms. + Args: + wfs_hg (ndarray): High gain waveforms. + wfs_lg (ndarray): Low gain waveforms. + **kwargs: Additional keyword arguments. + Returns: + tuple: Two arrays of zeros with the same shape as `wfs_hg` (or `wfs_lg`) but without the last dimension. + """ + log.warning("computation of broken pixels is not yet implemented") + return np.zeros((wfs_hg.shape[:-1]), dtype=bool), np.zeros( + (wfs_hg.shape[:-1]), dtype=bool + ) + + @staticmethod + def _compute_broken_pixels_event( + event: NectarCAMDataContainer, pixels_id: np.ndarray, **kwargs + ): + """ + Computes broken pixels for a specific event and pixel IDs. + Args: + event (NectarCAMDataContainer): An event. + pixels_id (list or np.ndarray): IDs of pixels. + **kwargs: Additional keyword arguments. + Returns: + tuple: Two arrays of zeros with the length of `pixels_id`. + """ + log.warning("computation of broken pixels is not yet implemented") + return np.zeros((len(pixels_id)), dtype=bool), np.zeros( + (len(pixels_id)), dtype=bool + ) + + @staticmethod + def _get_name_trigger(trigger: EventType): + """ + Gets the name of a trigger event. + Args: + trigger (EventType): A trigger event. + Returns: + str: The name of the trigger event. + """ + if trigger is None: + name = "None" + else: + name = trigger.name + return name + + def __call__( + self, event: NectarCAMDataContainer, *args, **kwargs + ): + """ + Method to extract data from the event. + + Args: + event (NectarCAMDataContainer): The event object. + trigger (EventType): The trigger type. + *args: Additional arguments that can be passed to the method. + **kwargs: Additional keyword arguments that can be passed to the method. + + Returns: + If the return_wfs keyword argument is True, the method returns the high and low gain waveforms from the event. + """ + name = __class__._get_name_trigger(event.trigger.event_type) + + if not(name in self.__event_id.keys()) : + self._init_trigger_type(event.trigger.event_type) + + self.__event_id[f"{name}"].append(np.uint32(event.index.event_id)) + self.__ucts_timestamp[f"{name}"].append( + event.nectarcam.tel[__class__.TEL_ID.default_value].evt.ucts_timestamp + ) + self.__event_type[f"{name}"].append(event.trigger.event_type.value) + self.__ucts_busy_counter[f"{name}"].append( + event.nectarcam.tel[__class__.TEL_ID.default_value].evt.ucts_busy_counter + ) + self.__ucts_event_counter[f"{name}"].append( + event.nectarcam.tel[__class__.TEL_ID.default_value].evt.ucts_event_counter + ) + self.__trig_patter_all[f"{name}"].append( + event.nectarcam.tel[__class__.TEL_ID.default_value].evt.trigger_pattern.T + ) + + if kwargs.get("return_wfs", False): + get_wfs_hg = event.r0.tel[0].waveform[constants.HIGH_GAIN][self.pixels_id] + get_wfs_lg = event.r0.tel[0].waveform[constants.LOW_GAIN][self.pixels_id] + return get_wfs_hg, get_wfs_lg + + + + @abstractmethod + def finish(self): + pass + + @staticmethod + def select_container_array_field( + container: ArrayDataContainer, pixel_id: np.ndarray, field: str + ) -> np.ndarray: + """ + Selects specific fields from an ArrayDataContainer object based on a given list of pixel IDs. + + Args: + container (ArrayDataContainer): An object of type ArrayDataContainer that contains the data. + pixel_id (ndarray): An array of pixel IDs for which the data needs to be selected. + field (str): The name of the field to be selected from the container. + + Returns: + ndarray: An array containing the selected data for the given pixel IDs. + """ + mask_contain_pixels_id = np.array( + [pixel in container.pixels_id for pixel in pixel_id], dtype=bool + ) + for pixel in pixel_id[~mask_contain_pixels_id]: + log.warning( + f"You asked for pixel_id {pixel} but it is not present in this container, skip this one" + ) + res = np.array( + [ + np.take( + container[field], + np.where(container.pixels_id == pixel)[0][0], + axis=1, + ) + for pixel in pixel_id[mask_contain_pixels_id] + ] + ) + ####could be nice to return np.ma.masked_array(data = res, mask = container.broken_pixels_hg.transpose(res.shape[1],res.shape[0],res.shape[2])) + return res + + @staticmethod + def merge( + container_a: ArrayDataContainer, container_b: ArrayDataContainer + ) -> ArrayDataContainer: + """method to merge 2 ArrayDataContainer into one single ArrayDataContainer + + Returns: + ArrayDataContainer: the merged object + """ + if type(container_a) != type(container_b): + raise Exception("The containers have to be instnace of the same class") + + if np.array_equal(container_a.pixels_id, container_b.pixels_id): + raise Exception("The containers have not the same pixels ids") + + merged_container = container_a.__class__.__new__() + + for field in container_a.keys(): + if ~isinstance(container_a[field], np.ndarray): + if container_a[field] != container_b[field]: + raise Exception( + f"merge impossible because of {field} filed (values are {container_a[field]} and {container_b[field]}" + ) + + for field in container_a.keys(): + if isinstance(container_a[field], np.ndarray): + merged_container[field] = np.concatenate( + container_a[field], container_a[field], axis=0 + ) + else: + merged_container[field] = container_a[field] + + return merged_container + + @property + def nsamples(self): + """ + Returns a deep copy of the nsamples attribute. + + Returns: + np.ndarray: A deep copy of the nsamples attribute. + """ + return copy.deepcopy(self.__nsamples) + + @property + def _nsamples(self): + """ + Returns the nsamples attribute. + + Returns: + np.ndarray: The nsamples attribute. + """ + return self.__nsamples + + + def nevents(self, trigger: EventType): + """ + Returns the number of events for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the number of events is requested. + + Returns: + int: The number of events for the specified trigger type. + """ + return len(self.__event_id[__class__._get_name_trigger(trigger)]) + + @property + def _broken_pixels_hg(self): + """ + Returns the broken_pixels_hg attribute. + + Returns: + np.ndarray: The broken_pixels_hg attribute. + """ + return self.__broken_pixels_hg + + def broken_pixels_hg(self, trigger: EventType): + """ + Returns an array of broken pixels for high gain for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the broken pixels for high gain are requested. + + Returns: + np.ndarray: An array of broken pixels for high gain for the specified trigger type. + """ + return np.array( + self.__broken_pixels_hg[__class__._get_name_trigger(trigger)], dtype=bool + ) + + @property + def _broken_pixels_lg(self): + """ + Returns the broken_pixels_lg attribute. + + Returns: + np.ndarray: The broken_pixels_lg attribute. + """ + return self.__broken_pixels_lg + + def broken_pixels_lg(self, trigger: EventType): + """ + Returns an array of broken pixels for low gain for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the broken pixels for low gain are requested. + + Returns: + np.ndarray: An array of broken pixels for low gain for the specified trigger type. + """ + return np.array( + self.__broken_pixels_lg[__class__._get_name_trigger(trigger)], dtype=bool + ) + + def ucts_timestamp(self, trigger: EventType): + """ + Returns an array of UCTS timestamps for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the UCTS timestamps are requested. + + Returns: + np.ndarray: An array of UCTS timestamps for the specified trigger type. + """ + return np.array( + self.__ucts_timestamp[__class__._get_name_trigger(trigger)], dtype=np.uint64 + ) + + def ucts_busy_counter(self, trigger: EventType): + """ + Returns an array of UCTS busy counters for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the UCTS busy counters are requested. + + Returns: + np.ndarray: An array of UCTS busy counters for the specified trigger type. + """ + return np.array( + self.__ucts_busy_counter[__class__._get_name_trigger(trigger)], + dtype=np.uint32, + ) + + def ucts_event_counter(self, trigger: EventType): + """ + Returns an array of UCTS event counters for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the UCTS event counters are requested. + + Returns: + np.ndarray: An array of UCTS event counters for the specified trigger type. + """ + return np.array( + self.__ucts_event_counter[__class__._get_name_trigger(trigger)], + dtype=np.uint32, + ) + + def event_type(self, trigger: EventType): + """ + Returns an array of event types for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the event types are requested. + + Returns: + np.ndarray: An array of event types for the specified trigger type. + """ + return np.array( + self.__event_type[__class__._get_name_trigger(trigger)], dtype=np.uint8 + ) + + def event_id(self, trigger: EventType): + """ + Returns an array of event IDs for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the event IDs are requested. + + Returns: + np.ndarray: An array of event IDs for the specified trigger type. + """ + return np.array( + self.__event_id[__class__._get_name_trigger(trigger)], dtype=np.uint32 + ) + + def multiplicity(self, trigger: EventType): + """ + Returns an array of multiplicities for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the multiplicities are requested. + + Returns: + np.ndarray: An array of multiplicities for the specified trigger type. + """ + tmp = self.trig_pattern(trigger) + if len(tmp) == 0: + return np.array([]) + else: + return np.uint16(np.count_nonzero(tmp, axis=1)) + + def trig_pattern(self, trigger: EventType): + """ + Returns an array of trigger patterns for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the trigger patterns are requested. + + Returns: + np.ndarray: An array of trigger patterns for the specified trigger type. + """ + tmp = self.trig_pattern_all(trigger) + if len(tmp) == 0: + return np.array([]) + else: + return tmp.any(axis=2) + + def trig_pattern_all(self, trigger: EventType): + """ + Returns an array of trigger patterns for all events for the specified trigger type. + + Args: + trigger (EventType): The trigger type for which the trigger patterns for all events are requested. + + Returns: + np.ndarray: An array of trigger patterns for all events for the specified trigger type. + """ + return np.array( + self.__trig_patter_all[f"{__class__._get_name_trigger(trigger)}"], + dtype=bool, + ) + + \ No newline at end of file diff --git a/src/nectarchain/makers/component/waveformsComponent.py b/src/nectarchain/makers/component/waveformsComponent.py new file mode 100644 index 00000000..6874ccfb --- /dev/null +++ b/src/nectarchain/makers/component/waveformsComponent.py @@ -0,0 +1,326 @@ +import logging + +logging.basicConfig(format="%(asctime)s %(name)s %(levelname)s %(message)s") +log = logging.getLogger(__name__) +log.handlers = logging.getLogger("__main__").handlers + + +import numpy as np +from abc import ABC, abstractmethod +import copy +import tqdm +from argparse import ArgumentError + + +from ctapipe.core import Component,TelescopeComponent +from ctapipe.instrument import CameraGeometry +from ctapipe.containers import EventType +from ctapipe.core.traits import List +from ctapipe.instrument import SubarrayDescription + + + + +from ctapipe_io_nectarcam.containers import NectarCAMDataContainer +from ctapipe_io_nectarcam import NectarCAMEventSource, constants + +from .core import ArrayDataComponent +from ...data.container import WaveformsContainer,WaveformsContainers + +__all__ = ["WaveformsComponent"] + +class WaveformsComponent(ArrayDataComponent) : + + def __init__(self, subarray, config=None, parent=None,*args, **kwargs): + super().__init__(subarray = subarray, config = config, parent = parent, *args,**kwargs) + + self.__geometry = subarray.tel[self.TEL_ID].camera + self.__wfs_hg = {} + self.__wfs_lg = {} + + @staticmethod + def create_from_events_list( + events_list: list, + run_number: int, + npixels: int, + nsamples: int, + subarray: SubarrayDescription, + pixels_id: int, + tel_id : int = None, + ) -> WaveformsContainer: + """Create a container for the extracted waveforms from a list of events. + + Args: + events_list (list[NectarCAMDataContainer]): A list of events to extract waveforms from. + run_number (int): The ID of the run to be loaded. + npixels (int): The number of pixels in the waveforms. + nsamples (int): The number of samples in the waveforms. + subarray (SubarrayDescription): The subarray description instance. + pixels_id (int): The ID of the pixels to extract waveforms from. + + Returns: + WaveformsContainer: A container object that contains the extracted waveforms and other relevant information. + """ + if tel_id is None : + tel_id = __class__.TEL_ID.default_value + + container = WaveformsContainer( + run_number=run_number, + npixels=npixels, + nsamples=nsamples, + subarray=subarray, + camera=__class__.CAMERA_NAME, + pixels_id=pixels_id, + ) + + ucts_timestamp = [] + ucts_busy_counter = [] + ucts_event_counter = [] + event_type = [] + event_id = [] + trig_pattern_all = [] + wfs_hg = [] + wfs_lg = [] + + for event in tqdm(events_list): + ucts_timestamp.append( + event.nectarcam.tel[tel_id].evt.ucts_timestamp + ) + ucts_busy_counter.append( + event.nectarcam.tel[tel_id].evt.ucts_busy_counter + ) + ucts_event_counter.append( + event.nectarcam.tel[tel_id].evt.ucts_event_counter + ) + event_type.append(event.trigger.event_type.value) + event_id.append(event.index.event_id) + trig_pattern_all.append( + event.nectarcam.tel[tel_id].evt.trigger_pattern.T + ) + wfs_hg.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][pixels_id]) + wfs_lg.append(event.r0.tel[0].waveform[constants.HIGH_GAIN][pixels_id]) + + container.wfs_hg = np.array(wfs_hg, dtype=np.uint16) + container.wfs_lg = np.array(wfs_lg, dtype=np.uint16) + + container.ucts_timestamp = np.array(ucts_timestamp, dtype=np.uint64) + container.ucts_busy_counter = np.array(ucts_busy_counter, dtype=np.uint32) + container.ucts_event_counter = np.array(ucts_event_counter, dtype=np.uint32) + container.event_type = np.array(event_type, dtype=np.uint8) + container.event_id = np.array(event_id, dtype=np.uint32) + container.trig_pattern_all = np.array(trig_pattern_all, dtype=bool) + container.trig_pattern = container.trig_pattern_all.any(axis=2) + container.multiplicity = np.uint16( + np.count_nonzero(container.trig_pattern, axis=1) + ) + + broken_pixels = __class__._compute_broken_pixels( + container.wfs_hg, container.wfs_lg + ) + container.broken_pixels_hg = broken_pixels[0] + container.broken_pixels_lg = broken_pixels[1] + return container + + def _init_trigger_type(self, trigger_type: EventType, **kwargs): + """Initialize the waveformsMaker following the trigger type. + + Args: + trigger_type: The type of trigger. + + """ + super()._init_trigger_type(trigger_type, **kwargs) + name = __class__._get_name_trigger(trigger_type) + log.info( + f"initialization of the waveformsMaker following trigger type : {name}" + ) + self.__wfs_hg[f"{name}"] = [] + self.__wfs_lg[f"{name}"] = [] + + def __call__( + self, event: NectarCAMDataContainer, *args, **kwargs + ): + """Process an event and extract waveforms. + + Args: + event (NectarCAMDataContainer): The event to process and extract waveforms from. + trigger (EventType): The type of trigger for the event. + + """ + wfs_hg_tmp = np.zeros((self.npixels, self.nsamples), dtype=np.uint16) + wfs_lg_tmp = np.zeros((self.npixels, self.nsamples), dtype=np.uint16) + + wfs_hg_tmp, wfs_lg_tmp = super(WaveformsComponent,self).__call__( + event=event, return_wfs=True, *args, **kwargs + ) + name = __class__._get_name_trigger(event.trigger.event_type) + + self.__wfs_hg[f"{name}"].append(wfs_hg_tmp.tolist()) + self.__wfs_lg[f"{name}"].append(wfs_lg_tmp.tolist()) + + broken_pixels_hg, broken_pixels_lg = __class__._compute_broken_pixels_event( + event, self._pixels_id + ) + self._broken_pixels_hg[f"{name}"].append(broken_pixels_hg.tolist()) + self._broken_pixels_lg[f"{name}"].append(broken_pixels_lg.tolist()) + + def finish(self, *args, **kwargs): + """Make the output container for the selected trigger types. + + Args: + trigger_type (EventType): The selected trigger types. + + Returns: + list[WaveformsContainer]: A list of output containers for the selected trigger types. + """ + output = WaveformsContainers() + for i,trigger in enumerate(self.trigger_list): + waveformsContainer = WaveformsContainer( + run_number=self._run_number, + npixels=self._npixels, + nsamples=self._nsamples, + #subarray=self.subarray, + camera=self.CAMERA_NAME, + pixels_id=self._pixels_id, + nevents=self.nevents(trigger), + wfs_hg=self.wfs_hg(trigger), + wfs_lg=self.wfs_lg(trigger), + broken_pixels_hg=self.broken_pixels_hg(trigger), + broken_pixels_lg=self.broken_pixels_lg(trigger), + ucts_timestamp=self.ucts_timestamp(trigger), + ucts_busy_counter=self.ucts_busy_counter(trigger), + ucts_event_counter=self.ucts_event_counter(trigger), + event_type=self.event_type(trigger), + event_id=self.event_id(trigger), + trig_pattern_all=self.trig_pattern_all(trigger), + trig_pattern=self.trig_pattern(trigger), + multiplicity=self.multiplicity(trigger), + ) + output.containers[trigger] = waveformsContainer + return output + + @staticmethod + def sort(waveformsContainer: WaveformsContainer, method: str = "event_id"): + """Sort the waveformsContainer based on a specified method. + + Args: + waveformsContainer (WaveformsContainer): The waveformsContainer to be sorted. + method (str, optional): The sorting method. Defaults to 'event_id'. + + Returns: + WaveformsContainer: The sorted waveformsContainer. + """ + output = WaveformsContainer( + run_number=waveformsContainer.run_number, + npixels=waveformsContainer.npixels, + nsamples=waveformsContainer.nsamples, + subarray=waveformsContainer.subarray, + camera=waveformsContainer.camera, + pixels_id=waveformsContainer.pixels_id, + nevents=waveformsContainer.nevents, + ) + if method == "event_id": + index = np.argsort(waveformsContainer.event_id) + for field in waveformsContainer.keys(): + if not ( + field + in [ + "run_number", + "npixels", + "nsamples", + "subarray", + "camera", + "pixels_id", + "nevents", + ] + ): + output[field] = waveformsContainer[field][index] + else: + raise ArgumentError(f"{method} is not a valid method for sorting") + return output + + @staticmethod + def select_waveforms_hg( + waveformsContainer: WaveformsContainer, pixel_id: np.ndarray + ): + """Select HIGH GAIN waveforms from the container. + + Args: + waveformsContainer (WaveformsContainer): The container object that contains the waveforms. + pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms from the container. + + Returns: + np.ndarray: An array of selected waveforms from the container. + """ + res = __class__.select_container_array_field( + container=waveformsContainer, pixel_id=pixel_id, field="wfs_lg" + ) + res = res.transpose(1, 0, 2) + return res + + @staticmethod + def select_waveforms_lg( + waveformsContainer: WaveformsContainer, pixel_id: np.ndarray + ): + """Select LOW GAIN waveforms from the container. + + Args: + waveformsContainer (WaveformsContainer): The container object that contains the waveforms. + pixel_id (np.ndarray): An array of pixel IDs to select specific waveforms from the container. + + Returns: + np.ndarray: An array of selected waveforms from the container. + """ + res = __class__.select_container_array_field( + container=waveformsContainer, pixel_id=pixel_id, field="wfs_hg" + ) + res = res.transpose(1, 0, 2) + return res + + @property + def _geometry(self): + """ + Returns the private __geometry attribute of the WaveformsMaker class. + + :return: The value of the private __geometry attribute. + """ + return self.__geometry + + + @property + def geometry(self): + """ + Returns a deep copy of the geometry attribute. + + Returns: + A deep copy of the geometry attribute. + """ + return copy.deepcopy(self.__geometry) + + + def wfs_hg(self, trigger: EventType): + """ + Returns the waveform data for the specified trigger type. + + Args: + trigger (EventType): The type of trigger for which the waveform data is requested. + + Returns: + An array of waveform data for the specified trigger type. + """ + return np.array( + self.__wfs_hg[__class__._get_name_trigger(trigger)], dtype=np.uint16 + ) + + def wfs_lg(self, trigger: EventType): + """ + Returns the waveform data for the specified trigger type in the low gain channel. + + Args: + trigger (EventType): The type of trigger for which the waveform data is requested. + + Returns: + An array of waveform data for the specified trigger type in the low gain channel. + """ + return np.array( + self.__wfs_lg[__class__._get_name_trigger(trigger)], dtype=np.uint16 + ) From ecaa250c34cdabbd99f33d1404f47f926f6f0fdd Mon Sep 17 00:00:00 2001 From: "guillaume.grolleron" Date: Mon, 23 Oct 2023 14:42:49 +0200 Subject: [PATCH 62/62] -preliminary tuto script to extract charges or waveforms using Tool and Component classes --- ...eforms_charges_extraction_new_framework.py | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 src/nectarchain/user_scripts/ggrolleron/notebooks/waveforms_charges_extraction_new_framework.py diff --git a/src/nectarchain/user_scripts/ggrolleron/notebooks/waveforms_charges_extraction_new_framework.py b/src/nectarchain/user_scripts/ggrolleron/notebooks/waveforms_charges_extraction_new_framework.py new file mode 100644 index 00000000..94b7fa53 --- /dev/null +++ b/src/nectarchain/user_scripts/ggrolleron/notebooks/waveforms_charges_extraction_new_framework.py @@ -0,0 +1,96 @@ +# %% +from nectarchain.makers import ChargesNectarCAMCalibrationTool,WaveformsNectarCAMCalibrationTool +from nectarchain.makers.component import get_valid_component +from nectarchain.data.container import ChargesContainers,ChargesContainer,WaveformsContainer,WaveformsContainers +from ctapipe.io import HDF5TableReader +from ctapipe.containers import EventType + + +# %% +get_valid_component() + +# %% +run_number = 3942 + +# %% +tool = WaveformsNectarCAMCalibrationTool(progress_bar = True,run_number = run_number,max_events = 500,log_level = 10) +tool + +# %% +tool = ChargesNectarCAMCalibrationTool(progress_bar = True, + method = 'LocalPeakWindowSum', + extractor_kwargs = {"window_width": 16, "window_shift": 4}, + run_number = run_number, + max_events = 5000, + log_level = 10 + ) +tool + +# %% +tool.initialize() + +# %% +tool.setup() + +# %% +tool.start() + +# %% +tool.finish() + +# %% +container = WaveformsContainers() +trigger_type = EventType.__members__ + + +with HDF5TableReader(f"/tmp/EventsLoopNectarCAMCalibrationTool_{run_number}.h5") as reader : + for key,trigger in trigger_type.items() : + try : + tableReader = reader.read(table_name = f"/data/WaveformsContainer_0/{trigger.name}", containers = WaveformsContainer) + container.containers[trigger] = next(tableReader) + except Exception as err: + print(err) + +# %% +container.containers + +# %% +container = ChargesContainers() +trigger_type = EventType.__members__ + + +with HDF5TableReader(f"/tmp/EventsLoopNectarCAMCalibrationTool_{run_number}.h5") as reader : + for key,trigger in trigger_type.items() : + try : + tableReader = reader.read(table_name = f"/data/ChargesContainer_0/{trigger.name}", containers = ChargesContainer) + container.containers[trigger] = next(tableReader) + except Exception as err: + print(err) + +# %% +container.containers + +# %% +from nectarchain.makers.component import ChargesComponent +import matplotlib.pyplot as plt +import numpy as np + +# %% +counts,charge = ChargesComponent.histo_hg(container.containers[EventType.FLATFIELD]) +charge.shape,counts.shape + +# %% +fig,ax = plt.subplots(1,1,figsize = (7,7)) +ax.errorbar( + charge[30], + counts[30], + np.sqrt(counts[30]), + zorder=0, + fmt=".", + label="data" + ) + +# %% + + +