From 2c3e55246422baf68479f427d2cc28d0a3fdf4d4 Mon Sep 17 00:00:00 2001 From: jlenain Date: Fri, 28 Jul 2023 15:40:49 +0200 Subject: [PATCH] Lint DQM --- src/nectarchain/dqm/__init__.py | 3 +- src/nectarchain/dqm/camera_monitoring.py | 118 ++++++++++--------- src/nectarchain/dqm/charge_integration.py | 37 +++--- src/nectarchain/dqm/dqm_summary_processor.py | 30 +++-- src/nectarchain/dqm/mean_camera_display.py | 47 +++----- src/nectarchain/dqm/mean_waveforms.py | 26 ++-- src/nectarchain/dqm/start_calib.py | 98 ++++++++------- src/nectarchain/dqm/trigger_statistics.py | 7 +- 8 files changed, 190 insertions(+), 176 deletions(-) diff --git a/src/nectarchain/dqm/__init__.py b/src/nectarchain/dqm/__init__.py index d7dc36cc..cb255dcd 100644 --- a/src/nectarchain/dqm/__init__.py +++ b/src/nectarchain/dqm/__init__.py @@ -1,3 +1,4 @@ from .db_utils import DQMDB +from .dqm_summary_processor import DQMSummary -__all__ = ["DQMDB"] +__all__ = ["DQMDB", "DQMSummary"] diff --git a/src/nectarchain/dqm/camera_monitoring.py b/src/nectarchain/dqm/camera_monitoring.py index ff56dece..3a698730 100644 --- a/src/nectarchain/dqm/camera_monitoring.py +++ b/src/nectarchain/dqm/camera_monitoring.py @@ -1,24 +1,26 @@ -from dqm_summary_processor import dqm_summary -from matplotlib import pyplot as plt -from ctapipe.visualization import CameraDisplay -from ctapipe.instrument import CameraGeometry -from ctapipe.coordinates import EngineeringCameraFrame -from astropy import time as astropytime -import numpy as np +import os import sqlite3 +import numpy as np +from astropy import time as astropytime +from ctapipe.coordinates import EngineeringCameraFrame +from ctapipe.visualization import CameraDisplay +from dqm_summary_processor import DQMSummary +from matplotlib import pyplot as plt -class CameraMonitoring(dqm_summary): + +class CameraMonitoring(DQMSummary): def __init__(self, gaink): self.k = gaink - return None def ConfigureForRun(self, path, Pix, Samp, Reader1): # define number of pixels and samples self.Pix = Pix self.Samp = Samp - self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame()) + self.camera = CameraGeometry.from_name("NectarCam-003").transform_to( + EngineeringCameraFrame() + ) self.cmap = "gnuplot2" self.subarray = Reader1.subarray @@ -29,26 +31,24 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): for i, evt1 in enumerate(Reader1): self.run_start1 = evt1.nectarcam.tel[0].svc.date - SqlFileDate = astropytime.Time(self.run_start1, format="unix").iso.split(" ")[ - 0 - ] - - SqlFilePath = "" - for i in range(len(path.split("/")) - 1): - SqlFilePath = SqlFilePath + path.split("/")[i] + "/" + SqlFileDate = astropytime.Time(self.run_start1, format="unix").iso.split(" ")[0] - SqlFileName = SqlFilePath + "nectarcam_monitoring_db_" + SqlFileDate + ".sqlite" + SqlFilePath = os.path.split(path)[0] + SqlFileName = ( + SqlFilePath + "/nectarcam_monitoring_db_" + SqlFileDate + ".sqlite" + ) print("SqlFileName", SqlFileName) - con = sqlite3.connect(SqlFileName) - cursor = con.cursor() + con = sqlite3.connect(SqlFileName) + cursor = con.cursor() try: # print(cursor.fetchall()) cursor.execute("SELECT name FROM sqlite_master WHERE type='table';") - TempData = cursor.execute('''SELECT * FROM monitoring_drawer_temperatures''') + TempData = cursor.execute( + """SELECT * FROM monitoring_drawer_temperatures""" + ) # print(TempData.description) self.DrawerTemp = cursor.fetchall() cursor.close() - except sqlite3.Error as err: print("Error Code: ", err) print("DRAWER TEMPERATURE COULD NOT BE RETRIEVED!") @@ -62,41 +62,48 @@ def ProcessEvent(self, evt, noped): def FinishRun(self): try: - self.event_id = np.array(self.event_id) self.event_times = np.array(self.event_times) - - self.run_start = self.event_times[self.event_id == np.min(self.event_id)] - 100 + + self.run_start = ( + self.event_times[self.event_id == np.min(self.event_id)] - 100 + ) self.run_end = np.max(self.event_times) + 100 - + self.DrawerTemp = np.array(self.DrawerTemp) self.DrawerTimes = np.array(self.DrawerTemp[:, 3]) - + for i in range(len(self.DrawerTimes)): - self.DrawerTimes[i] = astropytime.Time(self.DrawerTimes[i], format='iso').unix + self.DrawerTimes[i] = astropytime.Time( + self.DrawerTimes[i], format="iso" + ).unix self.DrawerTemp11 = self.DrawerTemp[:, 4][self.DrawerTimes > self.run_start] self.DrawerTemp21 = self.DrawerTemp[:, 5][self.DrawerTimes > self.run_start] self.DrawerNum1 = self.DrawerTemp[:, 2][self.DrawerTimes > self.run_start] - + self.DrawerTimes_new = self.DrawerTimes[self.DrawerTimes > self.run_start] - + self.DrawerTemp12 = self.DrawerTemp11[self.DrawerTimes_new < self.run_end] self.DrawerTemp22 = self.DrawerTemp21[self.DrawerTimes_new < self.run_end] - self.DrawerNum2 = self.DrawerNum1[self.DrawerTimes_new < self.run_end] - + self.DrawerNum2 = self.DrawerNum1[self.DrawerTimes_new < self.run_end] + self.DrawerTemp1_mean = [] self.DrawerTemp2_mean = [] TotalDrawers = np.max(self.DrawerNum2) - - for i in range(TotalDrawers+1): + + for i in range(TotalDrawers + 1): for j in range(7): - self.DrawerTemp1_mean.append(np.mean(self.DrawerTemp12[self.DrawerNum2 == i])) - self.DrawerTemp2_mean.append(np.mean(self.DrawerTemp22[self.DrawerNum2 == i])) + self.DrawerTemp1_mean.append( + np.mean(self.DrawerTemp12[self.DrawerNum2 == i]) + ) + self.DrawerTemp2_mean.append( + np.mean(self.DrawerTemp22[self.DrawerNum2 == i]) + ) self.DrawerTemp1_mean = np.array(self.DrawerTemp1_mean) self.DrawerTemp2_mean = np.array(self.DrawerTemp2_mean) - - self.DrawerTemp_mean = (self.DrawerTemp1_mean + self.DrawerTemp2_mean)/2 + + self.DrawerTemp_mean = (self.DrawerTemp1_mean + self.DrawerTemp2_mean) / 2 except Exception as err: print("Error Code: ", err) print("DRAWER TEMPERATURE COULD NOT BE RETRIEVED!") @@ -104,7 +111,9 @@ def FinishRun(self): def GetResults(self): self.CameraMonitoring_Results_Dict = {} try: - self.CameraMonitoring_Results_Dict["CAMERA-TEMPERATURE-AVERAGE"] = self.DrawerTemp_mean + self.CameraMonitoring_Results_Dict[ + "CAMERA-TEMPERATURE-AVERAGE" + ] = self.DrawerTemp_mean except Exception as err: print("Error Code: ", err) print("DRAWER TEMPERATURE COULD NOT BE RETRIEVED!") @@ -116,48 +125,51 @@ def PlotResults(self, name, FigPath): self.ChargeInt_Figures_Names_Dict = {} try: - fig, disp = plt.subplots() disp = CameraDisplay(self.camera) disp.image = self.DrawerTemp_mean disp.cmap = plt.cm.coolwarm - disp.axes.text(1.8, -0.3, 'Temperature', fontsize=12, rotation=90) + disp.axes.text(1.8, -0.3, "Temperature", fontsize=12, rotation=90) disp.add_colorbar() plt.title("Camera temperature average") - full_name = name + '_CameraTemperature_Mean.png' + full_name = name + "_CameraTemperature_Mean.png" FullPath = FigPath + full_name self.ChargeInt_Figures_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE"] = fig - self.ChargeInt_Figures_Names_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE"] = FullPath - + self.ChargeInt_Figures_Names_Dict[ + "CAMERA-TEMPERATURE-IMAGE-AVERAGE" + ] = FullPath + plt.close() fig1, disp = plt.subplots() disp = CameraDisplay(self.camera) disp.image = self.DrawerTemp1_mean disp.cmap = plt.cm.coolwarm - disp.axes.text(1.8, -0.3, 'Temperature 1', fontsize=12, rotation=90) + disp.axes.text(1.8, -0.3, "Temperature 1", fontsize=12, rotation=90) disp.add_colorbar() plt.title("Camera temperature average 1") - full_name = name + '_CameraTemperature_average1.png' + full_name = name + "_CameraTemperature_average1.png" FullPath = FigPath + full_name self.ChargeInt_Figures_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE-1"] = fig1 - self.ChargeInt_Figures_Names_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE-1"] = FullPath - + self.ChargeInt_Figures_Names_Dict[ + "CAMERA-TEMPERATURE-IMAGE-AVERAGE-1" + ] = FullPath + plt.close() fig2, disp = plt.subplots() disp = CameraDisplay(self.camera) disp.image = self.DrawerTemp2_mean disp.cmap = plt.cm.coolwarm - disp.axes.text(1.8, -0.3, 'Temperature 2', fontsize=12, rotation=90) + disp.axes.text(1.8, -0.3, "Temperature 2", fontsize=12, rotation=90) disp.add_colorbar() plt.title("Camera temperature average 2") - full_name = name + '_CameraTemperature_average2.png' + full_name = name + "_CameraTemperature_average2.png" FullPath = FigPath + full_name self.ChargeInt_Figures_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE-2"] = fig2 - self.ChargeInt_Figures_Names_Dict["CAMERA-TEMPERATURE-IMAGE-AVERAGE-2"] = FullPath - - plt.close() + self.ChargeInt_Figures_Names_Dict[ + "CAMERA-TEMPERATURE-IMAGE-AVERAGE-2" + ] = FullPath except Exception as err: print("Error Code: ", err) diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index 9279e54b..844791d0 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -1,18 +1,16 @@ -from dqm_summary_processor import dqm_summary -from matplotlib import pyplot as plt +import ctapipe.instrument.camera.readout import numpy as np -from ctapipe.visualization import CameraDisplay -from ctapipe.instrument import CameraGeometry from ctapipe.coordinates import EngineeringCameraFrame -from traitlets.config.loader import Config -import ctapipe.instrument.camera.readout from ctapipe.image import LocalPeakWindowSum +from ctapipe.visualization import CameraDisplay +from dqm_summary_processor import DQMSummary +from matplotlib import pyplot as plt +from traitlets.config.loader import Config -class ChargeIntegration_HighLowGain(dqm_summary): +class ChargeIntegrationHighLowGain(DQMSummary): def __init__(self, gaink): self.k = gaink - return None def ConfigureForRun(self, path, Pix, Samp, Reader1): # define number of pixels and samples @@ -22,10 +20,11 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.counter_evt = 0 self.counter_ped = 0 - self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame()) + self.camera = CameraGeometry.from_name("NectarCam-003").transform_to( + EngineeringCameraFrame() + ) self.cmap = "gnuplot2" - # reader1=EventSource(input_url=path, max_events = 1) self.subarray = Reader1.subarray subarray = Reader1.subarray subarray.tel[ @@ -33,9 +32,7 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): ].camera.readout = ctapipe.instrument.camera.readout.CameraReadout.from_name( "NectarCam" ) - config = Config( - {"LocalPeakWindowSum": {"window_shift": 4, "window_width": 12}} - ) + config = Config({"LocalPeakWindowSum": {"window_shift": 4, "window_width": 12}}) self.integrator = LocalPeakWindowSum(subarray, config=config) @@ -49,10 +46,12 @@ def ProcessEvent(self, evt, noped): self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels pixel = evt.nectarcam.tel[0].svc.pixel_ids if len(pixel) < self.Pix: - pixel_masked_shutter = list(np.arange(0, self.Pix - len(pixel), 1, dtype=int)) + pixel_masked_shutter = list( + np.arange(0, self.Pix - len(pixel), 1, dtype=int) + ) pixel = list(pixel) pixels = np.concatenate([pixel_masked_shutter, pixel]) - else: + else: pixels = pixel waveform = evt.r0.tel[0].waveform[self.k] @@ -60,14 +59,18 @@ def ProcessEvent(self, evt, noped): if noped: ped = np.mean(waveform[:, 20]) w_noped = waveform - ped - output = self.integrator(w_noped,0,np.zeros(self.Pix, dtype = int), self.pixelBAD) + output = self.integrator( + w_noped, 0, np.zeros(self.Pix, dtype=int), self.pixelBAD + ) image = output.image peakpos = output.peak_time image = image[pixels] peakpos = peakpos[pixels] else: - output = self.integrator(waveform,0,np.zeros(self.Pix, dtype = int), self.pixelBAD) + output = self.integrator( + waveform, 0, np.zeros(self.Pix, dtype=int), self.pixelBAD + ) image = output.image peakpos = output.peak_time image = image[pixels] diff --git a/src/nectarchain/dqm/dqm_summary_processor.py b/src/nectarchain/dqm/dqm_summary_processor.py index a9edaf28..efae05a0 100644 --- a/src/nectarchain/dqm/dqm_summary_processor.py +++ b/src/nectarchain/dqm/dqm_summary_processor.py @@ -2,7 +2,10 @@ from astropy.table import Table -class dqm_summary: +__all__ = ["DQMSummary"] + + +class DQMSummary: def __init__(self): print("Processor 0") @@ -18,7 +21,7 @@ def ConfigureForRun(self): print("Processor 1") def ProcessEvent(self, evt, noped): - print('Processor 2') + print("Processor 2") def FinishRun(self, M, M_ped, counter_evt, counter_ped): print("Processor 3") @@ -35,6 +38,7 @@ def WriteAllResults(self, path, DICT): data2 = Table() data1 = Table() data = Table() + hdu, hdu1, hdu2 = None, None hdulist = fits.HDUList() for i, j in DICT.items(): if i == "Results_TriggerStatistics": @@ -43,9 +47,11 @@ def WriteAllResults(self, path, DICT): hdu2 = fits.BinTableHDU(data2) hdu2.name = "Trigger" - elif (i == "Results_MeanWaveForms_HighGain") or (i == "Results_MeanWaveForms_LowGain"): + elif (i == "Results_MeanWaveForms_HighGain") or ( + i == "Results_MeanWaveForms_LowGain" + ): for n1, m1 in j.items(): - data1[n1] = m1 + data1[n1] = m1 hdu1 = fits.BinTableHDU(data1) hdu1.name = "MWF" @@ -54,19 +60,19 @@ def WriteAllResults(self, path, DICT): data[n] = m hdu = fits.BinTableHDU(data) hdu.name = "Camera" - try: + if hdu2: hdulist.append(hdu2) - except: + else: print("No trigger statistics requests") - try: - hdulist.append(hdu1) - except: + if hdu1: + hdulist.append(hdu1) + else: print("No MWF studies requests") - try: + if hdu: hdulist.append(hdu) - except: + else: print("No Camera studies requests") - FileName = path + '_Results.fits' + FileName = path + "_Results.fits" print(FileName) hdulist.writeto(FileName, overwrite=True) return None diff --git a/src/nectarchain/dqm/mean_camera_display.py b/src/nectarchain/dqm/mean_camera_display.py index 0483539a..837e2f84 100644 --- a/src/nectarchain/dqm/mean_camera_display.py +++ b/src/nectarchain/dqm/mean_camera_display.py @@ -1,12 +1,11 @@ -from dqm_summary_processor import dqm_summary -from matplotlib import pyplot as plt -from ctapipe.visualization import CameraDisplay -from ctapipe.instrument import CameraGeometry -from ctapipe.coordinates import EngineeringCameraFrame import numpy as np +from ctapipe.coordinates import EngineeringCameraFrame +from ctapipe.visualization import CameraDisplay +from dqm_summary_processor import DQMSummary +from matplotlib import pyplot as plt -class MeanCameraDisplay_HighLowGain(dqm_summary): +class MeanCameraDisplay_HighLowGain(DQMSummary): def __init__(self, gaink): self.k = gaink return None @@ -21,16 +20,15 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.counter_evt = 0 self.counter_ped = 0 - self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame()) - self.camera2 = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame()) + self.camera = Reader1.subarray.tel[0].camera.geometry.transform_to( + EngineeringCameraFrame() + ) self.cmap = "gnuplot2" - self.cmap2 = "gnuplot2" self.CameraAverage = [] self.CameraAverage_ped = [] - def ProcessEvent(self, evt, noped): self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels pixel = evt.nectarcam.tel[0].svc.pixel_ids @@ -38,36 +36,34 @@ def ProcessEvent(self, evt, noped): pixel21 = list(np.arange(0, self.Pix - len(pixel), 1, dtype=int)) pixel = list(pixel) pixels = np.concatenate([pixel21, pixel]) - else: + else: pixels = pixel if evt.trigger.event_type.value == 32: # count peds self.counter_ped += 1 - self.CameraAverage_ped1 = ( - evt.r0.tel[0].waveform[self.k].sum(axis=1)) + self.CameraAverage_ped1 = evt.r0.tel[0].waveform[self.k].sum(axis=1) self.CameraAverage_ped.append(self.CameraAverage_ped1[pixels]) else: self.counter_evt += 1 - self.CameraAverage1 = ( - evt.r0.tel[0].waveform[self.k].sum(axis=1)) + self.CameraAverage1 = evt.r0.tel[0].waveform[self.k].sum(axis=1) self.CameraAverage.append(self.CameraAverage1[pixels]) - + return None def FinishRun(self): if self.counter_evt > 0: self.CameraAverage = np.array(self.CameraAverage) - self.CameraAverage = self.CameraAverage.sum(axis = 0) - self.CameraAverage_overEvents = (self.CameraAverage / self.counter_evt) - + self.CameraAverage = self.CameraAverage.sum(axis=0) + self.CameraAverage_overEvents = self.CameraAverage / self.counter_evt + self.CameraAverage_overEvents_overSamp = ( self.CameraAverage_overEvents / self.Samp ) if self.counter_ped > 0: self.CameraAverage_ped = np.array(self.CameraAverage_ped) - self.CameraAverage_ped = self.CameraAverage_ped.sum(axis = 0) + self.CameraAverage_ped = self.CameraAverage_ped.sum(axis=0) self.CameraAverage_ped_overEvents = ( self.CameraAverage_ped / self.counter_ped ) @@ -81,7 +77,6 @@ def GetResults(self): # ASSIGN RESUTLS TO DICT if self.k == 0: - if self.counter_evt > 0: # self.MeanCameraDisplay_Results_Dict[ # "CAMERA-AVERAGE-OverEVENTS-HIGH-GAIN" @@ -132,10 +127,8 @@ def PlotResults(self, name, FigPath): self.disp1 = CameraDisplay( geometry=self.camera[~self.pixelBAD[0]], image=self.CameraAverage_overEvents_overSamp[~self.pixelBAD[0]], - cmap=self.cmap, + cmap=plt.cm.coolwarm, ) - self.disp1.cmap = self.cmap - self.disp1.cmap = plt.cm.coolwarm self.disp1.add_colorbar() self.disp1.axes.text(2.0, 0, "Charge (DC)", rotation=90) plt.title("Camera average %s gain (ALL)" % gain_c) @@ -153,12 +146,10 @@ def PlotResults(self, name, FigPath): if self.counter_ped > 0: fig2, self.disp2 = plt.subplots() self.disp2 = CameraDisplay( - geometry=self.camera2[~self.pixelBAD[0]], + geometry=self.camera[~self.pixelBAD[0]], image=self.CameraAverage_ped_overEvents_overSamp[~self.pixelBAD[0]], - cmap=self.cmap2, + cmap=plt.cm.coolwarm, ) - self.disp2.cmap = self.cmap2 - self.disp2.cmap = plt.cm.coolwarm self.disp2.add_colorbar() self.disp2.axes.text(2.0, 0, "Charge (DC)", rotation=90) plt.title("Camera average %s gain (PED)" % gain_c) diff --git a/src/nectarchain/dqm/mean_waveforms.py b/src/nectarchain/dqm/mean_waveforms.py index 2b134839..695d8943 100644 --- a/src/nectarchain/dqm/mean_waveforms.py +++ b/src/nectarchain/dqm/mean_waveforms.py @@ -1,9 +1,9 @@ -from dqm_summary_processor import dqm_summary -from matplotlib import pyplot as plt import numpy as np +from dqm_summary_processor import DQMSummary +from matplotlib import pyplot as plt -class MeanWaveForms_HighLowGain(dqm_summary): +class MeanWaveFormsHighLowGain(DQMSummary): def __init__(self, gaink): self.k = gaink return None @@ -69,18 +69,22 @@ def GetResults(self): # ASSIGN RESUTLS TO DICT if self.k == 0: - # self.MeanWaveForms_Results_Dict["WF-PHY-AVERAGE-HIGH-GAIN"] = self.Mwf_average - self.MeanWaveForms_Results_Dict["WF-PHY-AVERAGE-PIX-HIGH-GAIN"] = self.Mwf_Mean_overPix + self.MeanWaveForms_Results_Dict[ + "WF-PHY-AVERAGE-PIX-HIGH-GAIN" + ] = self.Mwf_Mean_overPix if self.counter_ped > 0: - # self.MeanWaveForms_Results_Dict["WF-PED-AVERAGE-HIGH-GAIN"] = self.Mwf_ped_average - self.MeanWaveForms_Results_Dict["WF-AVERAGE-PED-PIX-HIGH-GAIN"] = self.Mwf_ped_Mean_overPix + self.MeanWaveForms_Results_Dict[ + "WF-AVERAGE-PED-PIX-HIGH-GAIN" + ] = self.Mwf_ped_Mean_overPix if self.k == 1: - # self.MeanWaveForms_Results_Dict["WF-AVERAGE-LOW-GAIN"] = self.Mwf_average - self.MeanWaveForms_Results_Dict["WF-AVERAGE-PIX-LOW-GAIN"] = self.Mwf_Mean_overPix + self.MeanWaveForms_Results_Dict[ + "WF-AVERAGE-PIX-LOW-GAIN" + ] = self.Mwf_Mean_overPix if self.counter_ped > 0: - # self.MeanWaveForms_Results_Dict["WF-PHY-PED-AVERAGE-LOW-GAIN"] = self.Mwf_ped_average - self.MeanWaveForms_Results_Dict["WF-PHY-AVERAGE-PED-PIX-LOW-GAIN"] = self.Mwf_ped_Mean_overPix + self.MeanWaveForms_Results_Dict[ + "WF-PHY-AVERAGE-PED-PIX-LOW-GAIN" + ] = self.Mwf_ped_Mean_overPix return self.MeanWaveForms_Results_Dict diff --git a/src/nectarchain/dqm/start_calib.py b/src/nectarchain/dqm/start_calib.py index a9b08d11..900561f6 100644 --- a/src/nectarchain/dqm/start_calib.py +++ b/src/nectarchain/dqm/start_calib.py @@ -1,3 +1,4 @@ +import argparse import os import sys @@ -5,42 +6,42 @@ import argparse import time -from ctapipe.io import EventSource, EventSeeker -from ctapipe_io_nectarcam.constants import LOW_GAIN, HIGH_GAIN - -from mean_waveforms import MeanWaveForms_HighLowGain -from mean_camera_display import MeanCameraDisplay_HighLowGain -from charge_integration import ChargeIntegration_HighLowGain -from trigger_statistics import TriggerStatistics from camera_monitoring import CameraMonitoring - +from charge_integration import ChargeIntegrationHighLowGain +from ctapipe.io import EventSeeker, EventSource +from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN from db_utils import DQMDB +from matplotlib import pyplot as plt +from mean_camera_display import MeanCameraDisplay_HighLowGain +from mean_waveforms import MeanWaveFormsHighLowGain +from trigger_statistics import TriggerStatistics # Create an ArgumentParser object -parser = argparse.ArgumentParser(description='NectarCAM Data Quality Monitoring tool') -parser.add_argument('-p', '--plot', - action='store_true', - help='Enables plots to be generated') -parser.add_argument('--write-db', - action='store_true', - help='Write DQM output in DQM ZODB data base') -parser.add_argument('-n', '--noped', - action='store_true', - help='Enables pedestal subtraction in charge integration') -parser.add_argument('-r', '--runnb', - help='Optional run number, automatically found on DIRAC', - type=int) -parser.add_argument('-i', '--input-files', - nargs='+', - help='Local input files') - -parser.add_argument('input_paths', help='Input paths') -parser.add_argument('output_paths', help='Output paths') +parser = argparse.ArgumentParser(description="NectarCAM Data Quality Monitoring tool") +parser.add_argument( + "-p", "--plot", action="store_true", help="Enables plots to be generated" +) +parser.add_argument( + "--write-db", action="store_true", help="Write DQM output in DQM ZODB data base" +) +parser.add_argument( + "-n", + "--noped", + action="store_true", + help="Enables pedestal subtraction in charge integration", +) +parser.add_argument( + "-r", "--runnb", help="Optional run number, automatically found on DIRAC", type=int +) +parser.add_argument("-i", "--input-files", nargs="+", help="Local input files") + +parser.add_argument("input_paths", help="Input paths") +parser.add_argument("output_paths", help="Output paths") args, leftovers = parser.parse_known_args() # Reading arguments, paths and plot-boolean -NectarPath = args.input_paths # str(os.environ['NECTARDIR']) +NectarPath = args.input_paths print("Input file path:", NectarPath) # Defining and printing the paths of the output files. @@ -53,18 +54,19 @@ if args.runnb is not None: # Grab runs automatically from DIRAC is the -r option is provided from nectarchain.data.container import utils + dm = utils.DataManagement() _, filelist = dm.findrun(args.runnb) args.input_files = [s.name for s in filelist] elif args.input_files is None: - print('Input files should be provided, exiting...') + print("Input files should be provided, exiting...") sys.exit(1) # OTHERWISE READ THE RUNS FROM ARGS path1 = args.input_files[0] # THE PATH OF INPUT FILES -path = f'{NectarPath}/{path1}' +path = f"{NectarPath}/{path1}" print("Input files:") print(path) for arg in args.input_files[1:]: @@ -89,9 +91,9 @@ def CreateFigFolder(name, type): if type == 0: folder = "Plots" - ParentFolderName = name.split('_')[0] + '_' + name.split('_')[1] - ChildrenFolderName = './' + ParentFolderName + '/' + name + '_calib' - FolderPath = f'{output_path}/output/{ChildrenFolderName}/{folder}' + ParentFolderName = name.split("_")[0] + "_" + name.split("_")[1] + ChildrenFolderName = "./" + ParentFolderName + "/" + name + "_calib" + FolderPath = f"{output_path}/output/{ChildrenFolderName}/{folder}" if not os.path.exists(FolderPath): os.makedirs(FolderPath) @@ -104,7 +106,7 @@ def CreateFigFolder(name, type): # INITIATE path = path print(path) -cmap = 'gnuplot2' +cmap = "gnuplot2" # Read and seek reader = EventSource(input_url=path) @@ -114,7 +116,7 @@ def CreateFigFolder(name, type): name = GetName(path) ParentFolderName, ChildrenFolderName, FigPath = CreateFigFolder(name, 0) -ResPath = f'{output_path}/output/{ChildrenFolderName}/{name}' +ResPath = f"{output_path}/output/{ChildrenFolderName}/{name}" # LIST OF PROCESSES TO RUN @@ -124,8 +126,8 @@ def CreateFigFolder(name, type): c = MeanWaveForms_HighLowGain(LOW_GAIN) d = MeanCameraDisplay_HighLowGain(HIGH_GAIN) e = MeanCameraDisplay_HighLowGain(LOW_GAIN) -f = ChargeIntegration_HighLowGain(HIGH_GAIN) -g = ChargeIntegration_HighLowGain(LOW_GAIN) +f = ChargeIntegrationHighLowGain(HIGH_GAIN) +g = ChargeIntegrationHighLowGain(LOW_GAIN) h = CameraMonitoring(HIGH_GAIN) processors = list() @@ -174,10 +176,10 @@ def CreateFigFolder(name, type): for i, evt in enumerate(reader): for p in processors: p.ProcessEvent(evt, noped) - + # for the rest of the event files for arg in args.input_files[1:]: - path2 = f'{NectarPath}/{arg}' + path2 = f"{NectarPath}/{arg}" print(path2) reader = EventSource(input_url=path2) @@ -195,9 +197,8 @@ def CreateFigFolder(name, type): NESTED_DICT[NESTED_DICT_KEYS[dict_num]] = p.GetResults() dict_num += 1 - -name = name #in order to allow to change the name easily -p.WriteAllResults(ResPath, NESTED_DICT) #if we want to write all results in 1 fits file we do this. +# Write all results in 1 fits file: +p.WriteAllResults(ResPath, NESTED_DICT) if args.write_db: db = DQMDB(read_only=False) if db.insert(name, NESTED_DICT): @@ -205,25 +206,20 @@ def CreateFigFolder(name, type): else: db.abort_and_close() -# if -plot in args it will construct the figures and save them +# if plot option in arguments, it will construct the figures and save them if PlotFig: for p in processors: processor_figure_dict, processor_figure_name_dict = p.PlotResults(name, FigPath) - + for fig_plot in processor_figure_dict: fig = processor_figure_dict[fig_plot] SavePath = processor_figure_name_dict[fig_plot] - plt.gcf() fig.savefig(SavePath) - - plt.clf() - plt.cla() - plt.close() + plt.close() end = time.time() print("Processing time:", end - start) -# TODOS +# TODO # Reduce code by using loops: for figs and results -# MONGO: store results diff --git a/src/nectarchain/dqm/trigger_statistics.py b/src/nectarchain/dqm/trigger_statistics.py index 29d344a8..8effd922 100644 --- a/src/nectarchain/dqm/trigger_statistics.py +++ b/src/nectarchain/dqm/trigger_statistics.py @@ -1,11 +1,12 @@ import math -from matplotlib import pyplot as plt + import numpy as np from astropy import time as astropytime -from dqm_summary_processor import dqm_summary +from dqm_summary_processor import DQMSummary +from matplotlib import pyplot as plt -class TriggerStatistics(dqm_summary): +class TriggerStatistics(DQMSummary): def __init__(self, gaink): self.k = gaink