From e9f72e1f3589a35d72c77e233f59f7676c58a17a Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Fri, 27 Oct 2023 15:15:31 +0200 Subject: [PATCH 1/8] rebase:a new class: pixel_participation --- src/nectarchain/dqm/pixel_participation.py | 152 +++++++++++++++++++++ src/nectarchain/dqm/start_dqm.py | 13 ++ 2 files changed, 165 insertions(+) create mode 100644 src/nectarchain/dqm/pixel_participation.py diff --git a/src/nectarchain/dqm/pixel_participation.py b/src/nectarchain/dqm/pixel_participation.py new file mode 100644 index 00000000..86c94b70 --- /dev/null +++ b/src/nectarchain/dqm/pixel_participation.py @@ -0,0 +1,152 @@ +from dqm_summary_processor import dqm_summary +from matplotlib import pyplot as plt +from ctapipe.visualization import CameraDisplay +from ctapipe.instrument import CameraGeometry +from ctapipe.coordinates import EngineeringCameraFrame +import numpy as np + + +class PixelParticipation_HighLowGain(dqm_summary): + def __init__(self, gaink): + self.k = gaink + return None + + def ConfigureForRun(self, path, Pix, Samp, Reader1): + # define number of pixels and samples + self.Pix = Pix + self.Samp = Samp + + self.CameraAverage = np.zeros(self.Pix) + self.CameraAverage_ped = np.zeros(self.Pix) + self.counter_evt = 0 + self.counter_ped = 0 + + self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame()) + self.camera2 = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame()) + + self.cmap = "gnuplot2" + self.cmap2 = "gnuplot2" + + self.CameraAverage = [] + self.CameraAverage_ped = [] + + self.BadPixels_ped = np.zeros(self.Pix) + self.BadPixels = np.zeros(self.Pix) + + def ProcessEvent(self, evt, noped): + pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels[self.k] + pixel = evt.nectarcam.tel[0].svc.pixel_ids + if len(pixel) < self.Pix: + pixel21 = list(np.arange(0, self.Pix - len(pixel), 1, dtype=int)) + pixel = list(pixel) + pixels = np.concatenate([pixel21, pixel]) + else: + pixels = pixel + + if evt.trigger.event_type.value == 32: # count peds + self.counter_ped += 1 + BadPixels_ped1 = list(map(int, pixelBAD)) + self.BadPixels_ped += BadPixels_ped1 + + else: + self.counter_evt += 1 + BadPixels1 = list(map(int, pixelBAD)) + self.BadPixels += BadPixels1 + return None + + def FinishRun(self): + self.BadPixels_ped = np.array(self.BadPixels_ped) + self.BadPixels = np.array(self.BadPixels) + + + + def GetResults(self): + # INITIATE DICT + self.PixelParticipation_Results_Dict = {} + + # ASSIGN RESUTLS TO DICT + if self.k == 0: + + if self.counter_evt > 0: + self.PixelParticipation_Results_Dict[ + "CAMERA-BadPix-PHY-OverEVENTS-HIGH-GAIN" + ] = self.BadPixels + + if self.counter_ped > 0: + self.PixelParticipation_Results_Dict[ + "CAMERA-BadPix-PED-PHY-OverEVENTS-HIGH-GAIN" + ] = self.BadPixels_ped + + if self.k == 1: + if self.counter_evt > 0: + self.PixelParticipation_Results_Dict[ + "CAMERA-BadPix-PHY-OverEVENTS-LOW-GAIN" + ] = self.BadPixels + + if self.counter_ped > 0: + self.PixelParticipation_Results_Dict[ + "CAMERA-BadPix-PED-PHY-OverEVENTS-LOW-GAIN" + ] = self.BadPixels_ped + + return self.PixelParticipation_Results_Dict + + def PlotResults(self, name, FigPath): + self.PixelParticipation_Figures_Dict = {} + self.PixelParticipation_Figures_Names_Dict = {} + + # titles = ['All', 'Pedestals'] + if self.k == 0: + gain_c = "High" + if self.k == 1: + gain_c = "Low" + + if self.counter_evt > 0: + fig1, self.disp1 = plt.subplots() + self.disp1 = CameraDisplay( + geometry=self.camera, + image=self.BadPixels, + cmap=self.cmap, + ) + self.disp1.cmap = self.cmap + self.disp1.cmap = plt.cm.coolwarm + self.disp1.add_colorbar() + self.disp1.axes.text(2.0, 0, "Bad Pixels", rotation=90) + plt.title("Camera BPX %s gain (ALL)" % gain_c) + + self.PixelParticipation_Figures_Dict[ + "CAMERA-BADPIX-PHY-DISPLAY-%s-GAIN" % gain_c + ] = fig1 + full_name = name + "_Camera_BPX_%sGain.png" % gain_c + FullPath = FigPath + full_name + self.PixelParticipation_Figures_Names_Dict[ + "CAMERA-BADPIX-PHY-DISPLAY-%s-GAIN" % gain_c + ] = FullPath + plt.close() + + if self.counter_ped > 0: + fig2, self.disp2 = plt.subplots() + self.disp2 = CameraDisplay( + geometry=self.camera2, + image=self.BadPixels_ped, + cmap=self.cmap2, + ) + self.disp2.cmap = self.cmap2 + self.disp2.cmap = plt.cm.coolwarm + self.disp2.add_colorbar() + self.disp2.axes.text(2.0, 0, "Bad Pixels", rotation=90) + plt.title("Camera BPX %s gain (PED)" % gain_c) + + self.PixelParticipation_Figures_Dict[ + "CAMERA-BADPIX-PED-DISPLAY-%s-GAIN" % gain_c + ] = fig2 + full_name = name + "_Pedestal_BPX_%sGain.png" % gain_c + FullPath = FigPath + full_name + self.PixelParticipation_Figures_Names_Dict[ + "CAMERA-BADPIX-PED-DISPLAY-%s-GAIN" % gain_c + ] = FullPath + plt.close() + + return ( + self.PixelParticipation_Figures_Dict, + self.PixelParticipation_Figures_Names_Dict, + ) diff --git a/src/nectarchain/dqm/start_dqm.py b/src/nectarchain/dqm/start_dqm.py index 8afe6e6f..21e1b113 100644 --- a/src/nectarchain/dqm/start_dqm.py +++ b/src/nectarchain/dqm/start_dqm.py @@ -4,9 +4,14 @@ import time from camera_monitoring import CameraMonitoring +<<<<<<< HEAD:src/nectarchain/dqm/start_dqm.py from charge_integration import ChargeIntegrationHighLowGain from ctapipe.io import EventSeeker, EventSource from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN +======= +from pixel_participation import PixelParticipation_HighLowGain + +>>>>>>> b92cee1 (Adding a new class: pixel_participation):src/nectarchain/dqm/start_calib.py from db_utils import DQMDB from matplotlib import pyplot as plt from mean_camera_display import MeanCameraDisplay_HighLowGain @@ -126,6 +131,8 @@ def CreateFigFolder(name, type): f = ChargeIntegrationHighLowGain(HIGH_GAIN) g = ChargeIntegrationHighLowGain(LOW_GAIN) h = CameraMonitoring(HIGH_GAIN) +e = PixelParticipation_HighLowGain(HIGH_GAIN) +f = PixelParticipation_HighLowGain(LOW_GAIN) processors = list() @@ -137,6 +144,8 @@ def CreateFigFolder(name, type): processors.append(f) processors.append(g) processors.append(h) +processors.append(e) +processors.append(f) # LIST OF DICT RESULTS @@ -148,6 +157,8 @@ def CreateFigFolder(name, type): Results_ChargeIntegration_LowGain = {} Results_TriggerStatistics = {} Results_CameraMonitoring = {} +Results_PixelParticipation_HighGain = {} +Results_PixelParticipation_LowGain = {} NESTED_DICT = {} # The final results dictionary NESTED_DICT_KEYS = [ @@ -159,6 +170,8 @@ def CreateFigFolder(name, type): "Results_ChargeIntegration_HighGain", "Results_ChargeIntegration_LowGain", "Results_CameraMonitoring", + "Results_PixelParticipation_HighGain", + "Results_PixelParticipation_LowGain", ] # NESTED_DICT_KEYS = ["Results_CameraMonitoring"] From 1951f02d23d63b8e39805af465698b613dd84f37 Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Mon, 30 Oct 2023 18:13:26 +0100 Subject: [PATCH 2/8] REBASE: Adding BPX timeline module --- src/nectarchain/dqm/dqm_summary_processor.py | 25 ++- src/nectarchain/dqm/pixel_participation.py | 8 +- src/nectarchain/dqm/pixel_timeline.py | 155 +++++++++++++++++++ src/nectarchain/dqm/start_dqm.py | 23 ++- 4 files changed, 197 insertions(+), 14 deletions(-) create mode 100644 src/nectarchain/dqm/pixel_timeline.py diff --git a/src/nectarchain/dqm/dqm_summary_processor.py b/src/nectarchain/dqm/dqm_summary_processor.py index e9a04437..87cffe66 100644 --- a/src/nectarchain/dqm/dqm_summary_processor.py +++ b/src/nectarchain/dqm/dqm_summary_processor.py @@ -36,11 +36,12 @@ def PlotResults( def WriteAllResults(self, path, DICT): data2 = Table() data1 = Table() + data0 = Table() data = Table() hdu, hdu1, hdu2 = None, None, None hdulist = fits.HDUList() for i, j in DICT.items(): - if i == "Results_TriggerStatistics": + if (i == "Results_TriggerStatistics"): for n2, m2 in j.items(): data2[n2] = m2 hdu2 = fits.BinTableHDU(data2) @@ -52,7 +53,13 @@ def WriteAllResults(self, path, DICT): for n1, m1 in j.items(): data1[n1] = m1 hdu1 = fits.BinTableHDU(data1) - hdu1.name = "MWF" + hdu1.name = "MWF" + + elif (i == "Results_PixelTimeline_HighGain") or (i == "Results_PixelTimeline_LowGain"): + for n0, m0 in j.items(): + data0[n0] = m0 + hdu0 = fits.BinTableHDU(data0) + hdu0.name = "BPX" else: for n, m in j.items(): @@ -67,11 +74,25 @@ def WriteAllResults(self, path, DICT): hdulist.append(hdu1) else: print("No MWF studies requests") +<<<<<<< HEAD if hdu: +======= + try: + hdulist.append(hdu0) + except: + print("No Pixel Timeline studies requests") + try: +>>>>>>> cc29b1f (Adding BPX timeline module) hdulist.append(hdu) else: print("No Camera studies requests") +<<<<<<< HEAD FileName = path + "_Results.fits" +======= + + + FileName = path + '_Results.fits' +>>>>>>> cc29b1f (Adding BPX timeline module) print(FileName) hdulist.writeto(FileName, overwrite=True) return None diff --git a/src/nectarchain/dqm/pixel_participation.py b/src/nectarchain/dqm/pixel_participation.py index 86c94b70..c13bd8fd 100644 --- a/src/nectarchain/dqm/pixel_participation.py +++ b/src/nectarchain/dqm/pixel_participation.py @@ -16,8 +16,6 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.Pix = Pix self.Samp = Samp - self.CameraAverage = np.zeros(self.Pix) - self.CameraAverage_ped = np.zeros(self.Pix) self.counter_evt = 0 self.counter_ped = 0 @@ -27,8 +25,6 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.cmap = "gnuplot2" self.cmap2 = "gnuplot2" - self.CameraAverage = [] - self.CameraAverage_ped = [] self.BadPixels_ped = np.zeros(self.Pix) self.BadPixels = np.zeros(self.Pix) @@ -45,12 +41,12 @@ def ProcessEvent(self, evt, noped): if evt.trigger.event_type.value == 32: # count peds self.counter_ped += 1 - BadPixels_ped1 = list(map(int, pixelBAD)) + BadPixels_ped1 = list(map(int, pixelBAD[pixels])) self.BadPixels_ped += BadPixels_ped1 else: self.counter_evt += 1 - BadPixels1 = list(map(int, pixelBAD)) + BadPixels1 = list(map(int, pixelBAD[pixels])) self.BadPixels += BadPixels1 return None diff --git a/src/nectarchain/dqm/pixel_timeline.py b/src/nectarchain/dqm/pixel_timeline.py new file mode 100644 index 00000000..7df9173a --- /dev/null +++ b/src/nectarchain/dqm/pixel_timeline.py @@ -0,0 +1,155 @@ +from dqm_summary_processor import dqm_summary +from matplotlib import pyplot as plt +from ctapipe.visualization import CameraDisplay +from ctapipe.instrument import CameraGeometry +from ctapipe.coordinates import EngineeringCameraFrame +import numpy as np + + +class PixelTimeline_HighLowGain(dqm_summary): + def __init__(self, gaink): + self.k = gaink + return None + + def ConfigureForRun(self, path, Pix, Samp, Reader1): + # define number of pixels and samples + self.Pix = Pix + self.Samp = Samp + + + self.counter_evt = 0 + self.counter_ped = 0 + + self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame()) + self.camera2 = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame()) + + self.cmap = "gnuplot2" + self.cmap2 = "gnuplot2" + + + self.SumBadPixels_ped = [] + self.SumBadPixels = [] + + def ProcessEvent(self, evt, noped): + pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels[self.k] + pixel = evt.nectarcam.tel[0].svc.pixel_ids + if len(pixel) < self.Pix: + pixel21 = list(np.arange(0, self.Pix - len(pixel), 1, dtype=int)) + pixel = list(pixel) + pixels = np.concatenate([pixel21, pixel]) + else: + pixels = pixel + + if evt.trigger.event_type.value == 32: # count peds + self.counter_ped += 1 + self.counter_evt += 1 + BadPixels_ped1 = list(map(int, pixelBAD[pixels])) + SumBadPixelsEvent_ped = sum(BadPixels_ped1) + self.SumBadPixels_ped.append(SumBadPixelsEvent_ped) + self.SumBadPixels.append(0) + + else: + self.counter_evt += 1 + self.counter_ped += 1 + BadPixels1 = list(map(int, pixelBAD[pixels])) + SumBadPixelsEvent = sum(BadPixels1) + self.SumBadPixels.append(SumBadPixelsEvent) + self.SumBadPixels_ped.append(0) + + return None + + def FinishRun(self): + + self.BadPixelTimeline_ped = np.array(self.SumBadPixels_ped, dtype=float)/self.Pix + self.BadPixelTimeline = np.array(self.SumBadPixels, dtype=float)/self.Pix + print(self.BadPixelTimeline) + print( self.BadPixelTimeline_ped) + + + + + def GetResults(self): + # INITIATE DICT + self.PixelTimeline_Results_Dict = {} + + # ASSIGN RESUTLS TO DICT + if self.k == 0: + + if self.counter_evt > 0: + self.PixelTimeline_Results_Dict[ + "CAMERA-BadPixTimeline-PHY-HIGH-GAIN" + ] = self.BadPixelTimeline + + + if self.counter_ped > 0: + self.PixelTimeline_Results_Dict[ + "CAMERA-BadPixTimeline-PED-HIGH-GAIN" + ] = self.BadPixelTimeline_ped + + + if self.k == 1: + if self.counter_evt > 0: + self.PixelTimeline_Results_Dict[ + "CAMERA-BadPixTimeline-PHY-LOW-GAIN" + ] = self.BadPixelTimeline + + if self.counter_ped > 0: + self.PixelTimeline_Results_Dict[ + "CAMERA-BadPixTimeline-PED-LOW-GAIN" + ] = self.BadPixelTimeline_ped + + + return self.PixelTimeline_Results_Dict + + def PlotResults(self, name, FigPath): + self.PixelTimeline_Figures_Dict = {} + self.PixelTimeline_Figures_Names_Dict = {} + + # titles = ['All', 'Pedestals'] + if self.k == 0: + gain_c = "High" + if self.k == 1: + gain_c = "Low" + + if self.counter_evt > 0: + fig1, disp = plt.subplots() + plt.plot(np.arange(self.counter_evt), self.BadPixelTimeline*100, label = "Physical events") + plt.legend() + plt.xlabel("Timeline") + plt.ylabel("BPX fraction (%)") + plt.title("BPX Timeline %s gain (ALL)" % gain_c) + + full_name = name + "_BPX_Timeline_%sGain_All.png" % gain_c + FullPath = FigPath + full_name + self.PixelTimeline_Figures_Dict[ + "BPX-TIMELINE-ALL-%s-GAIN" % gain_c + ] = fig1 + self.PixelTimeline_Figures_Names_Dict[ + "BPX-TIMELINE-ALL-%s-GAIN" % gain_c + ] = FullPath + + plt.close() + + if self.counter_ped > 0: + fig2, disp = plt.subplots() + plt.plot(np.arange(self.counter_ped), self.BadPixelTimeline_ped*100, label = "Pedestal events") + plt.legend() + plt.xlabel("Timeline") + plt.ylabel("BPX fraction (%)") + plt.title("BPX Timeline %s gain (PED)" % gain_c) + + full_name = name + "_BPX_Timeline_%sGain_Ped.png" % gain_c + FullPath = FigPath + full_name + self.PixelTimeline_Figures_Dict[ + "BPX-TIMELINE-PED-%s-GAIN" % gain_c + ] = fig2 + self.PixelTimeline_Figures_Names_Dict[ + "BPX-TIMELINE-PED-%s-GAIN" % gain_c + ] = FullPath + + plt.close() + + return ( + self.PixelTimeline_Figures_Dict, + self.PixelTimeline_Figures_Names_Dict, + ) diff --git a/src/nectarchain/dqm/start_dqm.py b/src/nectarchain/dqm/start_dqm.py index 21e1b113..0a900cba 100644 --- a/src/nectarchain/dqm/start_dqm.py +++ b/src/nectarchain/dqm/start_dqm.py @@ -10,6 +10,7 @@ from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN ======= from pixel_participation import PixelParticipation_HighLowGain +from pixel_timeline import PixelTimeline_HighLowGain >>>>>>> b92cee1 (Adding a new class: pixel_participation):src/nectarchain/dqm/start_calib.py from db_utils import DQMDB @@ -131,8 +132,10 @@ def CreateFigFolder(name, type): f = ChargeIntegrationHighLowGain(HIGH_GAIN) g = ChargeIntegrationHighLowGain(LOW_GAIN) h = CameraMonitoring(HIGH_GAIN) -e = PixelParticipation_HighLowGain(HIGH_GAIN) -f = PixelParticipation_HighLowGain(LOW_GAIN) +i = PixelParticipation_HighLowGain(HIGH_GAIN) +j = PixelParticipation_HighLowGain(LOW_GAIN) +k = PixelTimeline_HighLowGain(HIGH_GAIN) +l = PixelTimeline_HighLowGain(LOW_GAIN) processors = list() @@ -144,23 +147,28 @@ def CreateFigFolder(name, type): processors.append(f) processors.append(g) processors.append(h) -processors.append(e) -processors.append(f) +processors.append(i) +processors.append(j) +processors.append(k) +processors.append(l) # LIST OF DICT RESULTS +Results_TriggerStatistics = {} Results_MeanWaveForms_HighGain = {} Results_MeanWaveForms_LowGain = {} Results_MeanCameraDisplay_HighGain = {} Results_MeanCameraDisplay_LowGain = {} Results_ChargeIntegration_HighGain = {} Results_ChargeIntegration_LowGain = {} -Results_TriggerStatistics = {} Results_CameraMonitoring = {} Results_PixelParticipation_HighGain = {} Results_PixelParticipation_LowGain = {} +Results_PixelTimeline_HighGain = {} +Results_PixelTimeline_LowGain = {} NESTED_DICT = {} # The final results dictionary + NESTED_DICT_KEYS = [ "Results_TriggerStatistics", "Results_MeanWaveForms_HighGain", @@ -172,8 +180,11 @@ def CreateFigFolder(name, type): "Results_CameraMonitoring", "Results_PixelParticipation_HighGain", "Results_PixelParticipation_LowGain", + "Results_PixelTimeline_HighGain", + "Results_PixelTimeline_LowGain", ] -# NESTED_DICT_KEYS = ["Results_CameraMonitoring"] + +#NESTED_DICT_KEYS = ["Results_PixelParticipation_HighGain", "Results_PixelTimeline_HighGain"] # START for p in processors: From 06550970b73145074f0c00f0147dcbf4aebb87ac Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Wed, 6 Dec 2023 11:57:26 +0100 Subject: [PATCH 3/8] Resolving conflits for merge --- src/nectarchain/dqm/dqm_summary_processor.py | 16 ++++------------ src/nectarchain/dqm/start_dqm.py | 14 ++++++-------- 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/src/nectarchain/dqm/dqm_summary_processor.py b/src/nectarchain/dqm/dqm_summary_processor.py index 87cffe66..d14a6756 100644 --- a/src/nectarchain/dqm/dqm_summary_processor.py +++ b/src/nectarchain/dqm/dqm_summary_processor.py @@ -38,7 +38,7 @@ def WriteAllResults(self, path, DICT): data1 = Table() data0 = Table() data = Table() - hdu, hdu1, hdu2 = None, None, None + hdu, hdu0, hdu1, hdu2 = None, None, None, None hdulist = fits.HDUList() for i, j in DICT.items(): if (i == "Results_TriggerStatistics"): @@ -74,25 +74,17 @@ def WriteAllResults(self, path, DICT): hdulist.append(hdu1) else: print("No MWF studies requests") -<<<<<<< HEAD - if hdu: -======= - try: + if: hdulist.append(hdu0) - except: + else: print("No Pixel Timeline studies requests") - try: ->>>>>>> cc29b1f (Adding BPX timeline module) + if: hdulist.append(hdu) else: print("No Camera studies requests") -<<<<<<< HEAD - FileName = path + "_Results.fits" -======= FileName = path + '_Results.fits' ->>>>>>> cc29b1f (Adding BPX timeline module) print(FileName) hdulist.writeto(FileName, overwrite=True) return None diff --git a/src/nectarchain/dqm/start_dqm.py b/src/nectarchain/dqm/start_dqm.py index 0a900cba..0d55677b 100644 --- a/src/nectarchain/dqm/start_dqm.py +++ b/src/nectarchain/dqm/start_dqm.py @@ -4,21 +4,19 @@ import time from camera_monitoring import CameraMonitoring -<<<<<<< HEAD:src/nectarchain/dqm/start_dqm.py from charge_integration import ChargeIntegrationHighLowGain -from ctapipe.io import EventSeeker, EventSource -from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN -======= from pixel_participation import PixelParticipation_HighLowGain from pixel_timeline import PixelTimeline_HighLowGain - ->>>>>>> b92cee1 (Adding a new class: pixel_participation):src/nectarchain/dqm/start_calib.py -from db_utils import DQMDB -from matplotlib import pyplot as plt from mean_camera_display import MeanCameraDisplay_HighLowGain from mean_waveforms import MeanWaveFormsHighLowGain from trigger_statistics import TriggerStatistics +from ctapipe.io import EventSeeker, EventSource +from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN +from db_utils import DQMDB +from matplotlib import pyplot as plt + + # Create an ArgumentParser object parser = argparse.ArgumentParser(description="NectarCAM Data Quality Monitoring tool") parser.add_argument( From 83f813345e95aa607e5f54f46c93e56b5236af8f Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Wed, 6 Dec 2023 11:58:51 +0100 Subject: [PATCH 4/8] olving further conflict bugs --- src/nectarchain/dqm/dqm_summary_processor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nectarchain/dqm/dqm_summary_processor.py b/src/nectarchain/dqm/dqm_summary_processor.py index d14a6756..8486487f 100644 --- a/src/nectarchain/dqm/dqm_summary_processor.py +++ b/src/nectarchain/dqm/dqm_summary_processor.py @@ -74,11 +74,11 @@ def WriteAllResults(self, path, DICT): hdulist.append(hdu1) else: print("No MWF studies requests") - if: + if hdu0: hdulist.append(hdu0) else: print("No Pixel Timeline studies requests") - if: + if hdu: hdulist.append(hdu) else: print("No Camera studies requests") From 04435d1b29e6534a5fb398303c3815e0e3ffc837 Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Wed, 6 Dec 2023 12:13:32 +0100 Subject: [PATCH 5/8] debugging the merger request Please enter the commit message for your changes. Lines starting --- src/nectarchain/dqm/camera_monitoring.py | 2 +- src/nectarchain/dqm/charge_integration.py | 3 ++- src/nectarchain/dqm/pixel_participation.py | 4 ++-- src/nectarchain/dqm/pixel_timeline.py | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/nectarchain/dqm/camera_monitoring.py b/src/nectarchain/dqm/camera_monitoring.py index 3a698730..c2a479a3 100644 --- a/src/nectarchain/dqm/camera_monitoring.py +++ b/src/nectarchain/dqm/camera_monitoring.py @@ -18,7 +18,7 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.Pix = Pix self.Samp = Samp - self.camera = CameraGeometry.from_name("NectarCam-003").transform_to( + self.camera = Reader1.subarray.tel[0].camera.geometry.transform_to( EngineeringCameraFrame() ) self.cmap = "gnuplot2" diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index 844791d0..66032fe2 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -20,9 +20,10 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.counter_evt = 0 self.counter_ped = 0 - self.camera = CameraGeometry.from_name("NectarCam-003").transform_to( + self.camera = Reader1.subarray.tel[0].camera.geometry.transform_to( EngineeringCameraFrame() ) + self.cmap = "gnuplot2" self.subarray = Reader1.subarray diff --git a/src/nectarchain/dqm/pixel_participation.py b/src/nectarchain/dqm/pixel_participation.py index c13bd8fd..fc8841a1 100644 --- a/src/nectarchain/dqm/pixel_participation.py +++ b/src/nectarchain/dqm/pixel_participation.py @@ -1,4 +1,4 @@ -from dqm_summary_processor import dqm_summary +from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt from ctapipe.visualization import CameraDisplay from ctapipe.instrument import CameraGeometry @@ -6,7 +6,7 @@ import numpy as np -class PixelParticipation_HighLowGain(dqm_summary): +class PixelParticipation_HighLowGain(DQMSummary): def __init__(self, gaink): self.k = gaink return None diff --git a/src/nectarchain/dqm/pixel_timeline.py b/src/nectarchain/dqm/pixel_timeline.py index 7df9173a..659ebfd8 100644 --- a/src/nectarchain/dqm/pixel_timeline.py +++ b/src/nectarchain/dqm/pixel_timeline.py @@ -1,4 +1,4 @@ -from dqm_summary_processor import dqm_summary +from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt from ctapipe.visualization import CameraDisplay from ctapipe.instrument import CameraGeometry @@ -6,7 +6,7 @@ import numpy as np -class PixelTimeline_HighLowGain(dqm_summary): +class PixelTimeline_HighLowGain(DQMSummary): def __init__(self, gaink): self.k = gaink return None From bb7bb95b2f90c39d859769d0fb58ca7ef543dfca Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Fri, 9 Feb 2024 13:17:35 +0100 Subject: [PATCH 6/8] Declaring all class members in init --- src/nectarchain/dqm/camera_monitoring.py | 31 +++++++--- src/nectarchain/dqm/charge_integration.py | 33 ++++++++--- src/nectarchain/dqm/dqm_summary_processor.py | 1 - src/nectarchain/dqm/mean_camera_display.py | 44 +++++++++------ src/nectarchain/dqm/mean_waveforms.py | 24 ++++---- src/nectarchain/dqm/pixel_participation.py | 59 ++++++++++++-------- src/nectarchain/dqm/pixel_timeline.py | 35 +++++------- src/nectarchain/dqm/start_dqm.py | 41 ++++++++++++++ src/nectarchain/dqm/trigger_statistics.py | 28 +++++++--- 9 files changed, 200 insertions(+), 96 deletions(-) diff --git a/src/nectarchain/dqm/camera_monitoring.py b/src/nectarchain/dqm/camera_monitoring.py index d012d2b1..57af29e9 100644 --- a/src/nectarchain/dqm/camera_monitoring.py +++ b/src/nectarchain/dqm/camera_monitoring.py @@ -13,6 +13,29 @@ class CameraMonitoring(DQMSummary): def __init__(self, gaink): self.k = gaink + self.Pix = None + self.Samp = None + self.camera = None + self.cmap = None + self.subarray = None + self.event_id = [] + self.event_times = [] + self.DrawerTemp = None + self.run_start = None + self.run_end = None + self.DrawerTimes = None + self.DrawerTemp11 = None + self.DrawerTemp21 = None + self.DrawerNum1 = None + self.DrawerTimes_new = None + self.DrawerTemp12 = None + self.DrawerTemp22 = None + self.DrawerNum2 = None + self.DrawerTemp1_mean = [] + self.DrawerTemp2_mean = [] + self.CameraMonitoring_Results_Dict = {} + self.ChargeInt_Figures_Dict = {} + self.ChargeInt_Figures_Names_Dict = {} def ConfigureForRun(self, path, Pix, Samp, Reader1): # define number of pixels and samples @@ -26,9 +49,6 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.subarray = Reader1.subarray - self.event_id = [] - self.event_times = [] - for i, evt1 in enumerate(Reader1): self.run_start1 = evt1.nectarcam.tel[0].svc.date @@ -89,8 +109,6 @@ def FinishRun(self): self.DrawerTemp22 = self.DrawerTemp21[self.DrawerTimes_new < self.run_end] self.DrawerNum2 = self.DrawerNum1[self.DrawerTimes_new < self.run_end] - self.DrawerTemp1_mean = [] - self.DrawerTemp2_mean = [] TotalDrawers = np.max(self.DrawerNum2) for i in range(TotalDrawers + 1): @@ -110,7 +128,6 @@ def FinishRun(self): print("DRAWER TEMPERATURE COULD NOT BE RETRIEVED!") def GetResults(self): - self.CameraMonitoring_Results_Dict = {} try: self.CameraMonitoring_Results_Dict[ "CAMERA-TEMPERATURE-AVERAGE" @@ -122,8 +139,6 @@ def GetResults(self): return self.CameraMonitoring_Results_Dict def PlotResults(self, name, FigPath): - self.ChargeInt_Figures_Dict = {} - self.ChargeInt_Figures_Names_Dict = {} try: fig, disp = plt.subplots() diff --git a/src/nectarchain/dqm/charge_integration.py b/src/nectarchain/dqm/charge_integration.py index c573af73..6417f04e 100644 --- a/src/nectarchain/dqm/charge_integration.py +++ b/src/nectarchain/dqm/charge_integration.py @@ -12,6 +12,32 @@ class ChargeIntegrationHighLowGain(DQMSummary): def __init__(self, gaink): self.k = gaink + self.Pix = None + self.Samp = None + self.counter_evt = None + self.counter_ped = None + self.image_all = [] + self.peakpos_all = [] + self.image_ped = [] + self.peakpos_ped = [] + self.camera = None + self.cmap = None + self.subarray = None + self.integrator = None + self.pixelBAD = None + self.image_all = [] + self.image_all_median = None + self.image_all_average = None + self.image_all_std = None + self.image_all_rms = None + self.image_ped = [] + self.image_ped_median = None + self.image_ped_average = None + self.image_ped_std = None + self.image_ped_rms = None + self.ChargeInt_Results_Dict = {} + self.ChargeInt_Figures_Dict = {} + self.ChargeInt_Figures_Names_Dict = {} def ConfigureForRun(self, path, Pix, Samp, Reader1): # define number of pixels and samples @@ -38,11 +64,7 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.integrator = LocalPeakWindowSum(subarray, config=config) - self.image_all = [] - self.peakpos_all = [] - self.image_ped = [] - self.peakpos_ped = [] def ProcessEvent(self, evt, noped): self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels @@ -107,7 +129,6 @@ def FinishRun(self): self.image_ped_rms = np.sqrt(np.sum(self.image_ped**2, axis=0)) def GetResults(self): - self.ChargeInt_Results_Dict = {} if self.k == 0: self.ChargeInt_Results_Dict[ @@ -168,8 +189,6 @@ def GetResults(self): return self.ChargeInt_Results_Dict def PlotResults(self, name, FigPath): - self.ChargeInt_Figures_Dict = {} - self.ChargeInt_Figures_Names_Dict = {} # titles = ['All', 'Pedestals'] if self.k == 0: diff --git a/src/nectarchain/dqm/dqm_summary_processor.py b/src/nectarchain/dqm/dqm_summary_processor.py index 8486487f..89584903 100644 --- a/src/nectarchain/dqm/dqm_summary_processor.py +++ b/src/nectarchain/dqm/dqm_summary_processor.py @@ -13,7 +13,6 @@ def DefineForRun(self, reader1): self.FirstReader = reader1 self.Samp = len(evt1.r0.tel[0].waveform[0][0]) self.Pix = len(evt1.r0.tel[0].waveform[0]) - return self.Pix, self.Samp def ConfigureForRun(self): diff --git a/src/nectarchain/dqm/mean_camera_display.py b/src/nectarchain/dqm/mean_camera_display.py index 837e2f84..43baf4b7 100644 --- a/src/nectarchain/dqm/mean_camera_display.py +++ b/src/nectarchain/dqm/mean_camera_display.py @@ -8,15 +8,31 @@ class MeanCameraDisplay_HighLowGain(DQMSummary): def __init__(self, gaink): self.k = gaink - return None + self.Pix = None + self.Samp = None + self.CameraAverage = None + self.CameraAverage_ped = None + self.counter_evt = None + self.counter_ped = None + self.camera = None + self.cmap = None + self.CameraAverage = [] + self.CameraAverage1 = [] + self.CameraAverage_ped = [] + self.CameraAverage_ped1 = [] + self.CameraAverage_overEvents = None + self.CameraAverage_overEvents_overSamp = None + self.CameraAverage_ped_overEvents = None + self.CameraAverage_ped_overEvents_overSamp = None + self.MeanCameraDisplay_Results_Dict = {} + self.MeanCameraDisplay_Figures_Dict = {} + self.MeanCameraDisplay_Figures_Names_Dict = {} def ConfigureForRun(self, path, Pix, Samp, Reader1): # define number of pixels and samples self.Pix = Pix self.Samp = Samp - self.CameraAverage = np.zeros(self.Pix) - self.CameraAverage_ped = np.zeros(self.Pix) self.counter_evt = 0 self.counter_ped = 0 @@ -26,8 +42,6 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.cmap = "gnuplot2" - self.CameraAverage = [] - self.CameraAverage_ped = [] def ProcessEvent(self, evt, noped): self.pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels @@ -72,8 +86,6 @@ def FinishRun(self): ) def GetResults(self): - # INITIATE DICT - self.MeanCameraDisplay_Results_Dict = {} # ASSIGN RESUTLS TO DICT if self.k == 0: @@ -113,8 +125,6 @@ def GetResults(self): return self.MeanCameraDisplay_Results_Dict def PlotResults(self, name, FigPath): - self.MeanCameraDisplay_Figures_Dict = {} - self.MeanCameraDisplay_Figures_Names_Dict = {} # titles = ['All', 'Pedestals'] if self.k == 0: @@ -123,14 +133,14 @@ def PlotResults(self, name, FigPath): gain_c = "Low" if self.counter_evt > 0: - fig1, self.disp1 = plt.subplots() - self.disp1 = CameraDisplay( + fig1, disp1 = plt.subplots() + disp1 = CameraDisplay( geometry=self.camera[~self.pixelBAD[0]], image=self.CameraAverage_overEvents_overSamp[~self.pixelBAD[0]], cmap=plt.cm.coolwarm, ) - self.disp1.add_colorbar() - self.disp1.axes.text(2.0, 0, "Charge (DC)", rotation=90) + disp1.add_colorbar() + disp1.axes.text(2.0, 0, "Charge (DC)", rotation=90) plt.title("Camera average %s gain (ALL)" % gain_c) self.MeanCameraDisplay_Figures_Dict[ @@ -144,14 +154,14 @@ def PlotResults(self, name, FigPath): plt.close() if self.counter_ped > 0: - fig2, self.disp2 = plt.subplots() - self.disp2 = CameraDisplay( + fig2, disp2 = plt.subplots() + disp2 = CameraDisplay( geometry=self.camera[~self.pixelBAD[0]], image=self.CameraAverage_ped_overEvents_overSamp[~self.pixelBAD[0]], cmap=plt.cm.coolwarm, ) - self.disp2.add_colorbar() - self.disp2.axes.text(2.0, 0, "Charge (DC)", rotation=90) + disp2.add_colorbar() + disp2.axes.text(2.0, 0, "Charge (DC)", rotation=90) plt.title("Camera average %s gain (PED)" % gain_c) self.MeanCameraDisplay_Figures_Dict[ diff --git a/src/nectarchain/dqm/mean_waveforms.py b/src/nectarchain/dqm/mean_waveforms.py index 9f61e40e..7e17b2cb 100644 --- a/src/nectarchain/dqm/mean_waveforms.py +++ b/src/nectarchain/dqm/mean_waveforms.py @@ -6,24 +6,30 @@ class MeanWaveFormsHighLowGain(DQMSummary): def __init__(self, gaink): self.k = gaink - return None + self.Pix = None + self.Samp = None + self.Mwf = None + self.Mwf_ped = None + self.counter_evt = None + self.counter_ped = None + self.Mwf_average = None + self.Mwf_ped_average = None + self.Mwf_Mean_overPix = [] + self.Mwf_ped_Mean_overPix = [] + self.MeanWaveForms_Results_Dict = {} + self.MeanWaveForms_Figures_Dict = {} + self.MeanWaveForms_Figures_Names_Dict = {} def ConfigureForRun(self, path, Pix, Samp, Reader1): # define number of pixels and samples self.Pix = Pix self.Samp = Samp - - # redefine everything self.Mwf = np.zeros((self.Pix, self.Samp)) self.Mwf_ped = np.zeros((self.Pix, self.Samp)) self.counter_evt = 0 self.counter_ped = 0 - self.Mwf_average = np.zeros((self.Pix, self.Samp)) self.Mwf_ped_average = np.zeros((self.Pix, self.Samp)) - self.Mwf_Mean_overPix = [] - self.Mwf_ped_Mean_overPix = [] - self.wf_list_plot = list(range(1, self.Samp + 1)) # used for plotting later on return None @@ -64,8 +70,6 @@ def FinishRun(self): return None def GetResults(self): - # INITIATE DICT - self.MeanWaveForms_Results_Dict = {} # ASSIGN RESUTLS TO DICT if self.k == 0: @@ -89,8 +93,6 @@ def GetResults(self): return self.MeanWaveForms_Results_Dict def PlotResults(self, name, FigPath): - self.MeanWaveForms_Figures_Dict = {} - self.MeanWaveForms_Figures_Names_Dict = {} wf_list = np.array(self.wf_list_plot) diff --git a/src/nectarchain/dqm/pixel_participation.py b/src/nectarchain/dqm/pixel_participation.py index fc8841a1..5d707ee4 100644 --- a/src/nectarchain/dqm/pixel_participation.py +++ b/src/nectarchain/dqm/pixel_participation.py @@ -6,29 +6,44 @@ import numpy as np -class PixelParticipation_HighLowGain(DQMSummary): +class PixelParticipationHighLowGain(DQMSummary): def __init__(self, gaink): self.k = gaink - return None + self.Pix = None + self.Samp = None + self.counter_evt = None + self.counter_ped = None + self.BadPixels_ped = None + self.BadPixels = None + self.camera = None + self.camera2 = None + self.cmap = None + self.cmap2 = None + self.PixelParticipation_Results_Dict = {} + self.PixelParticipation_Figures_Dict = {} + self.PixelParticipation_Figures_Names_Dict = {} def ConfigureForRun(self, path, Pix, Samp, Reader1): # define number of pixels and samples self.Pix = Pix self.Samp = Samp - self.counter_evt = 0 self.counter_ped = 0 + self.BadPixels_ped = np.zeros(self.Pix) + self.BadPixels = np.zeros(self.Pix) + + self.camera = Reader1.subarray.tel[0].camera.geometry.transform_to( + EngineeringCameraFrame() + ) - self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame()) - self.camera2 = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame()) + self.camera2 = Reader1.subarray.tel[0].camera.geometry.transform_to( + EngineeringCameraFrame() + ) self.cmap = "gnuplot2" self.cmap2 = "gnuplot2" - self.BadPixels_ped = np.zeros(self.Pix) - self.BadPixels = np.zeros(self.Pix) - def ProcessEvent(self, evt, noped): pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels[self.k] pixel = evt.nectarcam.tel[0].svc.pixel_ids @@ -57,8 +72,6 @@ def FinishRun(self): def GetResults(self): - # INITIATE DICT - self.PixelParticipation_Results_Dict = {} # ASSIGN RESUTLS TO DICT if self.k == 0: @@ -87,8 +100,6 @@ def GetResults(self): return self.PixelParticipation_Results_Dict def PlotResults(self, name, FigPath): - self.PixelParticipation_Figures_Dict = {} - self.PixelParticipation_Figures_Names_Dict = {} # titles = ['All', 'Pedestals'] if self.k == 0: @@ -97,16 +108,16 @@ def PlotResults(self, name, FigPath): gain_c = "Low" if self.counter_evt > 0: - fig1, self.disp1 = plt.subplots() - self.disp1 = CameraDisplay( + fig1, disp1 = plt.subplots() + disp1 = CameraDisplay( geometry=self.camera, image=self.BadPixels, cmap=self.cmap, ) - self.disp1.cmap = self.cmap - self.disp1.cmap = plt.cm.coolwarm - self.disp1.add_colorbar() - self.disp1.axes.text(2.0, 0, "Bad Pixels", rotation=90) + disp1.cmap = self.cmap + disp1.cmap = plt.cm.coolwarm + disp1.add_colorbar() + disp1.axes.text(2.0, 0, "Bad Pixels", rotation=90) plt.title("Camera BPX %s gain (ALL)" % gain_c) self.PixelParticipation_Figures_Dict[ @@ -120,16 +131,16 @@ def PlotResults(self, name, FigPath): plt.close() if self.counter_ped > 0: - fig2, self.disp2 = plt.subplots() - self.disp2 = CameraDisplay( + fig2, disp2 = plt.subplots() + disp2 = CameraDisplay( geometry=self.camera2, image=self.BadPixels_ped, cmap=self.cmap2, ) - self.disp2.cmap = self.cmap2 - self.disp2.cmap = plt.cm.coolwarm - self.disp2.add_colorbar() - self.disp2.axes.text(2.0, 0, "Bad Pixels", rotation=90) + disp2.cmap = self.cmap2 + disp2.cmap = plt.cm.coolwarm + disp2.add_colorbar() + disp2.axes.text(2.0, 0, "Bad Pixels", rotation=90) plt.title("Camera BPX %s gain (PED)" % gain_c) self.PixelParticipation_Figures_Dict[ diff --git a/src/nectarchain/dqm/pixel_timeline.py b/src/nectarchain/dqm/pixel_timeline.py index 659ebfd8..abe8f124 100644 --- a/src/nectarchain/dqm/pixel_timeline.py +++ b/src/nectarchain/dqm/pixel_timeline.py @@ -6,30 +6,31 @@ import numpy as np -class PixelTimeline_HighLowGain(DQMSummary): +class PixelTimelineHighLowGain(DQMSummary): def __init__(self, gaink): self.k = gaink - return None + self.Pix = None + self.Samp = None + self.counter_evt = None + self.counter_ped = None + self.SumBadPixels_ped = [] + self.SumBadPixels = [] + self.BadPixelTimeline_ped = None + self.BadPixelTimeline = None + self.camera = None + self.cmap = None + self.cmap2 = None + self.PixelTimeline_Results_Dict = {} + self.PixelTimeline_Figures_Dict = {} + self.PixelTimeline_Figures_Names_Dict = {} def ConfigureForRun(self, path, Pix, Samp, Reader1): # define number of pixels and samples self.Pix = Pix self.Samp = Samp - - self.counter_evt = 0 self.counter_ped = 0 - self.camera = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame()) - self.camera2 = CameraGeometry.from_name("NectarCam-003").transform_to(EngineeringCameraFrame()) - - self.cmap = "gnuplot2" - self.cmap2 = "gnuplot2" - - - self.SumBadPixels_ped = [] - self.SumBadPixels = [] - def ProcessEvent(self, evt, noped): pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels[self.k] pixel = evt.nectarcam.tel[0].svc.pixel_ids @@ -66,11 +67,7 @@ def FinishRun(self): print( self.BadPixelTimeline_ped) - - def GetResults(self): - # INITIATE DICT - self.PixelTimeline_Results_Dict = {} # ASSIGN RESUTLS TO DICT if self.k == 0: @@ -102,8 +99,6 @@ def GetResults(self): return self.PixelTimeline_Results_Dict def PlotResults(self, name, FigPath): - self.PixelTimeline_Figures_Dict = {} - self.PixelTimeline_Figures_Names_Dict = {} # titles = ['All', 'Pedestals'] if self.k == 0: diff --git a/src/nectarchain/dqm/start_dqm.py b/src/nectarchain/dqm/start_dqm.py index 38138ff7..daa66a3c 100644 --- a/src/nectarchain/dqm/start_dqm.py +++ b/src/nectarchain/dqm/start_dqm.py @@ -5,7 +5,17 @@ from camera_monitoring import CameraMonitoring from charge_integration import ChargeIntegrationHighLowGain +<<<<<<< HEAD from ctapipe.io import EventSource +======= +from pixel_participation import PixelParticipationHighLowGain +from pixel_timeline import PixelTimelineHighLowGain +from mean_camera_display import MeanCameraDisplay_HighLowGain +from mean_waveforms import MeanWaveFormsHighLowGain +from trigger_statistics import TriggerStatistics + +from ctapipe.io import EventSeeker, EventSource +>>>>>>> ba8e8ad (declaring all class members in init) from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN from db_utils import DQMDB from matplotlib import pyplot as plt @@ -140,6 +150,7 @@ def CreateFigFolder(name, type): # LIST OF PROCESSES TO RUN ######################################################################################## +<<<<<<< HEAD processors = [ TriggerStatistics(HIGH_GAIN), MeanWaveFormsHighLowGain(HIGH_GAIN), @@ -154,6 +165,36 @@ def CreateFigFolder(name, type): PixelTimeline_HighLowGain(HIGH_GAIN), PixelTimeline_HighLowGain(LOW_GAIN), ] +======= +a = TriggerStatistics(HIGH_GAIN) +b = MeanWaveFormsHighLowGain(HIGH_GAIN) +c = MeanWaveFormsHighLowGain(LOW_GAIN) +d = MeanCameraDisplay_HighLowGain(HIGH_GAIN) +e = MeanCameraDisplay_HighLowGain(LOW_GAIN) +f = ChargeIntegrationHighLowGain(HIGH_GAIN) +g = ChargeIntegrationHighLowGain(LOW_GAIN) +h = CameraMonitoring(HIGH_GAIN) +i = PixelParticipationHighLowGain(HIGH_GAIN) +j = PixelParticipationHighLowGain(LOW_GAIN) +k = PixelTimelineHighLowGain(HIGH_GAIN) +l = PixelTimelineHighLowGain(LOW_GAIN) + +processors = list() + +processors.append(a) +processors.append(b) +processors.append(c) +processors.append(d) +processors.append(e) +processors.append(f) +processors.append(g) +processors.append(h) +processors.append(i) +processors.append(j) +processors.append(k) +processors.append(l) + +>>>>>>> ba8e8ad (declaring all class members in init) # LIST OF DICT RESULTS Results_TriggerStatistics = {} diff --git a/src/nectarchain/dqm/trigger_statistics.py b/src/nectarchain/dqm/trigger_statistics.py index 8effd922..fe429223 100644 --- a/src/nectarchain/dqm/trigger_statistics.py +++ b/src/nectarchain/dqm/trigger_statistics.py @@ -9,17 +9,32 @@ class TriggerStatistics(DQMSummary): def __init__(self, gaink): self.k = gaink + self.Pix = None + self.Samp = None + self.event_type = [] + self.event_times = [] + self.event_id = [] + self.run_times = [] + self.run_start1 = None + self.run_start = None + self.run_end = None + self.event_ped_times = None + self.event_phy_times = None + self.event_other_times = None + self.event_ped_id = None + self.event_phy_id = None + self.event_other_id = None + self.event_wrong_times = None + self.TriggerStat_Results_Dict = {} + self.TriggerStat_Figures_Dict = {} + self.TriggerStat_Figures_Names_Dict = {} + def ConfigureForRun(self, path, Pix, Samp, Reader1): # define number of pixels and samples self.Pix = Pix self.Samp = Samp - self.event_type = [] - self.event_times = [] - self.event_id = [] - self.run_times = [] - def ProcessEvent(self, evt, noped): trigger_type = evt.trigger.event_type.value trigger_time = evt.trigger.time.value @@ -72,7 +87,6 @@ def FinishRun(self): self.event_times = self.event_times[self.event_times > self.run_start] def GetResults(self): - self.TriggerStat_Results_Dict = {} self.TriggerStat_Results_Dict["TRIGGER-TYPES"] = self.triggers self.TriggerStat_Results_Dict[ "TRIGGER-STATISTICS" @@ -97,8 +111,6 @@ def PlotResults(self, name, FigPath): n1 = np.array(self.event_times.max() - self.event_times.min(), dtype=object) n = math.ceil(n1 / w) - self.TriggerStat_Figures_Dict = {} - self.TriggerStat_Figures_Names_Dict = {} fig1, ax = plt.subplots() ax.hist( self.event_type, From b566041388026fa3d3965a068315703b0d05df8e Mon Sep 17 00:00:00 2001 From: Halim Ashkar Date: Fri, 9 Feb 2024 14:13:28 +0100 Subject: [PATCH 7/8] removing duplitcation from pixel_timeline --- src/nectarchain/dqm/pixel_participation.py | 72 +++++++++------------- src/nectarchain/dqm/start_dqm.py | 35 ++--------- 2 files changed, 35 insertions(+), 72 deletions(-) diff --git a/src/nectarchain/dqm/pixel_participation.py b/src/nectarchain/dqm/pixel_participation.py index 5d707ee4..b58b2aed 100644 --- a/src/nectarchain/dqm/pixel_participation.py +++ b/src/nectarchain/dqm/pixel_participation.py @@ -1,9 +1,8 @@ +import numpy as np +from ctapipe.coordinates import EngineeringCameraFrame +from ctapipe.visualization import CameraDisplay from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt -from ctapipe.visualization import CameraDisplay -from ctapipe.instrument import CameraGeometry -from ctapipe.coordinates import EngineeringCameraFrame -import numpy as np class PixelParticipationHighLowGain(DQMSummary): @@ -14,8 +13,8 @@ def __init__(self, gaink): self.counter_evt = None self.counter_ped = None self.BadPixels_ped = None - self.BadPixels = None - self.camera = None + self.BadPixels = None + self.camera = None self.camera2 = None self.cmap = None self.cmap2 = None @@ -30,7 +29,7 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.counter_evt = 0 self.counter_ped = 0 self.BadPixels_ped = np.zeros(self.Pix) - self.BadPixels = np.zeros(self.Pix) + self.BadPixels = np.zeros(self.Pix) self.camera = Reader1.subarray.tel[0].camera.geometry.transform_to( EngineeringCameraFrame() @@ -43,7 +42,6 @@ def ConfigureForRun(self, path, Pix, Samp, Reader1): self.cmap = "gnuplot2" self.cmap2 = "gnuplot2" - def ProcessEvent(self, evt, noped): pixelBAD = evt.mon.tel[0].pixel_status.hardware_failing_pixels[self.k] pixel = evt.nectarcam.tel[0].svc.pixel_ids @@ -51,7 +49,7 @@ def ProcessEvent(self, evt, noped): pixel21 = list(np.arange(0, self.Pix - len(pixel), 1, dtype=int)) pixel = list(pixel) pixels = np.concatenate([pixel21, pixel]) - else: + else: pixels = pixel if evt.trigger.event_type.value == 32: # count peds @@ -69,13 +67,9 @@ def FinishRun(self): self.BadPixels_ped = np.array(self.BadPixels_ped) self.BadPixels = np.array(self.BadPixels) - - def GetResults(self): - # ASSIGN RESUTLS TO DICT if self.k == 0: - if self.counter_evt > 0: self.PixelParticipation_Results_Dict[ "CAMERA-BadPix-PHY-OverEVENTS-HIGH-GAIN" @@ -100,7 +94,6 @@ def GetResults(self): return self.PixelParticipation_Results_Dict def PlotResults(self, name, FigPath): - # titles = ['All', 'Pedestals'] if self.k == 0: gain_c = "High" @@ -108,49 +101,44 @@ def PlotResults(self, name, FigPath): gain_c = "Low" if self.counter_evt > 0: - fig1, disp1 = plt.subplots() - disp1 = CameraDisplay( - geometry=self.camera, - image=self.BadPixels, - cmap=self.cmap, - ) - disp1.cmap = self.cmap - disp1.cmap = plt.cm.coolwarm - disp1.add_colorbar() - disp1.axes.text(2.0, 0, "Bad Pixels", rotation=90) - plt.title("Camera BPX %s gain (ALL)" % gain_c) + entity = self.BadPixels + title = "Camera BPX %s gain (ALL)" % gain_c + + if self.counter_ped > 0: + entity = self.BadPixels_ped + title = "Camera BPX %s gain (PED)" % gain_c + + fig, disp = plt.subplots() + disp = CameraDisplay( + geometry=self.camera, + image=entity, + cmap=self.cmap, + ) + disp.cmap = self.cmap + disp.cmap = plt.cm.coolwarm + disp.add_colorbar() + disp.axes.text(2.0, 0, "Bad Pixels", rotation=90) + plt.title(title) + if self.counter_ped > 0: self.PixelParticipation_Figures_Dict[ "CAMERA-BADPIX-PHY-DISPLAY-%s-GAIN" % gain_c - ] = fig1 + ] = fig full_name = name + "_Camera_BPX_%sGain.png" % gain_c FullPath = FigPath + full_name self.PixelParticipation_Figures_Names_Dict[ "CAMERA-BADPIX-PHY-DISPLAY-%s-GAIN" % gain_c ] = FullPath - plt.close() - - if self.counter_ped > 0: - fig2, disp2 = plt.subplots() - disp2 = CameraDisplay( - geometry=self.camera2, - image=self.BadPixels_ped, - cmap=self.cmap2, - ) - disp2.cmap = self.cmap2 - disp2.cmap = plt.cm.coolwarm - disp2.add_colorbar() - disp2.axes.text(2.0, 0, "Bad Pixels", rotation=90) - plt.title("Camera BPX %s gain (PED)" % gain_c) - + if self.counter_evt > 0: self.PixelParticipation_Figures_Dict[ "CAMERA-BADPIX-PED-DISPLAY-%s-GAIN" % gain_c - ] = fig2 + ] = fig full_name = name + "_Pedestal_BPX_%sGain.png" % gain_c FullPath = FigPath + full_name self.PixelParticipation_Figures_Names_Dict[ "CAMERA-BADPIX-PED-DISPLAY-%s-GAIN" % gain_c ] = FullPath + plt.close() return ( diff --git a/src/nectarchain/dqm/start_dqm.py b/src/nectarchain/dqm/start_dqm.py index daa66a3c..f6c91701 100644 --- a/src/nectarchain/dqm/start_dqm.py +++ b/src/nectarchain/dqm/start_dqm.py @@ -5,22 +5,14 @@ from camera_monitoring import CameraMonitoring from charge_integration import ChargeIntegrationHighLowGain -<<<<<<< HEAD from ctapipe.io import EventSource -======= -from pixel_participation import PixelParticipationHighLowGain -from pixel_timeline import PixelTimelineHighLowGain -from mean_camera_display import MeanCameraDisplay_HighLowGain -from mean_waveforms import MeanWaveFormsHighLowGain -from trigger_statistics import TriggerStatistics - -from ctapipe.io import EventSeeker, EventSource ->>>>>>> ba8e8ad (declaring all class members in init) from ctapipe_io_nectarcam.constants import HIGH_GAIN, LOW_GAIN from db_utils import DQMDB from matplotlib import pyplot as plt from mean_camera_display import MeanCameraDisplay_HighLowGain from mean_waveforms import MeanWaveFormsHighLowGain +from pixel_participation import PixelParticipationHighLowGain +from pixel_timeline import PixelTimelineHighLowGain from tqdm import tqdm from traitlets.config import Config from trigger_statistics import TriggerStatistics @@ -150,22 +142,6 @@ def CreateFigFolder(name, type): # LIST OF PROCESSES TO RUN ######################################################################################## -<<<<<<< HEAD -processors = [ - TriggerStatistics(HIGH_GAIN), - MeanWaveFormsHighLowGain(HIGH_GAIN), - MeanWaveFormsHighLowGain(LOW_GAIN), - MeanCameraDisplay_HighLowGain(HIGH_GAIN), - MeanCameraDisplay_HighLowGain(LOW_GAIN), - ChargeIntegrationHighLowGain(HIGH_GAIN), - ChargeIntegrationHighLowGain(LOW_GAIN), - CameraMonitoring(HIGH_GAIN), - PixelParticipation_HighLowGain(HIGH_GAIN), - PixelParticipation_HighLowGain(LOW_GAIN), - PixelTimeline_HighLowGain(HIGH_GAIN), - PixelTimeline_HighLowGain(LOW_GAIN), -] -======= a = TriggerStatistics(HIGH_GAIN) b = MeanWaveFormsHighLowGain(HIGH_GAIN) c = MeanWaveFormsHighLowGain(LOW_GAIN) @@ -177,7 +153,7 @@ def CreateFigFolder(name, type): i = PixelParticipationHighLowGain(HIGH_GAIN) j = PixelParticipationHighLowGain(LOW_GAIN) k = PixelTimelineHighLowGain(HIGH_GAIN) -l = PixelTimelineHighLowGain(LOW_GAIN) +ll = PixelTimelineHighLowGain(LOW_GAIN) processors = list() @@ -192,9 +168,8 @@ def CreateFigFolder(name, type): processors.append(i) processors.append(j) processors.append(k) -processors.append(l) +processors.append(ll) ->>>>>>> ba8e8ad (declaring all class members in init) # LIST OF DICT RESULTS Results_TriggerStatistics = {} @@ -227,7 +202,7 @@ def CreateFigFolder(name, type): "Results_PixelTimeline_LowGain", ] -#NESTED_DICT_KEYS = ["Results_PixelParticipation_HighGain", "Results_PixelTimeline_HighGain"] +# NESTED_DICT_KEYS = ["Results_PixelParticipation_HighGain"] # START for p in processors: From dc51915a5f0c09ecf0cc249021132a1fdd1a7fa4 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Lenain Date: Fri, 9 Feb 2024 14:27:07 +0100 Subject: [PATCH 8/8] Lint code with pre-commit hooks enabled --- src/nectarchain/dqm/pixel_timeline.py | 45 ++++++++++++--------------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/src/nectarchain/dqm/pixel_timeline.py b/src/nectarchain/dqm/pixel_timeline.py index abe8f124..e120f561 100644 --- a/src/nectarchain/dqm/pixel_timeline.py +++ b/src/nectarchain/dqm/pixel_timeline.py @@ -1,9 +1,6 @@ +import numpy as np from dqm_summary_processor import DQMSummary from matplotlib import pyplot as plt -from ctapipe.visualization import CameraDisplay -from ctapipe.instrument import CameraGeometry -from ctapipe.coordinates import EngineeringCameraFrame -import numpy as np class PixelTimelineHighLowGain(DQMSummary): @@ -19,7 +16,7 @@ def __init__(self, gaink): self.BadPixelTimeline = None self.camera = None self.cmap = None - self.cmap2 = None + self.cmap2 = None self.PixelTimeline_Results_Dict = {} self.PixelTimeline_Figures_Dict = {} self.PixelTimeline_Figures_Names_Dict = {} @@ -38,7 +35,7 @@ def ProcessEvent(self, evt, noped): pixel21 = list(np.arange(0, self.Pix - len(pixel), 1, dtype=int)) pixel = list(pixel) pixels = np.concatenate([pixel21, pixel]) - else: + else: pixels = pixel if evt.trigger.event_type.value == 32: # count peds @@ -60,30 +57,26 @@ def ProcessEvent(self, evt, noped): return None def FinishRun(self): - - self.BadPixelTimeline_ped = np.array(self.SumBadPixels_ped, dtype=float)/self.Pix - self.BadPixelTimeline = np.array(self.SumBadPixels, dtype=float)/self.Pix + self.BadPixelTimeline_ped = ( + np.array(self.SumBadPixels_ped, dtype=float) / self.Pix + ) + self.BadPixelTimeline = np.array(self.SumBadPixels, dtype=float) / self.Pix print(self.BadPixelTimeline) - print( self.BadPixelTimeline_ped) - + print(self.BadPixelTimeline_ped) def GetResults(self): - # ASSIGN RESUTLS TO DICT if self.k == 0: - if self.counter_evt > 0: self.PixelTimeline_Results_Dict[ "CAMERA-BadPixTimeline-PHY-HIGH-GAIN" ] = self.BadPixelTimeline - if self.counter_ped > 0: self.PixelTimeline_Results_Dict[ "CAMERA-BadPixTimeline-PED-HIGH-GAIN" ] = self.BadPixelTimeline_ped - if self.k == 1: if self.counter_evt > 0: self.PixelTimeline_Results_Dict[ @@ -95,11 +88,9 @@ def GetResults(self): "CAMERA-BadPixTimeline-PED-LOW-GAIN" ] = self.BadPixelTimeline_ped - return self.PixelTimeline_Results_Dict def PlotResults(self, name, FigPath): - # titles = ['All', 'Pedestals'] if self.k == 0: gain_c = "High" @@ -108,7 +99,11 @@ def PlotResults(self, name, FigPath): if self.counter_evt > 0: fig1, disp = plt.subplots() - plt.plot(np.arange(self.counter_evt), self.BadPixelTimeline*100, label = "Physical events") + plt.plot( + np.arange(self.counter_evt), + self.BadPixelTimeline * 100, + label="Physical events", + ) plt.legend() plt.xlabel("Timeline") plt.ylabel("BPX fraction (%)") @@ -116,9 +111,7 @@ def PlotResults(self, name, FigPath): full_name = name + "_BPX_Timeline_%sGain_All.png" % gain_c FullPath = FigPath + full_name - self.PixelTimeline_Figures_Dict[ - "BPX-TIMELINE-ALL-%s-GAIN" % gain_c - ] = fig1 + self.PixelTimeline_Figures_Dict["BPX-TIMELINE-ALL-%s-GAIN" % gain_c] = fig1 self.PixelTimeline_Figures_Names_Dict[ "BPX-TIMELINE-ALL-%s-GAIN" % gain_c ] = FullPath @@ -127,7 +120,11 @@ def PlotResults(self, name, FigPath): if self.counter_ped > 0: fig2, disp = plt.subplots() - plt.plot(np.arange(self.counter_ped), self.BadPixelTimeline_ped*100, label = "Pedestal events") + plt.plot( + np.arange(self.counter_ped), + self.BadPixelTimeline_ped * 100, + label="Pedestal events", + ) plt.legend() plt.xlabel("Timeline") plt.ylabel("BPX fraction (%)") @@ -135,9 +132,7 @@ def PlotResults(self, name, FigPath): full_name = name + "_BPX_Timeline_%sGain_Ped.png" % gain_c FullPath = FigPath + full_name - self.PixelTimeline_Figures_Dict[ - "BPX-TIMELINE-PED-%s-GAIN" % gain_c - ] = fig2 + self.PixelTimeline_Figures_Dict["BPX-TIMELINE-PED-%s-GAIN" % gain_c] = fig2 self.PixelTimeline_Figures_Names_Dict[ "BPX-TIMELINE-PED-%s-GAIN" % gain_c ] = FullPath