Merge pull request #193 from ctlearn-project/ctlearn_clean

Support only ctapipe data format v6.0.0

.gitignore

@@ -14,8 +14,9 @@
 # Compiled Python files
 __pycache__/
 *.py[cod]
-
+.DS_Store
+ctlearn/.DS_Store
 ctlearn.egg-info/
 
 # Sphinx documentation
-docs/build/
+docs/build/

.zenodo.json

@@ -62,5 +62,5 @@
     ],
     "title": "CTLearn: Deep learning for imaging atmospheric Cherenkov telescopes event reconstruction",
     "upload_type": "software",
-    "version": "0.7.0"
+    "version": "0.8.0"
 }

ctlearn/data_loader.py

@@ -7,7 +7,7 @@ class KerasBatchGenerator(tf.keras.utils.Sequence):
 
     def __init__(
         self,
-        DL1DataReaderDL1DH,
+        DLDataReader,
         indices,
         batch_size=64,
         mode="train",
@@ -16,7 +16,7 @@ def __init__(
         stack_telescope_images=False,
     ):
         "Initialization"
-        self.DL1DataReaderDL1DH = DL1DataReaderDL1DH
+        self.DLDataReader = DLDataReader
         self.batch_size = batch_size
         self.indices = indices
         self.mode = mode
@@ -30,31 +30,26 @@ def __init__(
         self.singleimg_shape = None
         self.trg_pos, self.trg_shape = None, None
         self.trgpatch_pos, self.trgpatch_shape = None, None
-        self.pon_pos = None
-        self.pointing = []
         self.wvf_pos, self.wvf_shape = None, None
         self.img_pos, self.img_shape = None, None
         self.prm_pos, self.prm_shape = None, None
         self.parameter_list = []
         # Additional info
         self.evt_pos, self.obs_pos = None, None
         self.event_list, self.obs_list = [], []
-        self.mjd_pos, self.milli_pos, self.nano_pos = None, None, None
-        self.mjd_list, self.milli_list, self.nano_list = [], [], []
         # Labels
         self.prt_pos, self.enr_pos, self.drc_pos = None, None, None
+        self.drc_unit = None
         self.prt_labels = []
         self.enr_labels = []
-        self.alt_labels, self.az_labels = [], []
+        self.az_labels, self.alt_labels, self.sep_labels = [], [], []
         self.trgpatch_labels = []
         self.energy_unit = None
 
-        for i, desc in enumerate(self.DL1DataReaderDL1DH.example_description):
+        for i, desc in enumerate(self.DLDataReader.example_description):
             if "HWtrigger" in desc["name"]:
                 self.trg_pos = i
                 self.trg_shape = desc["shape"]
-            elif "pointing" in desc["name"]:
-                self.pon_pos = i
             elif "waveform" in desc["name"]:
                 self.wvf_pos = i
                 self.wvf_shape = desc["shape"]
@@ -74,16 +69,11 @@ def __init__(
                 self.energy_unit = desc["unit"]
             elif "direction" in desc["name"]:
                 self.drc_pos = i
+                self.drc_unit = desc["unit"]
             elif "event_id" in desc["name"]:
                 self.evt_pos = i
             elif "obs_id" in desc["name"]:
                 self.obs_pos = i
-            elif "mjd" in desc["name"]:
-                self.mjd_pos = i
-            elif "milli_sec" in desc["name"]:
-                self.milli_pos = i
-            elif "nano_sec" in desc["name"]:
-                self.nano_pos = i
 
         # Retrieve shape from a single image in stereo analysis
         if self.trg_pos is not None and self.img_pos is not None:
@@ -132,14 +122,14 @@ def __data_generation(self, batch_indices):
             if self.enr_pos is not None:
                 energy = np.empty((self.batch_size))
             if self.drc_pos is not None:
-                direction = np.empty((self.batch_size, 2))
+                direction = np.empty((self.batch_size, 3))
             if self.trgpatch_pos is not None:
                 trigger_patches_true_image_sum = np.empty(
                     (self.batch_size, *self.trgpatch_shape)
                 )
         # Generate data
         for i, index in enumerate(batch_indices):
-            event = self.DL1DataReaderDL1DH[index]
+            event = self.DLDataReader[index]
             # Fill the features
             if self.trg_pos is not None:
                 triggers[i] = event[self.trg_pos]
@@ -155,7 +145,7 @@ def __data_generation(self, batch_indices):
                 if self.prt_pos is not None:
                     particletype[
                         i
-                    ] = self.DL1DataReaderDL1DH.shower_primary_id_to_class[
+                    ] = self.DLDataReader.shower_primary_id_to_class[
                         int(event[self.prt_pos])
                     ]
                 if self.enr_pos is not None:
@@ -169,15 +159,13 @@ def __data_generation(self, batch_indices):
                 if self.prt_pos is not None:
                     self.prt_labels.append(np.float32(event[self.prt_pos]))
                 if self.enr_pos is not None:
-                    self.enr_labels.append(np.float32(event[self.enr_pos][0]))
+                    self.enr_labels.append(np.float32(event[self.enr_pos]))
                 if self.drc_pos is not None:
-                    self.alt_labels.append(np.float32(event[self.drc_pos][0]))
-                    self.az_labels.append(np.float32(event[self.drc_pos][1]))
+                    self.az_labels.append(np.float32(event[self.drc_pos][0]))
+                    self.alt_labels.append(np.float32(event[self.drc_pos][1]))
+                    self.sep_labels.append(np.float32(event[self.drc_pos][2]))
                 if self.trgpatch_pos is not None:
                     self.trgpatch_labels.append(np.float32(event[self.trgpatch_pos]))
-                # Save pointing
-                if self.pon_pos is not None:
-                    self.pointing.append(event[self.pon_pos])
                 # Save all parameters for the prediction phase
                 if self.prm_pos is not None:
                     self.parameter_list.append(event[self.prm_pos])
@@ -186,13 +174,6 @@ def __data_generation(self, batch_indices):
                     self.event_list.append(np.float32(event[self.evt_pos]))
                 if self.obs_pos is not None:
                     self.obs_list.append(np.float32(event[self.obs_pos]))
-                # Save timestamp
-                if self.mjd_pos is not None:
-                    self.mjd_list.append(np.float32(event[self.mjd_pos]))
-                if self.milli_pos is not None:
-                    self.milli_list.append(np.float32(event[self.milli_pos]))
-                if self.nano_pos is not None:
-                    self.nano_list.append(np.float32(event[self.nano_pos]))
 
         features = {}
         if self.trg_pos is not None:
@@ -209,19 +190,19 @@ def __data_generation(self, batch_indices):
             if self.prt_pos is not None:
                 labels["type"] = tf.keras.utils.to_categorical(
                     particletype,
-                    num_classes=self.DL1DataReaderDL1DH.num_classes,
+                    num_classes=self.DLDataReader.num_classes,
                 )
                 label = tf.keras.utils.to_categorical(
                     particletype,
-                    num_classes=self.DL1DataReaderDL1DH.num_classes,
+                    num_classes=self.DLDataReader.num_classes,
                 )
             if self.enr_pos is not None:
                 labels["energy"] = energy.reshape((-1, 1))
                 label = energy.reshape((-1, 1))
             if self.drc_pos is not None:
                 labels["direction"] = direction
                 label = direction
-            if self.trgpatch_pos is not None and self.DL1DataReaderDL1DH.reco_cherenkov_photons:
+            if self.trgpatch_pos is not None and self.DLDataReader.reco_cherenkov_photons:
                 labels["cherenkov_photons"] = trigger_patches_true_image_sum
                 label = trigger_patches_true_image_sum
 

ctlearn/default_config_files/triggerSingleCNN.yml

@@ -3,8 +3,8 @@ Data:
     trigger_settings:
         number_of_trigger_patches: [7, 7]
         reco_cherenkov_photons: False
-        include_nsb_patches: 'auto'
-        trigger_patch_from_simulation: True
+        include_nsb_patches: 'off'
+        get_trigger_patch: 'simulation'
     waveform_settings:
         waveform_type: 'raw'
         waveform_sequence_length: 5

ctlearn/default_config_files/triggerTRN.yml

@@ -3,8 +3,8 @@ Data:
     trigger_settings:
         number_of_trigger_patches: [7, 7]
         reco_cherenkov_photons: False
-        include_nsb_patches: 'auto'
-        trigger_patch_from_simulation: True
+        include_nsb_patches: 'off'
+        get_trigger_patch: 'simulation'
     waveform_settings:
         waveform_type: 'raw'
         waveform_sequence_length: 5

ctlearn/default_models/head.py

@@ -43,7 +43,7 @@ def standard_head(inputs, tasks, params):
         logits["direction"] = fully_connect(
             inputs,
             standard_head_settings["direction"]["fc_head"],
-            expected_logits_dimension=2,
+            expected_logits_dimension=3,
             name="direction",
         )
         losses["direction"] = tf.keras.losses.MeanAbsoluteError(