Improvements to Tool and Component functionality (#1052)

* added initial function to turn Component or Tool into yaml

* show difference to default value in written config

* replaced custom log code with LoggingConfigurable

* added features to Component:

- `get_current_config()` returns the *full* configuration of the instance, unlike the `.config` attribute which only returns the non-default settings. This includes even changes made during the running of the program, not from the input configuration files.

- added a nicer HTML repr for notebooks

* Added features to Tool:

- properly set the default log_format trait
- allow Component *instances* to be registered (so we can get their config later)
- added `get_current_config()`, similar to the one in Component, but returning the actual config for the Tool instance + all registered Component instances
- add the full instance config to the provenance system
- added an HTML repr for notebooks
- fixed a type in test_component_current_config()
- add tool example
- always log the config (as info not debug)
- ensure allowed_tels is a set internally
- make sure set() is JSON serializable
- show difference to default value in written config
- added tests for new tool functionality
- Update notebooks

* remove CameraR1Calibrator

* code style cleanups

* added docstrings

* make main() func so

* fix to work with new CameraCalibrator

* fixed muon tool

* removed superflous import

* removed utils/tools.py and moved to core/traits.py

* fix some problems with the move of utils/tools.py

* fix imports borked during refactor

* small changes to fix style complaints

* just some reformatting

* fixed some strings that were not merged after reformatting

* removed duplicate log_format trait

* fixed typo

ctapipe/analysis/camera/charge_resolution.py

@@ -5,37 +5,38 @@
 
 
 class ChargeResolutionCalculator:
-    def __init__(self, mc_true=True):
-        """
-        Calculates the charge resolution with an efficient, low-memory,
-        interative approach, allowing the contribution of data/events
-        without reading the entire dataset into memory.
-
-        Utilises Pandas DataFrames, and makes no assumptions on the order of
-        the data, and does not require the true charge to be integer (as may
-        be the case for lab measurements where an average illumination
-        is used).
-
-        A list is filled with a dataframe for each contribution, and only
-        amalgamated into a single dataframe (reducing memory) once the memory
-        of the list becomes large (or at the end of the filling),
-        reducing the time required to produce the output.
-
-        Parameters
-        ----------
-        mc_true : bool
-            Indicate if the "true charge" values are from the sim_telarray
-            files, and therefore without poisson error. The poisson error will
-            therefore be included in the charge resolution calculation.
+    """
+    Calculates the charge resolution with an efficient, low-memory,
+    interative approach, allowing the contribution of data/events
+    without reading the entire dataset into memory.
+
+    Utilises Pandas DataFrames, and makes no assumptions on the order of
+    the data, and does not require the true charge to be integer (as may
+    be the case for lab measurements where an average illumination
+    is used).
+
+    A list is filled with a dataframe for each contribution, and only
+    amalgamated into a single dataframe (reducing memory) once the memory
+    of the list becomes large (or at the end of the filling),
+    reducing the time required to produce the output.
+
+    Parameters
+    ----------
+    mc_true : bool
+        Indicate if the "true charge" values are from the sim_telarray
+        files, and therefore without poisson error. The poisson error will
+        therefore be included in the charge resolution calculation.
+
+    Attributes
+    ----------
+    self._mc_true : bool
+    self._df_list : list
+    self._df : pd.DataFrame
+    self._n_bytes : int
+        Monitors the number of bytes being held in memory
+    """
 
-        Attributes
-        ----------
-        self._mc_true : bool
-        self._df_list : list
-        self._df : pd.DataFrame
-        self._n_bytes : int
-            Monitors the number of bytes being held in memory
-        """
+    def __init__(self, mc_true=True):
         self._mc_true = mc_true
         self._df_list = []
         self._df = pd.DataFrame()

ctapipe/core/__init__.py

@@ -1,4 +1,7 @@
 # Licensed under a 3-clause BSD style license - see LICENSE.rst
+"""
+Core functionality of ctapipe
+"""
 
 from .component import Component, non_abstract_children
 from .container import Container, Field, Map

ctapipe/core/component.py

@@ -1,9 +1,11 @@
 """ Class to handle configuration for algorithms """
 from abc import ABCMeta
-from logging import getLogger
 from inspect import isabstract
-from traitlets.config import Configurable
+from logging import getLogger
+
 from traitlets import TraitError
+from traitlets.config import Configurable
+
 from ctapipe.core.plugins import detect_and_import_io_plugins
 
 
@@ -31,10 +33,10 @@ def non_abstract_children(base):
 
 
 class AbstractConfigurableMeta(type(Configurable), ABCMeta):
-    '''
+    """
     Metaclass to be able to make Component abstract
     see: http://stackoverflow.com/a/7314847/3838691
-    '''
+    """
     pass
 
 
@@ -108,7 +110,8 @@ def __init__(self, config=None, parent=None, **kwargs):
             if not self.has_trait(key):
                 raise TraitError(f"Traitlet does not exist: {key}")
 
-        # set up logging
+        # set up logging (for some reason the logger registered by LoggingConfig
+        # doesn't use a child logger of the parent by default)
         if self.parent:
             self.log = self.parent.log.getChild(self.__class__.__name__)
         else:
@@ -130,11 +133,11 @@ def from_name(cls, name, config=None, parent=None):
             Used to set traitlet values.
             This argument is typically only specified when using this method
             from within a Tool.
-        tool : ctapipe.core.Tool
+        parent : ctapipe.core.Tool
             Tool executable that is calling this component.
-            Passes the correct logger to the component.
+            Passes the correct logger and configuration to the component.
             This argument is typically only specified when using this method
-            from within a Tool.
+            from within a Tool (config need not be passed if parent is used).
 
         Returns
         -------
@@ -149,3 +152,33 @@ def from_name(cls, name, config=None, parent=None):
         requested_subclass = subclasses[name]
 
         return requested_subclass(config=config, parent=parent)
+
+    def get_current_config(self):
+        """ return the current configuration as a dict (e.g. the values
+        of all traits, even if they were not set during configuration)
+        """
+        return {
+            self.__class__.__name__: {
+                k: v.get(self) for k, v in self.traits(config=True).items()
+            }
+        }
+
+    def _repr_html_(self):
+        """ nice HTML rep, with blue for non-default values"""
+        traits = self.traits()
+        name = self.__class__.__name__
+        lines = [
+            f"<b>{name}</b>",
+            f"<p> {self.__class__.__doc__ or 'Undocumented!'} </p>",
+            "<table>"
+        ]
+        for key, val in self.get_current_config()[name].items():
+            thehelp = f'{traits[key].help} (default: {traits[key].default_value})'
+            lines.append(f"<tr><th>{key}</th>")
+            if val != traits[key].default_value:
+                lines.append(f"<td><span style='color:blue'>{val}</span></td>")
+            else:
+                lines.append(f"<td>{val}</td>")
+            lines.append(f'<td style="text-align:left"><i>{thehelp}</i></td></tr>')
+        lines.append("</table>")
+        return "\n".join(lines)

ctapipe/core/container.py

@@ -35,7 +35,7 @@ def __repr__(self):
 
 
 class ContainerMeta(type):
-    '''
+    """
     The MetaClass for the Containers
 
     It reserves __slots__ for every class variable,
@@ -44,7 +44,8 @@ class ContainerMeta(type):
 
     This makes sure, that the metadata is immutable,
     and no new fields can be added to a container by accident.
-    '''
+    """
+
     def __new__(cls, name, bases, dct):
         field_names = [
             k for k, v in dct.items()
@@ -100,7 +101,7 @@ class Container(metaclass=ContainerMeta):
     >>>
     >>>    cont = MyContainer()
     >>>    print(cont.x)
-    >>>    # metdata will become header keywords in an output file:
+    >>>    # metadata will become header keywords in an output file:
     >>>    cont.meta['KEY'] = value
 
     `Field`s inside `Containers` can contain instances of other
@@ -119,6 +120,7 @@ class Container(metaclass=ContainerMeta):
     `meta` attribute, which is a `dict` of keywords to values.
 
     """
+
     def __init__(self, **fields):
         self.meta = {}
         # __slots__ cannot be provided with defaults
@@ -164,6 +166,8 @@ def as_dict(self, recursive=False, flatten=False, add_prefix=False):
         flatten: type
             return a flat dictionary, with any sub-field keys generated
             by appending the sub-Container name.
+        add_prefix: bool
+            include the container's prefix in the name of each item
         """
         if not recursive:
             return dict(self.items(add_prefix=add_prefix))

ctapipe/core/logging.py

@@ -1,3 +1,5 @@
+""" helpers for better logging """
+
 import logging
 
 
@@ -14,17 +16,19 @@ def format(self, record):
         reset_seq = "\033[0m"
         color_seq = "\033[1;%dm"
         colors = {
-            'WARNING': yellow,
             'INFO': green,
             'DEBUG': blue,
-            'CRITICAL': yellow,
+            'WARNING': yellow,
+            'CRITICAL': magenta,
             'ERROR': red
         }
 
         levelname = record.levelname
         if levelname in colors:
-            levelname_color = color_seq % (30 + colors[levelname]) \
+            levelname_color = (
+                color_seq % (30 + colors[levelname])
                 + levelname + reset_seq
+            )
             record.levelname = levelname_color
 
         if record.levelno >= self.highlevel_limit:

ctapipe/core/plugins.py

@@ -1,9 +1,11 @@
+""" Functions for dealing with IO plugins """
+
 import importlib
 import pkgutil
 
 
 def detect_and_import_plugins(prefix):
-    ''' detect and import  plugin modules with given prefix, '''
+    """ detect and import  plugin modules with given prefix, """
     return {
         name: importlib.import_module(name)
         for finder, name, ispkg

ctapipe/core/provenance.py

@@ -12,12 +12,12 @@
 import sys
 import uuid
 from contextlib import contextmanager
-from os.path import abspath
 from importlib import import_module
-from pkg_resources import get_distribution
+from os.path import abspath
 
 import psutil
 from astropy.time import Time
+from pkg_resources import get_distribution
 
 import ctapipe
 from .support import Singleton
@@ -157,7 +157,13 @@ def provenance(self):
     def as_json(self, **kwargs):
         """ return all finished provenance as JSON.  Kwargs for `json.dumps`
         may be included, e.g. `indent=4`"""
-        return json.dumps(self.provenance, **kwargs)
+
+        def set_default(obj):
+            """ handle sets (not part of JSON) by converting to list"""
+            if isinstance(obj, set):
+                return list(obj)
+
+        return json.dumps(self.provenance, default=set_default, **kwargs)
 
     @property
     def active_activity_names(self):

ctapipe/core/tests/__init__.py

@@ -0,0 +1,3 @@
+"""
+tests of core functionality of ctapipe
+"""