Prefix containers

ctapipe/core/container.py

@@ -46,11 +46,11 @@ class ContainerMeta(type):
     and no new fields can be added to a container by accident.
     '''
     def __new__(cls, name, bases, dct):
-        items = [
+        field_names = [
             k for k, v in dct.items()
             if isinstance(v, Field)
         ]
-        dct['__slots__'] = tuple(items + ['meta'])
+        dct['__slots__'] = tuple(field_names + ['meta', 'prefix'])
         dct['fields'] = {}
 
         # inherit fields from baseclasses
@@ -59,10 +59,16 @@ def __new__(cls, name, bases, dct):
                 for k, v in b.fields.items():
                     dct['fields'][k] = v
 
-        for k in items:
+        for k in field_names:
             dct['fields'][k] = dct.pop(k)
 
-        return type.__new__(cls, name, bases, dct)
+        new_cls = type.__new__(cls, name, bases, dct)
+
+        # if prefix was not set as a class variable, build a default one
+        if 'container_prefix' not in dct:
+            new_cls.container_prefix = name.lower().replace('container', '')
+
+        return new_cls
 
 
 class Container(metaclass=ContainerMeta):
@@ -72,7 +78,7 @@ class Container(metaclass=ContainerMeta):
     The purpose of this class is to provide a flexible data structure
     that works a bit like a dict or blank Python class, but prevents
     the user from accessing members that have not been defined a
-    priori (more like a C struct), and also keeps metdata information
+    priori (more like a C struct), and also keeps metadata information
     such as a description, defaults, and units for each item in the
     container.
 
@@ -114,8 +120,12 @@ class Container(metaclass=ContainerMeta):
 
     """
     def __init__(self, **fields):
-
         self.meta = {}
+        # __slots__ cannot be provided with defaults
+        # via class variables, so we use a `__prefix` class variable
+        # and a `_prefix` in `__slots__` together with a property.
+        self.prefix = self.container_prefix
+
         for k, v in self.fields.items():
             setattr(self, k, deepcopy(v.default))
 
@@ -128,9 +138,12 @@ def __getitem__(self, key):
     def __setitem__(self, key, value):
         return setattr(self, key, value)
 
-    def items(self):
+    def items(self, add_prefix=False):
         """Generator over (key, value) pairs for the items"""
-        return ((k, getattr(self, k)) for k in self.fields.keys())
+        if not add_prefix or self.prefix == '':
+            return ((k, getattr(self, k)) for k in self.fields.keys())
+
+        return ((self.prefix + '_' + k, getattr(self, k)) for k in self.fields.keys())
 
     def keys(self):
         """Get the keys of the container"""
@@ -140,7 +153,7 @@ def values(self):
         """Get the keys of the container"""
         return (getattr(self, k) for k in self.fields.keys())
 
-    def as_dict(self, recursive=False, flatten=False):
+    def as_dict(self, recursive=False, flatten=False, add_prefix=False):
         """
         convert the `Container` into a dictionary
 
@@ -153,32 +166,27 @@ def as_dict(self, recursive=False, flatten=False):
             by appending the sub-Container name.
         """
         if not recursive:
-            return dict(self.items())
+            return dict(self.items(add_prefix=add_prefix))
         else:
             d = dict()
-            for key, val in self.items():
-                if key.startswith("_"):
-                    continue
+            for key, val in self.items(add_prefix=add_prefix):
                 if isinstance(val, Container) or isinstance(val, Map):
                     if flatten:
-                        d.update({"{}_{}".format(key, k): v
-                                  for k, v in val.as_dict(recursive).items()})
+                        d.update({
+                            "{}_{}".format(key, k): v
+                            for k, v in val.as_dict(
+                                recursive,
+                                add_prefix=add_prefix
+                            ).items()
+                        })
                     else:
-                        d[key] = val.as_dict(recursive=recursive,
-                                             flatten=flatten)
-                    continue
-                d[key] = val
+                        d[key] = val.as_dict(
+                            recursive=recursive, flatten=flatten, add_prefix=add_prefix
+                        )
+                else:
+                    d[key] = val
             return d
 
-    @classmethod
-    def disable_attribute_check(cls):
-        """
-        Globally turn off attribute checking for all Containers,
-        which provides a ~5-10x speed up for setting attributes.
-        This may be used e.g. after code is tested to speed up operation.
-        """
-        cls.__setattr__ = object.__setattr__
-
     def reset(self, recursive=True):
         """ set all values back to their default values"""
         for name, value in self.fields.items():
@@ -219,19 +227,26 @@ class Map(defaultdict):
     by `tel_id` or algorithm name).
     """
 
-    def as_dict(self, recursive=False, flatten=False):
+    def as_dict(self, recursive=False, flatten=False, add_prefix=False):
         if not recursive:
             return dict(self.items())
         else:
             d = dict()
             for key, val in self.items():
                 if isinstance(val, Container) or isinstance(val, Map):
                     if flatten:
-                        d.update({"{}_{}".format(key, k): v
-                                  for k, v in val.as_dict(recursive).items()})
+                        d.update({
+                            "{}_{}".format(key, k): v
+                            for k, v in val.as_dict(
+                                recursive, add_prefix=add_prefix
+                            ).items()
+                        })
                     else:
-                        d[key] = val.as_dict(recursive=recursive,
-                                             flatten=flatten)
+                        d[key] = val.as_dict(
+                            recursive=recursive,
+                            flatten=flatten,
+                            add_prefix=add_prefix,
+                        )
                     continue
                 d[key] = val
             return d

ctapipe/core/tests/test_container.py

@@ -3,6 +3,41 @@
 from ctapipe.core import Container, Field, Map
 
 
+def test_prefix():
+    class AwesomeContainer(Container):
+        pass
+
+    # make sure the default prefix is class name without container
+    assert AwesomeContainer.container_prefix == 'awesome'
+    assert AwesomeContainer().prefix == 'awesome'
+
+    # make sure we can set the class level prefix at definition time
+    class ReallyAwesomeContainer(Container):
+        container_prefix = 'test'
+
+    assert ReallyAwesomeContainer.container_prefix == 'test'
+    r = ReallyAwesomeContainer()
+    assert r.prefix == 'test'
+
+    ReallyAwesomeContainer.container_prefix = 'test2'
+    # new instance should have the old prefix, old instance
+    # the one it was created with
+    assert ReallyAwesomeContainer().prefix == 'test2'
+    assert r.prefix == 'test'
+
+    # Make sure we can set the class level prefix at runtime
+    ReallyAwesomeContainer.container_prefix = 'foo'
+    assert ReallyAwesomeContainer().prefix == 'foo'
+
+    # make sure we can assign instance level prefixes
+    c1 = ReallyAwesomeContainer()
+    c2 = ReallyAwesomeContainer()
+    c2.prefix = 'c2'
+
+    assert c1.prefix == 'foo'
+    assert c2.prefix == 'c2'
+
+
 def test_inheritance():
 
     class ExampleContainer(Container):

ctapipe/io/containers.py

@@ -706,6 +706,8 @@ class MuonIntensityParameter(Container):
 
 
 class HillasParametersContainer(Container):
+    container_prefix = 'hillas'
+
     intensity = Field(nan, 'total intensity (size)')
 
     x = Field(nan, 'centroid x coordinate')
@@ -725,6 +727,8 @@ class LeakageContainer(Container):
     """
     Leakage
     """
+    container_prefix = ''
+
     leakage1_pixel = Field(
         nan,
         'Percentage of pixels after cleaning'

ctapipe/io/hdf5tableio.py

@@ -60,6 +60,8 @@ class HDF5TableWriter(TableWriter):
     group_name: str
         name of group into which to put all of the tables generated by this
         Writer (it will be placed under "/" in the file)
+    add_prefix: bool
+        if True, add the container prefix before each column name
     mode : str ('w', 'a')
         'w' if you want to overwrite the file
         'a' if you want to append data to the file
@@ -72,9 +74,17 @@ class HDF5TableWriter(TableWriter):
 
     """
 
-    def __init__(self, filename, group_name, mode='w', root_uep='/', **kwargs):
-
-        super().__init__()
+    def __init__(
+        self,
+        filename,
+        group_name,
+        add_prefix=False,
+        mode='w',
+        root_uep='/',
+        **kwargs
+    ):
+
+        super().__init__(add_prefix=add_prefix)
         self._schemas = {}
         self._tables = {}
 
@@ -129,18 +139,24 @@ class Schema(tables.IsDescription):
 
         # create pytables schema description for the given container
         for container in containers:
-            for col_name, value in container.items():
+            for col_name, value in container.items(add_prefix=self.add_prefix):
 
                 typename = ""
                 shape = 1
 
                 if self._is_column_excluded(table_name, col_name):
-                    self.log.debug("excluded column: %s/%s",
-                                   table_name, col_name)
+                    self.log.debug(
+                        "excluded column: %s/%s", table_name, col_name
+                    )
                     continue
 
                 if isinstance(value, Quantity):
-                    req_unit = container.fields[col_name].unit
+                    if self.add_prefix:
+                        key = col_name.replace(container.prefix + '_', '')
+                    else:
+                        key = col_name
+                    req_unit = container.fields[key].unit
+
                     if req_unit is not None:
                         tr = partial(tr_convert_and_strip_unit, unit=req_unit)
                         meta['{}_UNIT'.format(col_name)] = str(req_unit)
@@ -207,14 +223,17 @@ def _append_row(self, table_name, containers):
         row = table.row
 
         for container in containers:
-            for colname in filter(lambda c: c in table.colnames,
-                                  container.keys()):
+            selected_fields = filter(
+                lambda kv: kv[0] in table.colnames,
+                container.items(add_prefix=self.add_prefix)
+            )
+            for colname, value in selected_fields:
+
                 value = self._apply_col_transform(
-                    table_name, colname, container[colname]
+                    table_name, colname, value
                 )
 
                 row[colname] = value
-
         row.append()
 
     def write(self, table_name, containers):

ctapipe/io/tableio.py

@@ -18,17 +18,17 @@ class TableWriter(Component, metaclass=ABCMeta):
     - `ctapipe.io.HDF5TableWriter`
     """
 
-    def __init__(self, parent=None, **kwargs):
+    def __init__(self, parent=None, add_prefix=False, **kwargs):
         super().__init__(parent, **kwargs)
         self._transforms = defaultdict(dict)
         self._exclusions = defaultdict(list)
+        self.add_prefix = add_prefix
 
     def __enter__(self):
 
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-
         self.close()
 
     def exclude(self, table_name, pattern):