Merge pull request #128 from arroyoj/optional_template

Optional template to allow a value to be null, missing or validated by another template

confuse/templates.py

@@ -70,10 +70,14 @@ def value(self, view, template=None):
         return self.get_default_value(view.name)
 
     def get_default_value(self, key_name='default'):
-        if self.default is REQUIRED:
-            # Missing required value. This is an error.
+        """Get the default value to return when the value is missing.
+
+        May raise a `NotFoundError` if the value is required.
+        """
+        if not hasattr(self, 'default') or self.default is REQUIRED:
+            # The value is required. A missing value is an error.
             raise exceptions.NotFoundError(u"{} not found".format(key_name))
-        # Missing value, but not required.
+        # The value is not required.
         return self.default
 
     def convert(self, value, view):
@@ -594,6 +598,50 @@ def value(self, view, template=None):
         return pathlib.Path(value)
 
 
+class Optional(Template):
+    """A template that makes a subtemplate optional.
+
+    If the value is present and not null, it must validate against the
+    subtemplate. However, if the value is null or missing, the template will
+    still validate, returning a default value. If `allow_missing` is False,
+    the template will not allow missing values while still permitting null.
+    """
+
+    def __init__(self, subtemplate, default=None, allow_missing=True):
+        self.subtemplate = as_template(subtemplate)
+        if default is None:
+            # When no default is passed, try to use the subtemplate's
+            # default value as the default for this template
+            try:
+                default = self.subtemplate.get_default_value()
+            except exceptions.NotFoundError:
+                pass
+        self.default = default
+        self.allow_missing = allow_missing
+
+    def value(self, view, template=None):
+        try:
+            value, _ = view.first()
+        except exceptions.NotFoundError:
+            if self.allow_missing:
+                # Value is missing but not required
+                return self.default
+            # Value must be present even though it can be null. Raise an error.
+            raise exceptions.NotFoundError(u'{} not found'.format(view.name))
+
+        if value is None:
+            # None (ie, null) is always a valid value
+            return self.default
+        return self.subtemplate.value(view, self)
+
+    def __repr__(self):
+        return 'Optional({0}, {1}, allow_missing={2})'.format(
+            repr(self.subtemplate),
+            repr(self.default),
+            self.allow_missing,
+        )
+
+
 class TypeTemplate(Template):
     """A simple template that checks that a value is an instance of a
     desired Python type.

docs/examples.rst

@@ -118,3 +118,174 @@ provided to ``MappingValues``, then an error will be raised:
 ...     print(err)
 ...
 categories.no_description.description not found
+
+
+Optional
+--------
+
+While many templates like ``Integer`` and ``String`` can be configured to
+return a default value if the requested view is missing, validation with these
+templates will fail if the value is left blank in the YAML file or explicitly
+set to ``null`` in YAML (ie, ``None`` in python). The ``Optional`` template
+can be used with other templates to allow its subtemplate to accept ``null``
+as valid and return a default value. The default behavior of ``Optional``
+allows the requested view to be missing, but this behavior can be changed by
+passing ``allow_missing=False``, in which case the view must be present but its
+value can still be ``null``. In all cases, any value other than ``null`` will
+be passed to the subtemplate for validation, and an appropriate ``ConfigError``
+will be raised if validation fails. ``Optional`` can also be used with more
+complex templates like ``MappingTemplate`` to make entire sections of the
+configuration optional.
+
+Consider a configuration where ``log`` can be set to a filename to enable
+logging to that file or set to ``null`` or not included in the configuration to
+indicate logging to the console. All of the following are valid configurations
+using the ``Optional`` template with ``Filename`` as the subtemplate:
+
+>>> import sys
+>>> import confuse
+>>> def get_log_output(config):
+...     output = config['log'].get(confuse.Optional(confuse.Filename()))
+...     if output is None:
+...         return sys.stderr
+...     return output
+...
+>>> config = confuse.RootView([])
+>>> config.set({'log': '/tmp/log.txt'})  # `log` set to a filename
+>>> get_log_output(config)
+'/tmp/log.txt'
+>>> config.set({'log': None})  # `log` set to None (ie, null in YAML)
+>>> get_log_output(config)
+<_io.TextIOWrapper name='<stderr>' mode='w' encoding='UTF-8'>
+>>> config.clear()  # Clear config so that `log` is missing
+>>> get_log_output(config)
+<_io.TextIOWrapper name='<stderr>' mode='w' encoding='UTF-8'>
+
+However, validation will still fail with ``Optional`` if a value is given that
+is invalid for the subtemplate:
+
+>>> config.set({'log': True})
+>>> try:
+...     get_log_output(config)
+... except confuse.ConfigError as err:
+...     print(err)
+...
+log: must be a filename, not bool
+
+And without wrapping the ``Filename`` subtemplate in ``Optional``, ``null``
+values are not valid:
+
+>>> config.set({'log': None})
+>>> try:
+...     config['log'].get(confuse.Filename())
+... except confuse.ConfigError as err:
+...     print(err)
+...
+log: must be a filename, not NoneType
+
+If a program wants to require an item to be present in the configuration, while
+still allowing ``null`` to be valid, pass ``allow_missing=False`` when
+creating the ``Optional`` template:
+
+>>> def get_log_output_no_missing(config):
+...     output = config['log'].get(confuse.Optional(confuse.Filename(),
+...                                                 allow_missing=False))
+...     if output is None:
+...         return sys.stderr
+...     return output
+...
+>>> config.set({'log': None})  # `log` set to None is still OK...
+>>> get_log_output_no_missing(config)
+<_io.TextIOWrapper name='<stderr>' mode='w' encoding='UTF-8'>
+>>> config.clear()  # but `log` missing now raises an error
+>>> try:
+...     get_log_output_no_missing(config)
+... except confuse.ConfigError as err:
+...     print(err)
+...
+log not found
+
+The default value returned by ``Optional`` can be set explicitly by passing a
+value to its ``default`` parameter. However, if no explicit default is passed
+to ``Optional`` and the subtemplate has a default value defined, then
+``Optional`` will return the subtemplate's default value. For subtemplates that
+do not define default values, like ``MappingTemplate``, ``None`` will be
+returned as the default unless an explicit default is provided.
+
+In the following example, ``Optional`` is used to make an ``Integer`` template
+more lenient, allowing blank values to validate. In addition, the entire
+``extra_config`` block can be left out without causing validation errors. If
+we have a file named ``optional.yaml`` with the following contents:
+
+.. code-block:: yaml
+
+    favorite_number: # No favorite number provided, but that's OK
+    # This part of the configuration is optional. Uncomment to include.
+    # extra_config:
+    #   fruit: apple
+    #   number: 10
+
+Then the configuration can be validated as follows:
+
+>>> import confuse
+>>> source = confuse.YamlSource('optional.yaml')
+>>> config = confuse.RootView([source])
+>>> # The following `Optional` templates are all equivalent
+... config['favorite_number'].get(confuse.Optional(5))
+5
+>>> config['favorite_number'].get(confuse.Optional(confuse.Integer(5)))
+5
+>>> config['favorite_number'].get(confuse.Optional(int, default=5))
+5
+>>> # But a default passed to `Optional` takes precedence and can be any type
+... config['favorite_number'].get(confuse.Optional(5, default='five'))
+'five'
+>>> # `Optional` with `MappingTemplate` returns `None` by default
+... extra_config = config['extra_config'].get(confuse.Optional(
+...     {'fruit': str, 'number': int},
+... ))
+>>> print(extra_config is None)
+True
+>>> # But any default value can be provided, like an empty dict...
+... config['extra_config'].get(confuse.Optional(
+...     {'fruit': str, 'number': int},
+...     default={},
+... ))
+{}
+>>> # or a dict with default values
+... config['extra_config'].get(confuse.Optional(
+...     {'fruit': str, 'number': int},
+...     default={'fruit': 'orange', 'number': 3},
+... ))
+{'fruit': 'orange', 'number': 3}
+
+Without the ``Optional`` template wrapping the ``Integer``, the blank value
+in the YAML file will cause an error:
+
+>>> try:
+...     config['favorite_number'].get(5)
+... except confuse.ConfigError as err:
+...     print(err)
+...
+favorite_number: must be a number
+
+If the ``extra_config`` for this example configuration is supplied, it must
+still match the subtemplate. Therefore, this will fail:
+
+>>> config.set({'extra_config': {}})
+>>> try:
+...     config['extra_config'].get(confuse.Optional(
+...         {'fruit': str, 'number': int},
+...     ))
+... except confuse.ConfigError as err:
+...     print(err)
+...
+extra_config.fruit not found
+
+But this override of the example configuration will validate:
+
+>>> config.set({'extra_config': {'fruit': 'banana', 'number': 1}})
+>>> config['extra_config'].get(confuse.Optional(
+...     {'fruit': str, 'number': int},
+... ))
+{'fruit': 'banana', 'number': 1}

test/test_valid.py

@@ -559,3 +559,111 @@ def test_missing(self):
         config = _root({'foo': {'one': 1, 'two': 2, 'three': 3}})
         valid = config['bar'].get(confuse.MappingValues(int))
         self.assertEqual(valid, {})
+
+
+class OptionalTest(unittest.TestCase):
+    def test_optional_string_valid_type(self):
+        config = _root({'foo': 'bar'})
+        valid = config['foo'].get(confuse.Optional(confuse.String()))
+        self.assertEqual(valid, 'bar')
+
+    def test_optional_string_invalid_type(self):
+        config = _root({'foo': 5})
+        with self.assertRaises(confuse.ConfigTypeError):
+            config['foo'].get(confuse.Optional(confuse.String()))
+
+    def test_optional_string_null(self):
+        config = _root({'foo': None})
+        valid = config['foo'].get(confuse.Optional(confuse.String()))
+        self.assertIsNone(valid)
+
+    def test_optional_string_null_default_value(self):
+        config = _root({'foo': None})
+        valid = config['foo'].get(confuse.Optional(confuse.String(), 'baz'))
+        self.assertEqual(valid, 'baz')
+
+    def test_optional_string_null_string_provides_default(self):
+        config = _root({'foo': None})
+        valid = config['foo'].get(confuse.Optional(confuse.String('baz')))
+        self.assertEqual(valid, 'baz')
+
+    def test_optional_string_null_string_default_override(self):
+        config = _root({'foo': None})
+        valid = config['foo'].get(confuse.Optional(confuse.String('baz'),
+                                                   default='bar'))
+        self.assertEqual(valid, 'bar')
+
+    def test_optional_string_allow_missing_no_explicit_default(self):
+        config = _root({})
+        valid = config['foo'].get(confuse.Optional(confuse.String()))
+        self.assertIsNone(valid)
+
+    def test_optional_string_allow_missing_default_value(self):
+        config = _root({})
+        valid = config['foo'].get(confuse.Optional(confuse.String(), 'baz'))
+        self.assertEqual(valid, 'baz')
+
+    def test_optional_string_missing_not_allowed(self):
+        config = _root({})
+        with self.assertRaises(confuse.NotFoundError):
+            config['foo'].get(
+                confuse.Optional(confuse.String(), allow_missing=False)
+            )
+
+    def test_optional_string_null_missing_not_allowed(self):
+        config = _root({'foo': None})
+        valid = config['foo'].get(
+            confuse.Optional(confuse.String(), allow_missing=False)
+        )
+        self.assertIsNone(valid)
+
+    def test_optional_mapping_template_valid(self):
+        config = _root({'foo': {'bar': 5, 'baz': 'bak'}})
+        template = {'bar': confuse.Integer(), 'baz': confuse.String()}
+        valid = config.get({'foo': confuse.Optional(template)})
+        self.assertEqual(valid['foo']['bar'], 5)
+        self.assertEqual(valid['foo']['baz'], 'bak')
+
+    def test_optional_mapping_template_invalid(self):
+        config = _root({'foo': {'bar': 5, 'baz': 10}})
+        template = {'bar': confuse.Integer(), 'baz': confuse.String()}
+        with self.assertRaises(confuse.ConfigTypeError):
+            config.get({'foo': confuse.Optional(template)})
+
+    def test_optional_mapping_template_null(self):
+        config = _root({'foo': None})
+        template = {'bar': confuse.Integer(), 'baz': confuse.String()}
+        valid = config.get({'foo': confuse.Optional(template)})
+        self.assertIsNone(valid['foo'])
+
+    def test_optional_mapping_template_null_default_value(self):
+        config = _root({'foo': None})
+        template = {'bar': confuse.Integer(), 'baz': confuse.String()}
+        valid = config.get({'foo': confuse.Optional(template, {})})
+        self.assertIsInstance(valid['foo'], dict)
+
+    def test_optional_mapping_template_allow_missing_no_explicit_default(self):
+        config = _root({})
+        template = {'bar': confuse.Integer(), 'baz': confuse.String()}
+        valid = config.get({'foo': confuse.Optional(template)})
+        self.assertIsNone(valid['foo'])
+
+    def test_optional_mapping_template_allow_missing_default_value(self):
+        config = _root({})
+        template = {'bar': confuse.Integer(), 'baz': confuse.String()}
+        valid = config.get({'foo': confuse.Optional(template, {})})
+        self.assertIsInstance(valid['foo'], dict)
+
+    def test_optional_mapping_template_missing_not_allowed(self):
+        config = _root({})
+        template = {'bar': confuse.Integer(), 'baz': confuse.String()}
+        with self.assertRaises(confuse.NotFoundError):
+            config.get({'foo': confuse.Optional(template,
+                                                allow_missing=False)})
+
+    def test_optional_mapping_template_null_missing_not_allowed(self):
+        config = _root({'foo': None})
+        template = {'bar': confuse.Integer(), 'baz': confuse.String()}
+        valid = config.get({'foo': confuse.Optional(template,
+                                                    allow_missing=False)})
+        self.assertIsNone(valid['foo'])