Add ability to use a secrets.ini file + jinja templating to apply secrets in cli apps

docs/prosper_cli.rst

@@ -71,7 +71,8 @@ Using prosper_cli
 
 Meant to be used with `ProsperCookiecutters`_ for debug launching Flask apps.  
 
------
+CLI Features
+------------
 
 By using the Prosper framework, the following is handled automatically:
 
@@ -80,11 +81,25 @@ By using the Prosper framework, the following is handled automatically:
 - ``-v``/``--verbose`` bool for enabling STDOUT logging
 - ``--config`` for loading a custom config file
 - ``--dump-config`` for dumping default config to STDOUT
+- ``--secret-cfg`` for using a jinja2 template secret-keeping style
 - ``self.logger`` and ``self.config`` loaded automagically
 - Full ``ProsperLogging`` support
     - Slack and Discord support if webhooks are provided by config
     - Standardized log formatting
     - Platform and version information for webhook loggers
 
+Secret Config
+~~~~~~~~~~~~~
+
+Sometimes having a secrets file is preferrable to using environment variables.  This allows secrets to be more easily passed as keys.
+
+.. code-block:: cfg
+
+# credentials.ini
+[key]
+    value_1 = secret
+    value_2 = secret
+
+
 .. _Plumbum: http://plumbum.readthedocs.io/en/latest/cli.html
 .. _ProsperCookiecutters: https://github.com/EVEprosper/ProsperCookiecutters

prosper/common/common_config.cfg

@@ -15,10 +15,10 @@
 [TEST]
     request_logname = urllib3.connectionpool
     request_POST_endpoint = https://discordapp.com:443 "POST /api/webhooks/{serverid}/{api_key} HTTP/1.1" 204 0
-    request_new_connection = Starting new HTTPS connection (1): discordapp.com
-    slack_new_connection = Starting new HTTPS connection (1): hooks.slack.com
+    request_new_connection = Starting new HTTPS connection (1): discordapp.com:443
+    slack_new_connection = Starting new HTTPS connection (1): hooks.slack.com:443
     slack_POST_endpoint = https://hooks.slack.com:443 "POST {webhook_info} HTTP/1.1" 200 22
-    hipchat_new_connection = Starting new HTTPS connection (1): {hipchat_hostname}
+    hipchat_new_connection = Starting new HTTPS connection (1): {hipchat_hostname}:443
     hipchat_POST_endpoint = {hipchat_hostname}{hipchat_port} "POST {webhook_info} HTTP/1.1" 204 0
     hipchat_hostname = #SECRET
     hipchat_port = :443

prosper/common/prosper_cli.py

@@ -47,7 +47,16 @@ def dump_config(self):  # pragma: no cover
             base_config = cfg_fh.read()
 
         print(base_config)
-        exit()
+        exit(2)
+
+    @cli.switch(
+        ['--secret-cfg'],
+        cli.ExistingFile,
+        help='Secrets .ini file for jinja2 template'
+    )
+    def load_secrets(self, secret_path):
+        """render secrets into config object"""
+        self._config = p_config.render_secrets(self.config_path, secret_path)
 
     _logger = None
     @property

prosper/common/prosper_config.py

@@ -7,9 +7,43 @@
 from os import path, getenv
 import configparser
 from configparser import ExtendedInterpolation
-import warnings
 import logging
 
+import anyconfig
+import anytemplate
+
+
+def render_secrets(
+        config_path,
+        secret_path,
+):
+    """combine a jinja template with a secret .ini file
+
+    Args:
+        config_path (str): path to .cfg file with jinja templating
+        secret_path (str): path to .ini-like secrets file
+
+    Returns:
+        ProsperConfig: rendered configuration object
+
+    """
+    with open(secret_path, 'r') as s_fh:
+        secret_ini = anyconfig.load(s_fh, ac_parser='ini')
+
+    with open(config_path, 'r') as c_fh:
+        raw_cfg = c_fh.read()
+
+    rendered_cfg = anytemplate.renders(raw_cfg, secret_ini, at_engine='jinja2')
+
+    p_config = ProsperConfig(config_path)
+    local_config = configparser.ConfigParser()
+    local_config.optionxform = str
+    local_config.read_string(rendered_cfg)
+
+    p_config.local_config = local_config
+
+    return p_config
+
 
 class ProsperConfig(object):
     """configuration handler for all prosper projects
@@ -254,7 +288,8 @@ def get_local_config_filepath(
         str: Path to local config, or global if path DNE
 
     """
-    local_config_filepath = config_filepath.replace('.cfg', '_local.cfg')
+    local_config_name = path.basename(config_filepath).split('.')[0] + '_local.cfg'
+    local_config_filepath = path.join(path.split(config_filepath)[0], local_config_name)
 
     real_config_filepath = ''
     if path.isfile(local_config_filepath) or force_local:

setup.py

@@ -14,10 +14,10 @@ def get_version(package_name):
     """find __version__ for making package
 
     Args:
-        package_path (str): path to _version.py folder (abspath > relpath)
+        package_name (str): path to _version.py folder (abspath > relpath)
 
     Returns:
-        (str) __version__ value
+        str: __version__ value
 
     """
     module = 'prosper.' + package_name + '._version'
@@ -139,9 +139,12 @@ def initialize_options(self):
         ],
     },
     install_requires=[
+        'anyconfig',
+        'anytemplate',
+        'jinja2',
+        'plumbum',
         'requests',
         'semantic_version',
-        'plumbum',
     ],
     tests_require=[
         'pytest>=3.3.0',

tests/test_config.cfg.j2

@@ -0,0 +1,6 @@
+[TEST]
+    secret = {{test.secret}}
+    key1 = vals
+    key2 = 100
+    key3 =
+    nokey =

tests/test_prosper_cli.py

@@ -169,7 +169,7 @@ def test_help(self):
 
     def test_dump_config(self):
         """validate --dump-config works as expected"""
-        result = self.cli('--dump-config')
+        retcode, result, stderr = self.cli.run(['--dump-config'], retcode=2)
 
         config = configparser.ConfigParser()
         config.read_string(result)

tests/test_prosper_config.py

@@ -29,6 +29,18 @@ def test_setup_environment():
     expected_val = prosper_config.get_value_from_environment('TEST', 'dummy_val')
     assert expected_val == ENV_TEST_1
 
+def test_render_secrets():
+    """happypath test for p_config.render_secrets"""
+    config_path = path.join(HERE, 'test_config.cfg.j2')
+    secret_path = path.join(HERE, 'test_secret.ini')
+
+    config = prosper_config.render_secrets(config_path, secret_path)
+
+    assert isinstance(config, prosper_config.ProsperConfig)
+    assert config.get_option('TEST', 'secret') == 'butts'
+    os.environ['PROSPER_LOCAL__hello'] = 'world'
+    assert config.get_option('LOCAL', 'hello') == 'world'
+
 TEST_BAD_CONFIG_PATH = path.join(HERE, 'bad_config.cfg')
 TEST_BAD_PATH = path.join(HERE, 'no_file_here.cfg')
 def test_bad_config():

tests/test_secret.ini

@@ -0,0 +1,2 @@
+[test]
+    secret = butts