Merge pull request #2 from dirac-institute/config

Add a more flexible and secure authentication and security critical configuration handling.

requirements.txt

@@ -1,5 +1,6 @@
 asgiref==3.3.1
 astropy==4.2
+boto3==1.17.59
 certifi==2020.12.5
 click==7.1.2
 cycler==0.10.0
@@ -11,6 +12,7 @@ Jinja2==2.11.3
 kiwisolver==1.3.1
 MarkupSafe==1.1.1
 matplotlib==3.3.4
+moto==2.0.5
 numpy==1.20.1
 Pillow==8.1.2
 pyerfa==1.7.2

trail/trail/config.py

@@ -0,0 +1,186 @@
+import os
+import stat
+
+import yaml
+import boto3
+
+
+__all__ = ["CONF_FILE_ENVVAR", "CONF_FILE_PATH", "Config"]
+
+
+CONF_FILE_ENVVAR = "TARILBLAZER_CONFIG"
+"""Default name of the environmental variable that contains the path to the
+configuration file. When the env var does not exist, configuration is assumed
+to exist at its default location (see ``CONF_FILE_PATH``)."""
+
+CONF_FILE_PATH = "~/.trail/secrets.yaml"
+"""Default path at which it is expected the config file can be found. Will be
+ignored if ``CONF_FILE_ENVVAR`` env var exists."""
+
+
+class Config():
+    """Represents a general YAML configuration file, with keys being mapped to
+    attributes. Optionally resolving existing secrets via AWS Secrets Manager.
+
+    Parameters
+    ----------
+    confDict : `dict`
+        Dictionary whose keys will be mapped to attributes of the class.
+    useAwsSecrets : `bool`, optional
+        Resolve secrets using AWS Secrets manager. False by default.
+    awsRegion : `str`, optional
+        Region of the secret manager to use. Default: `us-west-2`.
+
+    Notes
+    -----
+    Secrets Manager can and will support any kind of string as a secret. For
+    RDS it will tests showed that secrets are stored as a JSON key-value string
+    pairs (i.e. output looks like a ``str(dict)``). This presents 3 different
+    scenarios when keys get resolved and set as Config attributes:
+    1) resolve a secret key into multiple keys and insert them, replacing the
+       secret key with the recieved key-value pairs;
+    2) resolve a secret and insert under original key, when returned  secrets
+       are simple strings so the name of the secret is replaced with the secret
+       itself;
+    3) and insert a key-value pair named in the YAML config file.
+    """
+
+    configKey = "*"
+    """Key which is read to create a config, the value `*` selects all keys."""
+
+    secretsKeys = []
+    """Specifies which keys are to be resolved as secrets."""
+
+    def __init__(self, confDict, useAwsSecrets=False, awsRegion="us-west-2"):
+        self._keys = []
+        self._subConfs = []
+        self._recurseDownDicts(confDict, useAwsSecrets, awsRegion=awsRegion)
+
+    def _recurseDownDicts(self, confDict, useAwsSecrets, awsRegion):
+        """Recursively walks the dictionary keys and values and maps keys to
+        instance attributes, resolving any existing secrets along the way.
+
+        Parameters
+        ----------
+        confDict : `dict`
+            Dictionary whose keys will be mapped to attributes of the class.
+        useAwsSecrets : `bool`, optional
+            Resolve secrets using AWS Secrets manager. False by default.
+        awsRegion : `str`, optional
+            Region of the secret manager to use. Default: `us-west-2`.
+        """
+        if self.configKey != "*":
+            if self.configKey not in confDict:
+                raise ValueError(f"Required config key {self.configKey} does "
+                                 "not exist in the config dictionary.")
+            confDict = confDict[self.configKey]
+
+        # if a region is set in the config use it, otherwise use the default
+        region = confDict.get("aws-region", awsRegion)
+
+        for key, val in confDict.items():
+            if isinstance(val, dict):
+                self._subConfs.append(key)
+                setattr(self, key, Config(val))
+            else:
+                # of course this is now ugly...
+                if useAwsSecrets and key in self.secretsKeys:
+                    secrets = self._parseAwsSecrets(val, region)
+                    if isinstance(secrets, dict):
+                        # scenario 1, replacing key with many
+                        for secretkey, secretval in secrets.items():
+                            if secretkey not in self._keys:
+                                self._keys.append(secretkey)
+                            setattr(self, secretkey, secretval)
+                        # skip inserting the replaced key 
+                        continue
+                    else:
+                        # scenario 2, resolve simple secret as key
+                        val = secrets
+                # scenario 2 or 3, insert key-value pair, resolving secrets
+                self._keys.append(key)
+                setattr(self, key, val)
+
+    @staticmethod
+    def _parseAwsSecrets(name, region):
+        smClient = boto3.client("secretsmanager", region_name=region)
+        secretString = smClient.get_secret_value(SecretId=name)["SecretString"]
+        # JSON is like YAML, right?
+        return yaml.safe_load(secretString)
+
+
+    def __repr__(self):
+        reprStr = f"{self.__class__.__name__}("
+
+        for key in self._subConfs:
+            reprStr += f"{key}={getattr(self, key)}, "
+
+        for key in self._keys:
+            reprStr += key + ", "
+        reprStr = reprStr[:-2]
+
+        return reprStr+")"
+
+
+    def __eq__(self, other):
+        equal = True
+
+        for key, subConf in zip(self._keys, self._subConfs):
+            try:
+                equal = equal and getattr(self, key) == getattr(other, key)
+                equal = equal and getattr(self, subConf) == getattr(other, subConf)
+            except AttributeError:
+                # other does not have a key, but self has - not equal
+                # or other does not have a subConf, but self has - not equal
+                return False
+
+        return equal
+
+    @classmethod
+    def fromYaml(cls, filePath=None, useAwsSecrets=False, awsRegion="us-west-2"):
+        """Create a new Config instance from a YAML file. By default will
+        look at location pointed to by the environmental variable named by
+        `CONF_FILE_ENVVAR`. If the env var is not set it will default to
+        location set by `CONF_FILE_PATH`.
+
+        Parameters
+        ----------
+        filePath : `str` or `None`, Optional
+            A file path to the YAML configuration. When not specified, first
+            the ``CONF_FILE_ENVVAR`` is used. If it doesn't exist the
+            ``CONF_FILE_PATH`` is used.
+        useAwsSecrets : `bool`, optional
+            Resolve secrets using AWS Secrets manager. False by default.
+        awsRegion : `str`, optional
+            Region of the secret manager to use. Default: `us-west-2`.
+        """
+        # resolve config file path
+        if filePath is None:
+            if CONF_FILE_ENVVAR in os.environ:
+                filePath = os.path.expanduser(os.environ[CONF_FILE_ENVVAR])
+            else:
+                filePath = os.path.expanduser(CONF_FILE_PATH)
+
+        # make sure file exists and its permissions are at 600 or more
+        if not os.path.isfile(filePath):
+            raise FileNotFoundError(f"No configuration file found: {filePath}")
+
+        mode = os.stat(filePath).st_mode
+        if mode & (stat.S_IRWXG | stat.S_IRWXO) != 0:
+            raise PermissionError(f"Configuration file {filePath} has "
+                                  f"incorrect permissions: {mode:o}")
+
+        with open(filePath, 'r') as stream:
+            confDict = yaml.safe_load(stream)
+
+        return cls(confDict, useAwsSecrets, awsRegion)
+
+
+class DbAuth(Config):
+    configKey = 'db'
+    secretsKeys = ["secret_name", ]
+
+
+class SiteConfig(Config):
+    configKey = 'settings'
+    secretsKeys = ["secret_key", ]

trail/trail/settings.py

@@ -13,6 +13,11 @@
 from pathlib import Path
 import os
 
+from .config import DbAuth, SiteConfig
+
+siteConfig = SiteConfig.fromYaml()
+dbConfig = DbAuth.fromYaml()
+
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent
 
@@ -24,14 +29,7 @@
 # See https://docs.djangoproject.com/en/3.1/howto/deployment/checklist/
 
 # SECURITY WARNING: keep the secret key used in production secret!
-home = str(Path.home())
-try:
-    secret_key_file = os.path.join(home, '.trail/secret_key.txt')
-    with open(secret_key_file) as f:
-        SECRET_KEY = f.read().strip()
-except FileNotFoundError:
-    print('Unable to find secret key file {secret_key_file}.')
-
+SECRET_KEY = siteConfig.secret_key
 
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = True

trail/trail/tests/config/awsSecretsConf.yaml

@@ -0,0 +1,2 @@
+db:
+  secret_name: "db-secret"

trail/trail/tests/config/conf.yaml

@@ -0,0 +1,11 @@
+settings:
+  secret_key: nonsense
+  static_root: ~/trail/static
+  media_root: ~/trail/media
+db:
+  engine: django.db.backend.postgresql_psycopg2
+  name: dbname
+  user: dbuser
+  password: dbpassword
+  host: dbhost.alala.com
+  port: 5432

trail/trail/tests/test_config.py

@@ -0,0 +1,119 @@
+import os
+
+from django.test import TestCase
+from moto import mock_secretsmanager
+import boto3
+import yaml
+
+
+import trail.config as ConfigModule
+from trail.config import Config, DbAuth, SiteConfig
+
+
+TESTDIR = os.path.abspath(os.path.dirname(__file__))
+
+
+class ConfigTestCase(TestCase):
+    testConfigDir = os.path.join(TESTDIR, "config")
+
+    def setUp(self):
+        self.badConf = os.path.join(self.testConfigDir, "badPermissionConf.yaml")
+        self.goodConf = os.path.join(self.testConfigDir, "conf.yaml")
+        self.noExists = os.path.join(self.testConfigDir, "noexist.yaml")
+
+    def tearDown(self):
+        pass
+
+    def testInstantiation(self):
+        # Test 600 permissions
+        with self.assertRaises(PermissionError):
+            Config.fromYaml(self.badConf)
+
+        # Test missing file
+        with self.assertRaises(FileNotFoundError):
+            Config.fromYaml(self.noExists)
+
+        # Test that fromYaml and direct instantiation produce same result
+        # Test it's possible to instantiate without errors, test env var and
+        # global var default instantiations.
+        try:
+            conf1 = Config.fromYaml(self.goodConf)
+        except Exception as e:
+            self.fail(f"ConfigTestCase.testConfig conf1 failed with:\n{e}")
+
+        with open(self.goodConf, 'r') as stream:
+            confDict = yaml.safe_load(stream)
+        try:
+            conf2 = Config(confDict)
+        except Exception as e:
+            self.fail(f"ConfigTestCase.testConfig conf2 failed with:\n{e}")
+
+        self.assertEqual(conf1, conf2)
+
+        ConfigModule.CONF_FILE_PATH = self.goodConf
+        try:
+            conf3 = Config.fromYaml()
+        except Exception as e:
+            self.fail(f"ConfigTestCase.testConfig conf3 failed with:\n{e}")
+
+        self.assertEqual(conf2, conf3)
+
+        # Switch to a different conf file to verify overriding with env var
+        # works as intended
+        os.environ[ConfigModule.CONF_FILE_ENVVAR] = self.badConf
+        with self.assertRaises(PermissionError):
+            conf4 = Config.fromYaml()
+
+    def testConfigKey(self):
+        Config.configKey = "noexists"
+        with self.assertRaises(ValueError):
+            Config.fromYaml(self.goodConf)
+
+        # this is a bit silly I think because it doesn't test correctness?
+        Config.configKey = "db"
+        conf1 = Config.fromYaml(self.goodConf)
+        conf2 = DbAuth.fromYaml(self.goodConf)
+        self.assertEqual(conf1, conf2)
+
+
+class AwsSecretsTestCase(TestCase):
+    testConfigDir = os.path.join(TESTDIR, "config")
+
+    def setUp(self):
+        self.goodConf = os.path.join(self.testConfigDir, "conf.yaml")
+        self.awsSecretsConf = os.path.join(self.testConfigDir, "awsSecretsConf.yaml")
+
+    @mock_secretsmanager
+    def testSimpleAwsSecrets(self):
+        smClient = boto3.client("secretsmanager", region_name="us-west-2")
+        smClient.create_secret(Name="nonsense", SecretString="test-secret-key")
+
+        conf = SiteConfig.fromYaml(self.goodConf, useAwsSecrets=True)
+
+        self.assertEqual(conf.secret_key, "test-secret-key")
+
+    @mock_secretsmanager
+    def testMultiKeyedSecret(self):
+        multiKeyedSecret = {
+            "engine": "postgresql",
+            "name": "dbname",
+            "user": "dbuser",
+            "password": "dbpassword",
+            "host": "dbhost.alala.com",
+            "port": 5432,
+        }
+        smClient = boto3.client("secretsmanager", region_name="us-west-2")
+        smClient.create_secret(Name="db-secret", SecretString=str(multiKeyedSecret))
+
+        conf = DbAuth.fromYaml(self.awsSecretsConf, useAwsSecrets=True)
+
+        # verify the secret_key was expanded
+        for key, val in multiKeyedSecret.items():
+            self.assertEqual(getattr(conf, key), val)
+
+        # verify that the replaced key was not inserted
+        with self.assertRaises(AttributeError):
+            conf.secret_name
+
+
+