Merge pull request #270 from mendix/check_db_config_options

support vcap, database url or custom runtime settings for the database config

README.md

@@ -54,7 +54,12 @@ In our trial we found the service `elephantsql` which offered the free `turtle`
 
     cf bind-service <YOUR_APP> <SERVICE_NAME>
 
-Note that not all databases are automatically picked up by the buildpack. If `cf push` returns an error like `Could not parse database credentials`, you need to set the `DATABASE_URL` variable manually using the details included in the service.
+Note that not all databases are automatically picked up by the buildpack. If `cf push` returns an error like `Could not parse database credentials`, you need to set the `DATABASE_URL` variable manually or set database [Mendix custom runtime variables](https://docs.mendix.com/refguide/custom-settings) to configure a database. Note these variables need to be prefixed with `MXRUNTIME_`, as per example:
+
+    cf set-env <YOUR_APP> MXRUNTIME_DatabaseType PostgreSQL
+    cf set-env <YOUR_APP> MXRUNTIME_DatabaseJdbcUrl postgres://host/databasename
+    cf set-env <YOUR_APP> MXRUNTIME_DatabaseUsername user
+    cf set-env <YOUR_APP> MXRUNTIME_DatabasePassword password
 
 Now we need to push the application once more.
 
@@ -405,8 +410,8 @@ cf restart <YOUR_APP>
 Contributing
 ====
 
-Make sure your code complies with PEP8. Make sure your code is styled using [Black](https://github.com/psf/black). 
-We enforce this using `flake8` and `black` in our travis CI. 
+Make sure your code complies with PEP8. Make sure your code is styled using [Black](https://github.com/psf/black).
+We enforce this using `flake8` and `black` in our travis CI.
 
 This simplest way to use these tools is by installing them as a plugin for
 your editor; for example in Vim, one can auto-format files with `black` on writing out a buffer, and it will also display `flake8` errors.

lib/database_config.py

@@ -9,15 +9,35 @@
 
 
 def get_database_config(development_mode=False):
+    """
+    the following options are validated to get database credentials
+    1) existence of custom runtime settings Database.... values
+    2) VCAP with database credentials
+    3) existence of DATABASE_URL env var
+
+    In case we find MXRUNTIME_Database.... values we don't interfere and
+    return nothing. VCAP or DATABASE_URL return m2ee configuration
+    """
     if any(
         [x.startswith("MXRUNTIME_Database") for x in list(os.environ.keys())]
     ):
+        logging.debug(
+            "Detected database configuration using custom runtime settings."
+        )
         return None
 
     factory = DatabaseConfigurationFactory()
     configuration = factory.get_instance()
 
-    return configuration.get_m2ee_configuration()
+    if configuration:
+        m2ee_config = configuration.get_m2ee_configuration()
+        if m2ee_config and "DatabaseType" in m2ee_config:
+            return m2ee_config
+
+    raise RuntimeError(
+        "Can't find database configuration from environment variables. "
+        "Check README for supported configuration options."
+    )
 
 
 class DatabaseConfigurationFactory:
@@ -38,10 +58,10 @@ def get_instance(self):
 
         # fallback to original configuration
         url = self.get_database_uri_from_vcap(self.vcap_services)
-        if url is None:
+        if not url and "DATABASE_URL" in os.environ:
             url = os.environ["DATABASE_URL"]
 
-        if url is not None:
+        if url:
             return UrlDatabaseConfiguration(url)
 
         return None

lib/datadog.py

@@ -252,6 +252,11 @@ def _set_up_postgres():
         "DatabaseHost",
     ):
         if k not in dbconfig:
+            logger.warn(
+                "Skipping database configuration for DataDog because "
+                "configuration is not found. See database_config.py "
+                "for details"
+            )
             return
     if dbconfig["DatabaseType"] != "PostgreSQL":
         return

lib/metrics.py

@@ -231,6 +231,7 @@ def _get_database_storage(self):
     def _get_database_mutations(self):
         conn = self._get_db_conn()
         db_config = database_config.get_database_config()
+
         with conn.cursor() as cursor:
             cursor.execute(
                 "SELECT xact_commit, "
@@ -327,12 +328,19 @@ def _get_db_conn(self):
             self.db = None
 
         if not self.db:
+            # get_database config may return None or empty
             db_config = database_config.get_database_config()
+            if not db_config or "DatabaseType" not in db_config:
+                raise ValueError(
+                    "Database not set as VCAP or DATABASE_URL. Check "
+                    "documentation to see supported configuration options."
+                )
             if db_config["DatabaseType"] != "PostgreSQL":
                 raise Exception(
                     "Metrics only supports postgresql, not %s"
                     % db_config["DatabaseType"]
                 )
+
             host_and_port = db_config["DatabaseHost"].split(":")
             host = host_and_port[0]
             if len(host_and_port) > 1:

start.py

@@ -29,7 +29,7 @@
 from nginx import get_path_config, gen_htpasswd  # noqa: E402
 from buildpackutil import i_am_primary_instance  # noqa: E402
 
-BUILDPACK_VERSION = "3.5.0"
+BUILDPACK_VERSION = "3.6.0"
 
 
 logger.setLevel(buildpackutil.get_buildpack_loglevel())
@@ -667,11 +667,15 @@ def set_runtime_config(metadata, mxruntime_config, vcap_data, m2ee):
 
     buildpackutil.mkdir_p(os.path.join(os.getcwd(), "model", "resources"))
     mxruntime_config.update(app_config)
-    mxruntime_config.update(
-        database_config.get_database_config(
-            development_mode=is_development_mode()
-        )
+
+    # db configuration might be None, database should then be set up with
+    # MXRUNTIME_Database... custom runtime settings.
+    runtime_db_config = database_config.get_database_config(
+        development_mode=is_development_mode()
     )
+    if runtime_db_config:
+        mxruntime_config.update(runtime_db_config)
+
     mxruntime_config.update(get_filestore_config(m2ee))
     mxruntime_config.update(get_certificate_authorities())
     mxruntime_config.update(get_client_certificates())

tests/usecase/test_db_config_options.py

@@ -0,0 +1,85 @@
+import os
+import unittest
+from database_config import get_database_config
+
+
+class TestDatabaseConfigOptions(unittest.TestCase):
+    def clean_env(self):
+        """
+        Setting different environment variables for test in the same process
+        can lead to flaky tests.
+        """
+        if "DATABASE_URL" in os.environ.keys():
+            del os.environ["DATABASE_URL"]
+
+        for key in filter(
+            lambda x: x.startswith("MXRUNTIME_Database"),
+            list(os.environ.keys()),
+        ):
+            del os.environ[key]
+
+    def test_no_setup(self):
+        self.clean_env()
+        with self.assertRaises(RuntimeError):
+            get_database_config()
+
+    def test_mx_runtime_db_config(self):
+        """
+        Test is MXRUNTIME variables are set up no database configuration is returned
+        based on DATABASE_URL or VCAP_SERVICES
+        """
+        self.clean_env()
+        os.environ["MXRUNTIME_DatabaseType"] = "PostgreSQL"
+        os.environ[
+            "MXRUNTIME_DatabaseJdbcUrl"
+        ] = "postgres://username:password@rdsbroker-testfree-nonprod-1-eu-west-1.asdbjasdg.eu-west-1.rds.amazonaws.com:5432/testdatabase"  # noqa E501
+
+        config = get_database_config()
+        assert not config
+
+    def test_database_url(self):
+        self.clean_env()
+        os.environ[
+            "DATABASE_URL"
+        ] = "jdbc:postgres://user:secret@host/database"
+
+        config = get_database_config()
+        assert config
+        assert config["DatabaseType"] == "PostgreSQL"
+
+    def test_vcap(self):
+        self.clean_env()
+        os.environ[
+            "VCAP_SERVICES"
+        ] = """
+{
+ "rds-testfree": [
+   {
+    "binding_name": null,
+    "credentials": {
+     "db_name": "dbuajsdhkasdhaks",
+     "host": "rdsbroker-testfree-nonprod-1-eu-west-1.asdbjasdg.eu-west-1.rds.amazonaws.com",
+     "password": "na8nanlayaona0--anbs",
+     "uri": "postgres://ua98s7ananla:na8nanlayaona0--anbs@rdsbroker-testfree-nonprod-1-eu-west-1.asdbjasdg.eu-west-1.rds.amazonaws.com:5432/dbuajsdhkasdhaks",
+     "username": "ua98s7ananla"
+    },
+    "instance_name": "ops-432a659e.test.foo.io-database",
+    "label": "rds-testfree",
+    "name": "ops-432a659e.test.foo.io-database",
+    "plan": "shared-psql-testfree",
+    "provider": null,
+    "syslog_drain_url": null,
+    "tags": [
+     "database",
+     "RDS",
+     "postgresql"
+    ],
+    "volume_mounts": []
+   }
+  ]
+}
+    """  # noqa
+
+        config = get_database_config()
+        assert config
+        assert config["DatabaseType"] == "PostgreSQL"