Release 2.6.2 - Cluster DB, abstract db, more translations (#195)

* E2E tests (#163)

* Add E2E tests

add kosh

remove indiviual prod envs

* add test for random and search ops

* update script key

* Updates for security vunerablities (#165)

* Add local db running in docker-container (#169)

* Add lcoal db running docker-container

* update start strict

* update docker-compose

* update readme and arrange package.json script alphabetically

* fix typo

* add await for container to be ready and start script DB port

* Update README.md

* More transl srcs (#182)

* fix issue with blank visraam #181
add manmohan singh to punjabi
fix some docker script problems

* Automatically serve any available punjabi translations

* remove lodash

* resolving issues from sttm-api repo

* fix for ceremonies

* remove lodash

* Delete package-lock.json

* Add translation sources

* Failover db (#186)

* Adds failover to database connection #179

* Dbname to process (#189)

* Move db configs to process env

* increment version

* Review fixes - change to array and add examples

* remove extra truthiness

* add package-lock.json back

* Update maria db connector (#192)

* Use ordered cluster, keep idle connections, compress db stream (#194)

Co-authored-by: tsingh777 <[email protected]>
Co-authored-by: Aman Singh <[email protected]>

.eslintrc

@@ -14,8 +14,9 @@
   "rules": {
     "no-console": 0,
     "no-use-before-define": 0,
+    "no-extra-boolean-cast": 0,
     "prettier/prettier": [
       "error"
     ]
   }
-}
+}

api/config/index.js

@@ -1,11 +1,76 @@
-module.exports = {
-  mysql: {
+/**
+ * Expects process-dev.json or process.json (or run local)
+ *
+ * @example
+ *
+ * {
+ *    "name" : "banidb",
+ *    "script" : "app.js",
+ *    "env": {
+ *        "NODE_ENV": "development|production",
+ *        "DB_USER": "",
+ *        "DB_PASSWORD": "",
+ *        "DB_POOL_SIZE": 5,
+ *        "DB_NODES": [
+ *          {
+ *            "host": "localhost",
+ *            "port": 3306,
+ *            "isPrimary": true,
+ *          },
+ *          {
+ *            "host": "dba.contoso.net",
+ *          },
+ *          {
+ *            "host": "dbb.contoso.net",
+ *          },
+ *        ],
+ *    },
+ *    "error_file": "err.log",
+ *    "out_file": "out.log",
+ *    "log_date_format": "YYYY-MM-DD HH:mm:ss Z"
+ * }
+ */
+
+const metadata = {
+  user: process.env.DB_USER || 'root',
+  password: process.env.DB_PASSWORD || 'root',
+  database: process.env.NODE_ENV === 'development' ? 'khajana_dev_khajana' : 'khajana_khajana',
+  dateStrings: true,
+  compress: true,
+  acquireTimeout: 6000,
+  connectionLimit: process.env.DB_POOL_SIZE,
+};
+const standbyMetadata = {
+  minimumIdle: 2,
+};
+
+// everything below is to pull from process.json
+const configArry = [];
+
+// if npm run local, then just define what cli gives, otherwise use process.json
+if (!!process.env.DB_NODES) {
+  const dbs = JSON.parse(process.env.DB_NODES);
+  dbs.forEach(db => {
+    const thisObj = {
+      ...metadata,
+      host: db.host,
+      port: db.port || 3306,
+    };
+    if (!!db.isPrimary) {
+      configArry.push(thisObj);
+    } else {
+      configArry.push({
+        ...thisObj,
+        ...standbyMetadata,
+      });
+    }
+  });
+} else {
+  configArry.push({
+    ...metadata,
     host: process.env.DB_HOST || 'localhost',
     port: process.env.DB_PORT || 3306,
-    user: process.env.DB_USER || 'root',
-    password: process.env.DB_PASSWORD || 'root',
-    database: process.env.NODE_ENV === 'development' ? 'khajana_dev_khajana' : 'khajana_khajana',
-    dateStrings: true,
-    connectionLimit: process.env.DB_POOL_SIZE,
-  },
-};
+  });
+}
+
+module.exports = configArry;

app.js

@@ -2,15 +2,19 @@ const express = require('express');
 const bodyParser = require('body-parser');
 const cors = require('cors');
 const cacheControl = require('express-cache-controller');
-const { createPool } = require('mariadb');
+const { createPoolCluster } = require('mariadb');
 const config = require('./api/config');
 const routes = require('./api/routes');
 
 const app = express();
 const port = process.env.NODE_ENV === 'development' ? '3001' : '3000';
 
-app.locals.pool = createPool(config.mysql);
+// database
+const dbCluster = createPoolCluster();
+config.forEach(dbConfig => dbCluster.add(dbConfig.host, dbConfig));
+app.locals.pool = dbCluster.of(/.*?/, 'ORDER');
 
+// app
 app.use(cors());
 app.use(cacheControl({ maxAge: 21600 }));
 app.use(bodyParser.urlencoded({ extended: true }));

bin/start-docker.sh

@@ -9,21 +9,21 @@ else
     dbPort=${DB_PORT}
 fi
 
-if [[ -z "$is_running" || "$is_running" -eq "false" ]]
+if [[ -z "$is_running" && "$is_running" -eq "false" ]]
 then
      echo "Starting docker container"
-     docker run -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=khajana_dev_khajana -d -p $dbPort:3306 --name $container_name khalisfoundation/banidb-dev:latest
+     docker run -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=khajana_dev_khajana -d -p $dbPort:3306 --name $container_name khalisfoundation/banidb-dev:latest || docker start $container_name
      attempt=0
       while [ $attempt -le 59 ]; do
-      attempt=$(( $attempt + 1 ))
-      echo "Waiting for container to be up (attempt: $attempt)..."
-      result=$(docker exec banidb-api mysqladmin ping -u root -proot 2> /dev/null || echo 'fail' )
-      if grep -q 'mysqld is alive' <<< $result ; then
-            echo "DB container is up"
-            echo "Waiting so connections stabilize"
-            break
-      fi
-      sleep 10
+        attempt=$(( $attempt + 1 ))
+        echo "Waiting for container to be up (attempt: $attempt)..."
+        result=$(docker exec banidb-api mysqladmin ping -u root -proot 2> /dev/null || echo 'fail' )
+        if grep -q 'mysqld is alive' <<< $result ; then
+              echo "DB container is up"
+              echo "Waiting for connections to stabilize"
+              break
+        fi
+        sleep 10
       done
       sleep 20
       echo "~~~~Ready~~~~~~"