Add DB initialization

Add lambda to initialize the database and run it during TF run

code/lambda-functions/waze-db-initialize/package.json

@@ -0,0 +1,29 @@
+{
+  "name": "waze-db-initialize",
+  "version": "1.0.0",
+  "description": "",
+  "main": "waze-db-initialize.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "postinstall": "webpack --config webpack.config.js",
+    "build": "webpack --config webpack.config.js"
+  },
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "pg": "^7.4.3"
+  },
+  "devDependencies": {
+    "@types/aws-lambda": "^8.10.8",
+    "@types/node": "^10.5.3",
+    "@types/node-fetch": "^2.1.2",
+    "@types/pg": "^7.4.10",
+    "aws-sdk": "^2.280.1",
+    "copy-webpack-plugin": "^4.5.2",
+    "ts-loader": "^4.4.2",
+    "typescript": "^2.9.2",
+    "webpack": "^4.16.2",
+    "webpack-cli": "^3.1.0",
+    "zip-webpack-plugin": "^3.0.0"
+  }
+}

code/lambda-functions/waze-db-initialize/src/waze-db-initialize.ts

@@ -0,0 +1,114 @@
+import AWS = require('aws-sdk');
+import { Handler, Context, Callback } from 'aws-lambda';
+import pg = require('pg'); 
+import fs = require('fs');
+
+
+const initializeDatabase: Handler = async (event: any, context: Context, callback: Callback) => {
+    try{
+
+        // get a DB client to run our queries with
+        var dbClient = new pg.Client();
+
+        //open the connection
+        await dbClient.connect();
+
+        //grab the env vars we'll need for the usernames and passwords to create
+        let lambda_username = process.env.LAMBDAPGUSER;
+        let lambda_password = process.env.LAMBDAPGPASSWORD;
+        let readonly_username = process.env.READONLYPGUSER;
+        let readonly_password = process.env.READONLYPASSWORD;
+        let current_version = process.env.CURRENTVERSION;
+
+        //check if the schema already exists, and if so, check the version installed against what we're trying to install
+        //if versions are same, just log info message and exit, otherwise log warning and exit
+        let schemaResult = await dbClient.query("SELECT 1 FROM information_schema.schemata WHERE schema_name = 'waze';");
+        if (schemaResult.rowCount > 0){
+            //the schema exists, see if we have a version table (that gets its own special error)
+            console.log("SCHEMA exists, verifying versions");
+
+            let versionTableExistsResult = await dbClient.query("SELECT 1 FROM information_schema.tables WHERE table_schema = 'waze' AND table_name = 'application_version';")
+            if(versionTableExistsResult.rowCount === 0){
+                //there IS NO version table, which is a problem
+                console.error('Version table not found');
+                return { response: formatTerraformWarning('Version table not found, please verify SQL schema is up to date.') };
+            }
+
+            //version table found, so we need to make sure it is the same version as what we would be trying to install
+            let versionCheckResult = await dbClient.query("SELECT version_number from waze.application_version ORDER BY install_date DESC LIMIT 1");
+            //if we didn't get a result, or get a result that isn't an exact match, warn about it
+            if(versionCheckResult.rowCount === 0){
+                console.error('No version records found');
+                return { response: formatTerraformWarning('No version records found, please verify SQL schema is up to date.') };
+            }
+            else if(versionCheckResult.rows[0].version_number !== current_version){
+                console.error('Version mismatch');
+                return { response: formatTerraformWarning('Version mismatch, please verify SQL schema is up to date.') };
+            }
+            else{
+                //versions match up, so just return a notice that nothing needed to be done
+                console.log('Versions match, no DB changes needed');
+                return { response: "Database is up-to-date" };
+            }
+        }
+
+        //the schema didn't exist, so we need to create everything
+        //first, load up the initialize script
+        let initFile = fs.readFileSync('./initialize-schema-and-roles.sql', 'utf-8');
+
+        //now we need to replace the placeholders
+        //we'll also do a quick check that they actually exist, and throw an error if not, just in case someone broke the script
+        const lambdaUserPlaceholder = 'LAMBDA_ROLE_NAME_PLACEHOLDER';
+        const lambdaPassPlaceholder = 'LAMBDA_ROLE_PASSWORD_PLACEHOLDER';
+        const readonlyUserPlaceholder = 'READONLY_ROLE_NAME_PLACEHOLDER';
+        const readonlyPassPlaceholder = 'READONLY_ROLE_PASSWORD_PLACEHOLDER';
+
+        if(initFile.indexOf(lambdaUserPlaceholder) < 0 || initFile.indexOf(lambdaPassPlaceholder) < 0 || 
+           initFile.indexOf(readonlyUserPlaceholder) < 0 || initFile.indexOf(readonlyPassPlaceholder) < 0){
+            throw new Error('DB initialization script is missing placeholders and cannot be run');
+        }
+
+        //run all the replacements
+        initFile = initFile.replace(new RegExp(lambdaUserPlaceholder, 'g'), lambda_username)
+                           .replace(new RegExp(lambdaPassPlaceholder, 'g'), lambda_password)
+                           .replace(new RegExp(readonlyUserPlaceholder, 'g'), readonly_username)
+                           .replace(new RegExp(readonlyPassPlaceholder, 'g'), readonly_password);
+
+        //execute the sql!
+        await dbClient.query(initFile);
+
+        //load and run the table creation
+        let schemaFile = fs.readFileSync('./schema.sql', 'utf-8');
+        await dbClient.query(schemaFile);
+
+        //update the version table
+        await dbClient.query('INSERT INTO waze.application_version VALUES ($1, current_timestamp)', [current_version]);
+
+        //return success
+        console.log('Database intialization succeeded');
+        return { response: "Database intialization succeeded" }
+
+    }
+    catch (err) {
+        console.error(err);
+        callback(err);
+        return err;
+    }
+    finally{
+        // CLOSE THAT CLIENT!
+        await dbClient.end();
+    }
+};
+
+export {initializeDatabase}
+
+//build a terraform-output-friendly warning message
+function formatTerraformWarning(warningMessage:string):string {
+    return `
+    
+    WARNING! ********************* WARNING! ********************* WARNING!
+    ${warningMessage}
+    WARNING! ********************* WARNING! ********************* WARNING!
+
+    `;
+}

code/lambda-functions/waze-db-initialize/tsconfig.json

@@ -0,0 +1,16 @@
+{
+    "compilerOptions": {
+        "outDir": "./built",
+        "allowJs": true,
+        "noImplicitAny": true,
+        "removeComments": true,
+        "target": "es6",
+        "alwaysStrict": true,
+        "checkJs": true,
+        "noEmitOnError": true,
+        "module": "commonjs"
+    },
+    "include": [
+        "./src/**/*"
+    ]
+}

code/lambda-functions/waze-db-initialize/webpack.config.js

@@ -0,0 +1,42 @@
+var path = require('path');
+var ZipPlugin = require('zip-webpack-plugin');
+var CopyWebpackPlugin = require('copy-webpack-plugin')
+
+module.exports = {
+    entry: './src/waze-db-initialize.ts',
+    target: 'node',
+    mode: 'production',
+    module: {
+      rules: [
+        {
+          test: /\.tsx?$/,
+          use: 'ts-loader',
+          exclude: /node_modules/
+        }
+      ]
+    },
+    resolve: {
+      extensions: [ '.tsx', '.ts', '.js' ]
+    },
+    externals: ['pg-native'],
+    output: {
+      libraryTarget: 'commonjs',
+      filename: 'waze-db-initialize.js',
+      path: path.resolve(__dirname, 'dist')
+    },
+    plugins: [
+        new CopyWebpackPlugin([
+          '../../sql/initialize-schema-and-roles.sql',
+          '../../sql/schema.sql'
+        ]),
+        new ZipPlugin({
+          // OPTIONAL: defaults to the Webpack output path (above)
+          // can be relative (to Webpack output path) or absolute
+          path: '../../',
+
+          // OPTIONAL: defaults to the Webpack output filename (above) or,
+          // if not present, the basename of the path
+          filename: 'waze-db-initialize.zip',
+        })
+      ]
+  };

infrastructure/terraform/environment/env-dev/outputs.tf

@@ -31,4 +31,9 @@ output "data_in_queue_alarm_sns_topic_arn" {
 output "data_processing_dlq_sns_topic_arn" {
     value = "${module.environment.data_processing_dlq_sns_topic_arn}"
     description = "ARN of the SNS topic that will receive notifications when records are found in the dead letter queue"
+}
+
+output "db_init_response" {
+    value = "${module.environment.db_init_response}"
+    description = "Response returned by DB initialization invocation"
 }

infrastructure/terraform/environment/env-prod/outputs.tf

@@ -31,4 +31,9 @@ output "data_in_queue_alarm_sns_topic_arn" {
 output "data_processing_dlq_sns_topic_arn" {
     value = "${module.environment.data_processing_dlq_sns_topic_arn}"
     description = "ARN of the SNS topic that will receive notifications when records are found in the dead letter queue"
+}
+
+output "db_init_response" {
+    value = "${module.environment.db_init_response}"
+    description = "Response returned by DB initialization invocation"
 }

infrastructure/terraform/modules/current-app-version/Readme.md

@@ -0,0 +1,3 @@
+## Current App Version Module
+
+Provides a single easy to find location for tracking the current application version, so that we can pass it to lambdas and such as needed.

infrastructure/terraform/modules/current-app-version/current-app-version.tf

@@ -0,0 +1,4 @@
+output "version_number" {
+    value = "2.1"
+    description = "The current version of the application"
+}

infrastructure/terraform/modules/environment/environment.tf

@@ -3,6 +3,12 @@ provider "aws" {
   region = "${var.default_resource_region}"
 }
 
+# load up the current app version module
+module "current_app_version" {
+  source      = "../current-app-version"
+}
+
+
 ###############################################
 # Cloudwatch
 ###############################################
@@ -380,7 +386,7 @@ resource "aws_iam_role_policy_attachment" "data_retrieval_lambda_basic_logging_r
 resource "aws_lambda_function" "waze_data_retrieval_function" {
   filename      = "${var.lambda_artifacts_path}/waze-data-download.zip"
   function_name = "${var.object_name_prefix}-waze-data-retrieval"
-  runtime       = "nodejs6.10"
+  runtime       = "nodejs8.10"
   role          = "${aws_iam_role.data_retrieval_execution_role.arn}"
   handler       = "waze-data-download.downloadData"
   timeout       = 300
@@ -467,7 +473,7 @@ resource "aws_lambda_function" "waze_data_alerts_processing_function" {
 resource "aws_lambda_function" "waze_data_jams_processing_function" {
   filename      = "${var.lambda_artifacts_path}/waze-data-process.zip"
   function_name = "${var.object_name_prefix}-waze-data-jams-processing"
-  runtime       = "nodejs6.10"
+  runtime       = "nodejs8.10"
   role          = "${aws_iam_role.data_retrieval_execution_role.arn}"
   handler       = "waze-data-process.processDataJams"
   timeout       = 300
@@ -494,7 +500,7 @@ resource "aws_lambda_function" "waze_data_jams_processing_function" {
 resource "aws_lambda_function" "waze_data_irregularities_processing_function" {
   filename      = "${var.lambda_artifacts_path}/waze-data-process.zip"
   function_name = "${var.object_name_prefix}-waze-data-irregularities-processing"
-  runtime       = "nodejs6.10"
+  runtime       = "nodejs8.10"
   role          = "${aws_iam_role.data_retrieval_execution_role.arn}"
   handler       = "waze-data-process.processDataIrregularities"
   timeout       = 300
@@ -517,6 +523,63 @@ resource "aws_lambda_function" "waze_data_irregularities_processing_function" {
   }
 }
 
+# setup the db initialize function
+resource "aws_lambda_function" "waze_db_initialize_function" {
+
+  # this function will be used to intialize the DB, so a DB instance must be in service first
+  depends_on = ["aws_rds_cluster_instance.waze_database_instances"]
+
+  filename      = "${var.lambda_artifacts_path}/waze-db-initialize.zip"
+  source_code_hash  = "${base64sha256(file("${var.lambda_artifacts_path}/waze-db-initialize.zip"))}"
+  function_name = "${var.object_name_prefix}-waze-db-initialize"
+  runtime       = "nodejs8.10"
+  role          = "${aws_iam_role.data_retrieval_execution_role.arn}"
+  handler       = "waze-db-initialize.initializeDatabase"
+  timeout       = 300
+  memory_size   = 512
+
+  environment {
+    variables = {
+      PGHOST     = "${aws_rds_cluster.waze_database_cluster.endpoint}"
+      PGUSER     = "${aws_rds_cluster.waze_database_cluster.master_username}" 
+      PGPASSWORD = "${aws_rds_cluster.waze_database_cluster.master_password}"
+      PGDATABASE = "${aws_rds_cluster.waze_database_cluster.database_name}"
+      PGPORT     = "${var.rds_port}"
+      POOLSIZE   = "1"
+
+      LAMBDAPGUSER = "${var.lambda_db_username}"
+      LAMBDAPGPASSWORD = "${var.lambda_db_password}"
+
+      READONLYPGUSER = "${var.rds_readonly_username}"
+      READONLYPASSWORD = "${var.rds_readonly_password}"
+
+      CURRENTVERSION = "${module.current_app_version.version_number}"
+    }
+  }
+
+  tags {
+    Environment = "${var.environment}"
+    Scripted    = "true"
+  }
+}
+
+# invoke the db init lambda so that the database gets initialized
+data "aws_lambda_invocation" "waze_db_init_invocation" {
+  function_name = "${aws_lambda_function.waze_db_initialize_function.function_name}"
+  # input is required by the invocation data source, but not required by the function, so pass empty JSON
+  input = <<JSON
+{
+  
+}
+JSON
+
+}
+
+output "db_init_response" {
+  value = "${data.aws_lambda_invocation.waze_db_init_invocation.result_map["response"]}"
+}
+
+
 ################################################
 # VPC
 ################################################