diff --git a/code/lambda-functions/waze-data-process.zip b/code/lambda-functions/waze-data-process.zip index e01e0ac..a46f8f8 100644 Binary files a/code/lambda-functions/waze-data-process.zip and b/code/lambda-functions/waze-data-process.zip differ diff --git a/code/lambda-functions/waze-data-process/src/db/commands.ts b/code/lambda-functions/waze-data-process/src/db/commands.ts index f0960f8..9330113 100644 --- a/code/lambda-functions/waze-data-process/src/db/commands.ts +++ b/code/lambda-functions/waze-data-process/src/db/commands.ts @@ -59,7 +59,8 @@ export async function upsertAlertCommand(alert: entities.Alert): Promise { report_by_municipality_user, thumbs_up, jam_uuid, - datafile_id + datafile_id, + dayofweek ) VALUES ( $1, -- id @@ -81,7 +82,8 @@ export async function upsertAlertCommand(alert: entities.Alert): Promise { $17, -- report_by_municipality_user $18, -- thumbs_up $19, -- jam_uuid - $20 -- datafile_id + $20, -- datafile_id + $21 -- dayofweek ) ON CONFLICT (id) DO UPDATE SET uuid=$2, @@ -102,7 +104,8 @@ export async function upsertAlertCommand(alert: entities.Alert): Promise { report_by_municipality_user=$17, thumbs_up=$18, jam_uuid=$19, - datafile_id=$20`; + datafile_id=$20, + dayofweek=$21`; //#endregion let result = await connectionPool.getPool().query(sql, [ @@ -126,12 +129,21 @@ export async function upsertAlertCommand(alert: entities.Alert): Promise { alert.thumbs_up, //thumbs_up alert.jam_uuid , //jam_uuid alert.datafile_id, //datafile_id + alert.dayofweek //dayofweek ]); //nothing currently to alter on the alert object based on SQL return return; } +// update geometry for a jam. This uses a function that converst jsonb to geometry +// the function definition is assumed to match the SRID of the column definitions +export async function updateAlertGeometry(id: string) { + const sql = `update waze.alerts set geom_point=waze.location_to_geometry(location) where id=$1`; + let result = await connectionPool.getPool().query(sql, [id]); + return; +} + // upsert a jam record export async function upsertJamCommand(jam: entities.Jam): Promise { //for simplicity, we'll always insert and update all fields, since our hash should ensure there aren't unexpected changes @@ -159,7 +171,10 @@ export async function upsertJamCommand(jam: entities.Jam): Promise { line, datafile_id, type, - turn_line + turn_line, + ns_direction, + ew_direction, + dayofweek ) VALUES ( $1, -- id @@ -182,7 +197,10 @@ export async function upsertJamCommand(jam: entities.Jam): Promise { $18, -- line $19, -- datafile_id $20, -- type - $21 -- turn_line + $21, -- turn_line + $22, -- ns_direction + $23, -- ew_direction + $24 -- dayofweek ) ON CONFLICT (id) DO UPDATE SET uuid=$2, @@ -204,7 +222,10 @@ export async function upsertJamCommand(jam: entities.Jam): Promise { line=$18, datafile_id=$19, type=$20, - turn_line=$21`; + turn_line=$21, + ns_direction=$22, + ew_direction=$23, + dayofweek=$24`; //#endregion let result = await connectionPool.getPool().query(sql, [ @@ -229,6 +250,9 @@ export async function upsertJamCommand(jam: entities.Jam): Promise { jam.datafile_id, //datafile_id jam.type, //type jam.turn_line, //turn_line + jam.ns_direction, //ns_direction + jam.ew_direction, //ew_direction + jam.dayofweek //dayofweek ]); //nothing currently to update on the jam object based on SQL return @@ -236,6 +260,14 @@ export async function upsertJamCommand(jam: entities.Jam): Promise { } +// update geometry for a jam. This uses a function that converst jsonb to geometry +// the function definition is assumed to match the SRID of the column definitions +export async function updateJamGeometry(id: string) { + const sql = `update waze.jams set geom_line=waze.line_to_geometry(line) where id=$1`; + let result = await connectionPool.getPool().query(sql, [id]); + return; +} + // upsert an irregularity record export async function upsertIrregularityCommand(irregularity: entities.Irregularity): Promise { //for simplicity, we'll always insert and update all fields, since our hash should ensure there aren't unexpected changes diff --git a/code/lambda-functions/waze-data-process/src/entities.ts b/code/lambda-functions/waze-data-process/src/entities.ts index d3a4314..5fa0d05 100644 --- a/code/lambda-functions/waze-data-process/src/entities.ts +++ b/code/lambda-functions/waze-data-process/src/entities.ts @@ -32,6 +32,7 @@ export class Alert { thumbs_up: number; jam_uuid: string; datafile_id: number; + dayofweek: number; } export class Jam { @@ -56,6 +57,9 @@ export class Jam { type: string; turn_line: string; datafile_id: number; + ew_direction: string; + ns_direction: string; + dayofweek: number; } export class Irregularity { diff --git a/code/lambda-functions/waze-data-process/src/waze-data-process.ts b/code/lambda-functions/waze-data-process/src/waze-data-process.ts index b89d2a0..58e09f9 100644 --- a/code/lambda-functions/waze-data-process/src/waze-data-process.ts +++ b/code/lambda-functions/waze-data-process/src/waze-data-process.ts @@ -15,7 +15,7 @@ const sqs = new AWS.SQS(); const lambda = new AWS.Lambda(); const sns = new AWS.SNS(); -const throttleOpts:throttle.Options = { +const throttleOpts: throttle.Options = { failFast: true, maxInProgress: parseInt(process.env.POOLSIZE) } @@ -26,11 +26,11 @@ const hashOpts = { }; const processDataFile: Handler = async (event: any, context: Context, callback: Callback) => { - try{ + try { //patch the console so we can get more searchable logging //would be nice to make this global, but couldn't quickly get that working consolePatch(); - + //we'll loop and process as long as there are records and the queue //and there is twice as much time left as the max loop time let isQueueDrained = false; @@ -38,7 +38,7 @@ const processDataFile: Handler = async (event: any, context: Context, callback: //keep this lambda alive and processing items from the queue as long as the time remaining //minus a 10 second buffer is greater than double the max iteration run time - while(context.getRemainingTimeInMillis() - 10000 > maxLoopTimeInMillis * 2){ + while (context.getRemainingTimeInMillis() - 10000 > maxLoopTimeInMillis * 2) { console.info('Continuing - Function time remaining: %d, Max iteration time: %d', context.getRemainingTimeInMillis(), maxLoopTimeInMillis); //"start" a timer for this iteration let loopStart = process.hrtime(); @@ -52,7 +52,7 @@ const processDataFile: Handler = async (event: any, context: Context, callback: let sqsResponse = await sqs.receiveMessage(sqsParams).promise(); // make sure we got a record - if(!sqsResponse.Messages || sqsResponse.Messages.length == 0){ + if (!sqsResponse.Messages || sqsResponse.Messages.length == 0) { //got no response, so set the flag and exit the loop isQueueDrained = true; break; @@ -67,9 +67,9 @@ const processDataFile: Handler = async (event: any, context: Context, callback: Key: s3Key, }; let s3Response = await s3.getObject(s3Params).promise(); - + //make sure we got something - if(s3Response.Body){ + if (s3Response.Body) { let fileData: wazeTypes.dataFileWithInternalId = JSON.parse(s3Response.Body.toString()); //first need to see if we've seen this file before or not, based on hash @@ -78,9 +78,9 @@ const processDataFile: Handler = async (event: any, context: Context, callback: let data_file = await db.getDataFilesByHashQuery(jsonHash); //if we got something, we need to check more stuff before updating anything - if(data_file){ + if (data_file) { //see if the file name has changed, and if so throw an error - if(data_file.file_name !== s3Key){ + if (data_file.file_name !== s3Key) { throw new Error(util.format("Found existing record for hash '%s' with file name '%s' (id: %d)", jsonHash, data_file.file_name, data_file.id)); } @@ -89,7 +89,7 @@ const processDataFile: Handler = async (event: any, context: Context, callback: //no change to the name, so the only thing we need to do is update the date_updated field await db.updateDataFileUpdateDateByIdCommand(data_file.id); } - else{ + else { //we didn't get a record, so we need to save one //build up the object we'll (potentially) be inserting into the DB data_file = new entities.DataFile(); @@ -103,7 +103,7 @@ const processDataFile: Handler = async (event: any, context: Context, callback: console.info(util.format('Creating data_file: %s', s3Key)); data_file = await db.insertDataFileCommand(data_file); - + } //now that we for sure have a data_file record in the DB, we'll need to pass the id on to everything else @@ -116,60 +116,60 @@ const processDataFile: Handler = async (event: any, context: Context, callback: let alerts = fileData.alerts; let jams = fileData.jams; let irregularities = fileData.irregularities; - + //now delete all 3 from the root delete fileData.alerts; delete fileData.jams; delete fileData.irregularities; - + //we'll need to keep track of the promises to process let promises = new Array>>(); //now we can check if we have each one and send them off for processing - if(alerts && alerts.length > 0){ - + if (alerts && alerts.length > 0) { + //add the alerts back to the object fileData.alerts = alerts; - + console.info(util.format('Invoking alert processor with %d alerts', alerts.length)); //send it off to be processed promises.push(invokeListProcessor(fileData, process.env.ALERTPROCESSORARN)); - + //remove alerts from the object again delete fileData.alerts; } - if(jams && jams.length > 0){ - + if (jams && jams.length > 0) { + //add the jams back to the object fileData.jams = jams; console.info(util.format('Invoking jam processor with %d jams', jams.length)); - + //send it off to be processed promises.push(invokeListProcessor(fileData, process.env.JAMPROCESSORARN)); - + //remove jams from the object again delete fileData.jams; } - if(irregularities && irregularities.length > 0){ - + if (irregularities && irregularities.length > 0) { + //add the irregularities back to the object fileData.irregularities = irregularities; console.info(util.format('Invoking irregularity processor with %d irregularities', irregularities.length)); - + //send it off to be processed promises.push(invokeListProcessor(fileData, process.env.IRREGULARITYPROCESSORARN)); - + //remove irregularities from the object again delete fileData.irregularities; } //wait for all of the promises to finish - if(promises.length > 0){ + if (promises.length > 0) { let promResult = await Promise.all(promises); let wereAllSuccessful = promResult.every(res => { //make sure we got a 200 and either have no FunctionError or an empty one @@ -179,7 +179,7 @@ const processDataFile: Handler = async (event: any, context: Context, callback: //if they were NOT all successful, log an error with the whole response //most likely the individual processor that failed will have more info logged, //but this will at least get us *something* just in case - if(!wereAllSuccessful) { + if (!wereAllSuccessful) { console.error(promResult); callback(new Error('Error processing alerts/jams/irregularities, review logs for more info')); return; @@ -194,9 +194,9 @@ const processDataFile: Handler = async (event: any, context: Context, callback: CopySource: util.format('/%s/%s', process.env.WAZEDATAINCOMINGBUCKET, s3Key) } let s3CopyResponse = await s3.copyObject(copyObjParams).promise(); - + //make sure the copy didn't fail without throwing an error - if(!s3CopyResponse.CopyObjectResult){ + if (!s3CopyResponse.CopyObjectResult) { console.error(s3CopyResponse); callback(new Error('Error copying object to processed bucket: ' + s3Key)); return; @@ -217,7 +217,7 @@ const processDataFile: Handler = async (event: any, context: Context, callback: await sqs.deleteMessage(deleteSqsParams).promise(); //send a message to the SNS topic if enabled - if(process.env.SNSTOPIC && process.env.SNSTOPIC.length > 0){ + if (process.env.SNSTOPIC && process.env.SNSTOPIC.length > 0) { await publishSnsMessage('Successfully processed ' + s3Key, process.env.SNSTOPIC); } @@ -230,13 +230,13 @@ const processDataFile: Handler = async (event: any, context: Context, callback: //convert loopend to milliseconds let loopTimeInMillis = (loopEnd["0"] * 1000) + (loopEnd["1"] / 1e6); //if this run was longer than the max, set a new max - if(loopTimeInMillis > maxLoopTimeInMillis){ + if (loopTimeInMillis > maxLoopTimeInMillis) { maxLoopTimeInMillis = loopTimeInMillis } } //if the queue is not drained, send a message to the SNS topic that retriggers this lambda - if(!isQueueDrained){ + if (!isQueueDrained) { //there's more in the queue, but we're low on time, so let's retrigger using sns await publishSnsMessage('Triggering self to continue work', process.env.RETRIGGERSNSTOPIC); } @@ -252,7 +252,7 @@ const processDataFile: Handler = async (event: any, context: Context, callback: }; const processDataAlerts: Handler = async (event: wazeTypes.dataFileWithInternalId, context: Context, callback: Callback) => { - try{ + try { //patch the console so we can get more searchable logging //would be nice to make this global, but couldn't quickly get that working consolePatch(); @@ -266,7 +266,7 @@ const processDataAlerts: Handler = async (event: wazeTypes.dataFileWithInternalI let data_file_id = event.data_file_id; //create a list of tasks to process alerts _asynchronously_ - let taskList = event.alerts.map((alert:wazeTypes.alert) => async () => { + let taskList = event.alerts.map((alert: wazeTypes.alert) => async () => { //hash the whole alert along with the rootStart to get a unique id let alertHash = generateAJIUniqueIdentifierHash(alert, rootStart); @@ -291,15 +291,22 @@ const processDataAlerts: Handler = async (event: wazeTypes.dataFileWithInternalI report_by_municipality_user: alert.reportByMunicipalityUser, thumbs_up: alert.nThumbsUp, jam_uuid: alert.jamUuid, - datafile_id: data_file_id + datafile_id: data_file_id, + dayofweek: null } + // process dependent fields + alertEntity.dayofweek = alertEntity.pub_utc_date.getDay(); + //upsert the alert - await db.upsertAlertCommand(alertEntity); + await db.upsertAlertCommand(alertEntity); + + // process geometry fields + await db.updateAlertGeometry(alertEntity.id); //add the individual coordinate record from the location field //alerts only have 1 of these, in the location object - if(alert.location){ + if (alert.location) { let coord: entities.Coordinate = { id: generateCoordinateUniqueIdentifierHash(alert.location, entities.CoordinateType.Location, alertHash, null, null), alert_id: alertHash, @@ -315,7 +322,7 @@ const processDataAlerts: Handler = async (event: wazeTypes.dataFileWithInternalI //run the tasks in a throttled fashion await throttle.all(taskList, throttleOpts); } - catch (err) { + catch (err) { console.error(err); callback(err); return err; @@ -323,7 +330,7 @@ const processDataAlerts: Handler = async (event: wazeTypes.dataFileWithInternalI }; const processDataJams: Handler = async (event: wazeTypes.dataFileWithInternalId, context: Context, callback: Callback) => { - try{ + try { //patch the console so we can get more searchable logging //would be nice to make this global, but couldn't quickly get that working consolePatch(); @@ -337,7 +344,7 @@ const processDataJams: Handler = async (event: wazeTypes.dataFileWithInternalId, let data_file_id = event.data_file_id; //create a list of tasks to process jams _asynchronously_ - let taskList = event.jams.map((jam:wazeTypes.jam) => async () => { + let taskList = event.jams.map((jam: wazeTypes.jam) => async () => { //hash the whole jam along with the rootStart to get a unique id let jamHash = generateAJIUniqueIdentifierHash(jam, rootStart); @@ -363,19 +370,55 @@ const processDataJams: Handler = async (event: wazeTypes.dataFileWithInternalId, line: JSON.stringify(jam.line), type: jam.type, datafile_id: data_file_id, - turn_line: JSON.stringify(jam.turnLine) + turn_line: JSON.stringify(jam.turnLine), + ns_direction: null, + ew_direction: null, + dayofweek: null + } + + // calculate derived things + + // jam.NS / EW direction: + { + jamEntity.ns_direction = null; + jamEntity.ew_direction = null; + if (jam.line.length >= 2) { + let firstPoint = jam.line[0]; + let lastPoint = jam.line[jam.line.length - 1]; + + if (lastPoint.y > firstPoint.y) { + jamEntity.ns_direction = 'N'; + } else if (lastPoint.y < firstPoint.y) { + jamEntity.ns_direction = 'S'; + } else { + jamEntity.ns_direction = null; // rare, but possible + } + + if (lastPoint.x > firstPoint.x) { + jamEntity.ew_direction = 'E'; + } else if (lastPoint.x < firstPoint.x) { + jamEntity.ew_direction = 'W'; + } else { + jamEntity.ew_direction = null; // rare, but possible + } + } } + jamEntity.dayofweek = jamEntity.pub_utc_date.getDay(); + //upsert the jam await db.upsertJamCommand(jamEntity); + // also update its geometry.. sorry, double write. + db.updateJamGeometry(jamEntity.id); + //add the individual coordinate records from the line and turnLine fields //we won't do these in parallel because we're already running jams async //and don't want to just blast the database if (jam.line) { for (let index = 0; index < jam.line.length; index++) { const lineCoord = jam.line[index]; - + let coord: entities.Coordinate = { id: generateCoordinateUniqueIdentifierHash(lineCoord, entities.CoordinateType.Line, null, jamHash, null), jam_id: jamHash, @@ -388,11 +431,11 @@ const processDataJams: Handler = async (event: wazeTypes.dataFileWithInternalId, await db.upsertCoordinateCommand(coord); } } - - if(jam.turnLine){ + + if (jam.turnLine) { for (let index = 0; index < jam.turnLine.length; index++) { const turnLineCoord = jam.turnLine[index]; - + let coord: entities.Coordinate = { id: generateCoordinateUniqueIdentifierHash(turnLineCoord, entities.CoordinateType.TurnLine, null, jamHash, null), jam_id: jamHash, @@ -409,7 +452,7 @@ const processDataJams: Handler = async (event: wazeTypes.dataFileWithInternalId, //run the tasks in a throttled fashion await throttle.all(taskList, throttleOpts); } - catch (err) { + catch (err) { console.error(err); callback(err); return err; @@ -417,7 +460,7 @@ const processDataJams: Handler = async (event: wazeTypes.dataFileWithInternalId, }; const processDataIrregularities: Handler = async (event: wazeTypes.dataFileWithInternalId, context: Context, callback: Callback) => { - try{ + try { //patch the console so we can get more searchable logging //would be nice to make this global, but couldn't quickly get that working consolePatch(); @@ -431,7 +474,7 @@ const processDataIrregularities: Handler = async (event: wazeTypes.dataFileWithI let data_file_id = event.data_file_id; //create a list of tasks to process irregularities _asynchronously_ - let taskList = event.irregularities.map((irregularity:wazeTypes.irregularity) => async () => { + let taskList = event.irregularities.map((irregularity: wazeTypes.irregularity) => async () => { //hash the whole irreg along with the rootStart to get a unique id let irregularityHash = generateAJIUniqueIdentifierHash(irregularity, rootStart); @@ -479,7 +522,7 @@ const processDataIrregularities: Handler = async (event: wazeTypes.dataFileWithI if (irregularity.line) { for (let index = 0; index < irregularity.line.length; index++) { const lineCoord = irregularity.line[index]; - + let coord: entities.Coordinate = { id: generateCoordinateUniqueIdentifierHash(lineCoord, entities.CoordinateType.Line, null, null, irregularityHash), irregularity_id: irregularityHash, @@ -496,7 +539,7 @@ const processDataIrregularities: Handler = async (event: wazeTypes.dataFileWithI //run the tasks in a throttled fashion await throttle.all(taskList, throttleOpts); } - catch (err) { + catch (err) { console.error(err); callback(err); return err; @@ -506,14 +549,14 @@ const processDataIrregularities: Handler = async (event: wazeTypes.dataFileWithI // publish a simple sns message async function publishSnsMessage(message: string, topicARN: string) { let snsParams: AWS.SNS.PublishInput = { - Message: JSON.stringify({source: 'waze-data-processor', message: message}), + Message: JSON.stringify({ source: 'waze-data-processor', message: message }), TopicArn: topicARN }; await sns.publish(snsParams).promise(); } // Read the S3 key from the retrieved SQS message -function getS3KeyFromMessage(message:AWS.SQS.Message): string { +function getS3KeyFromMessage(message: AWS.SQS.Message): string { // the info we need is down in the Body, because it came in via SNS // parse the body into JSON and grab the Message param let snsMessage = JSON.parse(message.Body).Message; @@ -524,12 +567,12 @@ function getS3KeyFromMessage(message:AWS.SQS.Message): string { } // Compute the hash of the given object using our standard options -function computeHash(object:Object): string { +function computeHash(object: Object): string { return hash(object, hashOpts); } // Generate a unique id based on the timestamp from the data root and the specific object (the whole alert, jam, irregularity, etc) -function generateAJIUniqueIdentifierHash(object:Object, rootStartTime:number): string { +function generateAJIUniqueIdentifierHash(object: Object, rootStartTime: number): string { //hash the object first let objHash = computeHash(object); //combine that hash with the timestamp @@ -542,7 +585,7 @@ function generateAJIUniqueIdentifierHash(object:Object, rootStartTime:number): s } // Generate a unique id based on the coordinate, type, and parent id -function generateCoordinateUniqueIdentifierHash(coordinate: wazeTypes.coordinate, type: entities.CoordinateType, alertId: string, jamId: string, irregularityId: string){ +function generateCoordinateUniqueIdentifierHash(coordinate: wazeTypes.coordinate, type: entities.CoordinateType, alertId: string, jamId: string, irregularityId: string) { //hash the coordinate first let coordHash = computeHash(coordinate); //combine the coord hash with the other data into a single object @@ -558,12 +601,12 @@ function generateCoordinateUniqueIdentifierHash(coordinate: wazeTypes.coordinate } // trigger an invocation of one of the list processor lambdas -function invokeListProcessor(data:wazeTypes.dataFileWithInternalId, lambdaARN:string){ +function invokeListProcessor(data: wazeTypes.dataFileWithInternalId, lambdaARN: string) { return lambda.invoke({ FunctionName: lambdaARN, InvocationType: 'RequestResponse', - Payload: JSON.stringify(data) + Payload: JSON.stringify(data) }).promise(); } -export {processDataFile, processDataAlerts, processDataJams, processDataIrregularities} \ No newline at end of file +export { processDataFile, processDataAlerts, processDataJams, processDataIrregularities } diff --git a/code/lambda-functions/waze-db-initialize.zip b/code/lambda-functions/waze-db-initialize.zip index 47953bd..57401e6 100644 Binary files a/code/lambda-functions/waze-db-initialize.zip and b/code/lambda-functions/waze-db-initialize.zip differ diff --git a/code/lambda-functions/waze-db-initialize/package-lock.json b/code/lambda-functions/waze-db-initialize/package-lock.json index 2a86f1d..5c48123 100644 --- a/code/lambda-functions/waze-db-initialize/package-lock.json +++ b/code/lambda-functions/waze-db-initialize/package-lock.json @@ -16,6 +16,23 @@ "integrity": "sha512-KEIlhXnIutzKwRbQkGWb/I4HFqBuUykAdHgDED6xqwXJfONCjF5VoE0cXEiurh3XauygxzeDzgtXUqvLkxFzzA==", "dev": true }, + "@types/glob": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@types/glob/-/glob-7.1.1.tgz", + "integrity": "sha512-1Bh06cbWJUHMC97acuD6UMG29nMt0Aqz1vF3guLfG+kHHJhy3AyohZFFxYk2f7Q1SQIrNwvncxAE0N/9s70F2w==", + "dev": true, + "requires": { + "@types/events": "1.2.0", + "@types/minimatch": "3.0.3", + "@types/node": "10.5.3" + } + }, + "@types/minimatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.3.tgz", + "integrity": "sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA==", + "dev": true + }, "@types/node": { "version": "10.5.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-10.5.3.tgz", @@ -508,8 +525,7 @@ "balanced-match": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", - "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", - "dev": true + "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=" }, "base": { "version": "0.11.2", @@ -600,7 +616,6 @@ "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", - "dev": true, "requires": { "balanced-match": "1.0.0", "concat-map": "0.0.1" @@ -760,7 +775,7 @@ "requires": { "bluebird": "3.5.1", "chownr": "1.0.1", - "glob": "7.1.2", + "glob": "7.1.3", "graceful-fs": "4.1.11", "lru-cache": "4.1.3", "mississippi": "2.0.0", @@ -960,8 +975,7 @@ "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", - "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", - "dev": true + "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, "concat-stream": { "version": "1.6.2", @@ -1598,8 +1612,7 @@ "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", - "dev": true + "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, "fsevents": { "version": "1.2.4", @@ -2149,10 +2162,9 @@ "dev": true }, "glob": { - "version": "7.1.2", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", - "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", - "dev": true, + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", "requires": { "fs.realpath": "1.0.0", "inflight": "1.0.6", @@ -2197,7 +2209,7 @@ "requires": { "array-union": "1.0.2", "dir-glob": "2.0.0", - "glob": "7.1.2", + "glob": "7.1.3", "ignore": "3.3.10", "pify": "3.0.0", "slash": "1.0.0" @@ -2337,7 +2349,6 @@ "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", - "dev": true, "requires": { "once": "1.4.0", "wrappy": "1.0.2" @@ -2346,8 +2357,7 @@ "inherits": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", - "dev": true + "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" }, "inquirer": { "version": "6.0.0", @@ -2754,7 +2764,6 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", - "dev": true, "requires": { "brace-expansion": "1.1.11" } @@ -3003,7 +3012,6 @@ "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", - "dev": true, "requires": { "wrappy": "1.0.2" } @@ -3132,8 +3140,7 @@ "path-is-absolute": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", - "dev": true + "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=" }, "path-key": { "version": "2.0.1", @@ -3468,7 +3475,7 @@ "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", "dev": true, "requires": { - "glob": "7.1.2" + "glob": "7.1.3" } }, "ripemd160": { @@ -4397,8 +4404,7 @@ "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", - "dev": true + "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" }, "xml2js": { "version": "0.4.19", diff --git a/code/lambda-functions/waze-db-initialize/package.json b/code/lambda-functions/waze-db-initialize/package.json index 6a2aa60..5ada1e1 100644 --- a/code/lambda-functions/waze-db-initialize/package.json +++ b/code/lambda-functions/waze-db-initialize/package.json @@ -11,9 +11,11 @@ "author": "", "license": "ISC", "dependencies": { + "glob": "^7.1.3", "pg": "^7.4.3" }, "devDependencies": { + "@types/glob": "^7.1.1", "@types/aws-lambda": "^8.10.8", "@types/node": "^10.5.3", "@types/node-fetch": "^2.1.2", diff --git a/code/lambda-functions/waze-db-initialize/src/waze-db-initialize.ts b/code/lambda-functions/waze-db-initialize/src/waze-db-initialize.ts index b5fb240..fc727b3 100644 --- a/code/lambda-functions/waze-db-initialize/src/waze-db-initialize.ts +++ b/code/lambda-functions/waze-db-initialize/src/waze-db-initialize.ts @@ -1,12 +1,12 @@ import AWS = require('aws-sdk'); import { Handler, Context, Callback } from 'aws-lambda'; -import pg = require('pg'); +import pg = require('pg'); import fs = require('fs'); - +import glob = require('glob'); const initializeDatabase: Handler = async (event: any, context: Context, callback: Callback) => { - try{ - + try { + // get a DB client to run our queries with var dbClient = new pg.Client(); @@ -20,90 +20,139 @@ const initializeDatabase: Handler = async (event: any, context: Context, callbac let readonly_password = process.env.READONLYPASSWORD; let current_version = process.env.CURRENTVERSION; - //check if the schema already exists, and if so, check the version installed against what we're trying to install - //if versions are same, just log info message and exit, otherwise log warning and exit - let schemaResult = await dbClient.query("SELECT 1 FROM information_schema.schemata WHERE schema_name = 'waze';"); - if (schemaResult.rowCount > 0){ - //the schema exists, see if we have a version table (that gets its own special error) - console.log("SCHEMA exists, verifying versions"); - - let versionTableExistsResult = await dbClient.query("SELECT 1 FROM information_schema.tables WHERE table_schema = 'waze' AND table_name = 'application_version';") - if(versionTableExistsResult.rowCount === 0){ - //there IS NO version table, which is a problem - console.error('Version table not found'); - return { response: formatTerraformWarning('Version table not found, please verify SQL schema is up to date.') }; + let whatWeExecuted:string[] = []; + + // how I debug / work on this: + + // Terraform: + // I commented the lambda invocation in terraform to prevent running this code, and had the out variable return a constant + // I had to terraform apply everything, as there are things necessary for cloudwatch logs and stuff that are not obvious + // if you just -target this function + + // command lines to build and upload new zip: + // npm run build "run the script called build" .. creates a new .zip file + // aws lambda update-function-code --function-name development-tf-waze-db-initialize --region us-west-2 --zip-file fileb://../waze-db-initialize.zip + + // Rewriting logic from version 2 to version 3: + + // 1. Problem: Version 2 didn't create readonly password + // Change to: Always run schema create script, and use "if not exist" and update the password. + // Everything else will be updated too, so better update database to match. + + // 2. Problem: New Version 3 has all the things already in it + // if we have infrastructure set up and run this terraform, its going to invoke this lambda + // Therefore a lot of things already exist, and are going to fail + // Solution: Everything is idempotent + + // 3. Run additional update scripts + // ... + // + + // So pretty much, what this is looking like: + // a) always create if not exists + // b) force update of passwords every time (rewrite create schema script) + // c) represent version 2 as its own (set of) scripts (except idempotent). + // d) always apply all scripts in a particular order + + // e) ALSO! Terraform plan ends up invoking this function. + // If it takes too long, then thats bad. + // So make it quickly determine not to invoke itself + // So it compares script filenames vs loaded versions and skips + // so if you want to reexcute a thing, delete from application_version + + let fileNames = glob.sync("*.sql", {}); // sort defaults to true; + let loadedVersions = []; + + let versionTableExistsResult = await dbClient.query(` + SELECT * + FROM information_schema.tables + WHERE table_schema = 'waze' + AND table_name = 'application_version'`); + if(versionTableExistsResult.rowCount === 0){ + // no versionTable, so don't populate alreadyLoadedVersions + } else { + let result = await dbClient.query("SELECT * FROM waze.application_version"); + if (result.rowCount>0) { + for(let row of result.rows) { + if (row['version_number']) { + loadedVersions[row['version_number']] = row; + console.log("detected loaded version "+row['version_number']+": "+JSON.stringify(row)); + } + } + } + } + + for (let fileName of fileNames) { + let m = fileName.match(/^([0-9\.]+)/); + + if (!m || m.length<1) { + console.log("Skipping "+fileName+" because it doesn't start with a version"); + continue; } - //version table found, so we need to make sure it is the same version as what we would be trying to install - let versionCheckResult = await dbClient.query("SELECT version_number from waze.application_version ORDER BY install_date DESC LIMIT 1"); - //if we didn't get a result, or get a result that isn't an exact match, warn about it - if(versionCheckResult.rowCount === 0){ - console.error('No version records found'); - return { response: formatTerraformWarning('No version records found, please verify SQL schema is up to date.') }; + let version = m[0]; + if (loadedVersions[version]) { + console.log("Skipping "+fileName+" because version is already loaded"); + continue; } - else if(versionCheckResult.rows[0].version_number !== current_version){ - console.error('Version mismatch'); - return { response: formatTerraformWarning('Version mismatch, please verify SQL schema is up to date.') }; + + // It is up to the script to insert rows into application_version + + let fileContent = fs.readFileSync(fileName, 'utf-8'); + // just in case it was saved as UTF8 BOM .. + + fileContent = fileContent.replace(/^\uFEFF/, ''); + + //now we need to replace the placeholders + //we'll also do a quick check that they actually exist, and throw an error if not, just in case someone broke the script + const lambdaUserPlaceholder = 'LAMBDA_ROLE_NAME_PLACEHOLDER'; + const lambdaPassPlaceholder = 'LAMBDA_ROLE_PASSWORD_PLACEHOLDER'; + const readonlyUserPlaceholder = 'READONLY_ROLE_NAME_PLACEHOLDER'; + const readonlyPassPlaceholder = 'READONLY_ROLE_PASSWORD_PLACEHOLDER'; + + //run all the replacements + let replacedFileContent = fileContent.replace(new RegExp(lambdaUserPlaceholder, 'g'), lambda_username) + .replace(new RegExp(lambdaPassPlaceholder, 'g'), lambda_password) + .replace(new RegExp(readonlyUserPlaceholder, 'g'), readonly_username) + .replace(new RegExp(readonlyPassPlaceholder, 'g'), readonly_password); + + let wasReplaced = ""; + if (fileContent != replacedFileContent) { + wasReplaced = " (with replacements)"; } - else{ - //versions match up, so just return a notice that nothing needed to be done - console.log('Versions match, no DB changes needed'); - return { response: "Database is up-to-date" }; + + //execute the sql! + console.log("Executing " + fileName + wasReplaced); + let results = await dbClient.query(replacedFileContent); + whatWeExecuted[fileName] = "ok"; + for(let index in results as any) { + let result = (results as any)[index]; + // hopefully this is enough to figure out which statement is a problem in the future + console.log(index + ". "+result.command +" "+(result.rowCount === null ? "" : result.rowCount)); } } - - //the schema didn't exist, so we need to create everything - //first, load up the initialize script - let initFile = fs.readFileSync('./initialize-schema-and-roles.sql', 'utf-8'); - - //now we need to replace the placeholders - //we'll also do a quick check that they actually exist, and throw an error if not, just in case someone broke the script - const lambdaUserPlaceholder = 'LAMBDA_ROLE_NAME_PLACEHOLDER'; - const lambdaPassPlaceholder = 'LAMBDA_ROLE_PASSWORD_PLACEHOLDER'; - const readonlyUserPlaceholder = 'READONLY_ROLE_NAME_PLACEHOLDER'; - const readonlyPassPlaceholder = 'READONLY_ROLE_PASSWORD_PLACEHOLDER'; - - if(initFile.indexOf(lambdaUserPlaceholder) < 0 || initFile.indexOf(lambdaPassPlaceholder) < 0 || - initFile.indexOf(readonlyUserPlaceholder) < 0 || initFile.indexOf(readonlyPassPlaceholder) < 0){ - throw new Error('DB initialization script is missing placeholders and cannot be run'); - } - - //run all the replacements - initFile = initFile.replace(new RegExp(lambdaUserPlaceholder, 'g'), lambda_username) - .replace(new RegExp(lambdaPassPlaceholder, 'g'), lambda_password) - .replace(new RegExp(readonlyUserPlaceholder, 'g'), readonly_username) - .replace(new RegExp(readonlyPassPlaceholder, 'g'), readonly_password); - - //execute the sql! - await dbClient.query(initFile); - - //load and run the table creation - let schemaFile = fs.readFileSync('./schema.sql', 'utf-8'); - await dbClient.query(schemaFile); - - //update the version table - await dbClient.query('INSERT INTO waze.application_version VALUES ($1, current_timestamp)', [current_version]); - //return success - console.log('Database intialization succeeded'); - return { response: "Database intialization succeeded" } - + if (whatWeExecuted.length == 0 ) { + return { response: "No changes"} + } else { + return { response: "Database intialization succeeded - " + whatWeExecuted.length+" scripts executed" }; + } } catch (err) { console.error(err); callback(err); - return err; + return { response: formatTerraformWarning(err) }; } - finally{ + finally { // CLOSE THAT CLIENT! await dbClient.end(); } }; -export {initializeDatabase} +export { initializeDatabase } //build a terraform-output-friendly warning message -function formatTerraformWarning(warningMessage:string):string { +function formatTerraformWarning(warningMessage: string): string { return ` WARNING! ********************* WARNING! ********************* WARNING! @@ -111,4 +160,4 @@ function formatTerraformWarning(warningMessage:string):string { WARNING! ********************* WARNING! ********************* WARNING! `; -} \ No newline at end of file +} diff --git a/code/lambda-functions/waze-db-initialize/webpack.config.js b/code/lambda-functions/waze-db-initialize/webpack.config.js index 872debf..ee800db 100644 --- a/code/lambda-functions/waze-db-initialize/webpack.config.js +++ b/code/lambda-functions/waze-db-initialize/webpack.config.js @@ -26,8 +26,13 @@ module.exports = { }, plugins: [ new CopyWebpackPlugin([ - '../../sql/initialize-schema-and-roles.sql', - '../../sql/schema.sql' + // cannot do wildcards because it messes with watches + '../../sql/2.01-00-initialize-schema-and-roles.sql', + '../../sql/2.01-01-create-tables.sql', + '../../sql/2.01-02-insert-defaults.sql', + '../../sql/3.0-00-add-readonly-role.sql', + '../../sql/3.0-01-add-columns.sql', + '../../sql/3.0-02-indexes.sql', ]), new ZipPlugin({ // OPTIONAL: defaults to the Webpack output path (above) diff --git a/code/sql/2.01-00-initialize-schema-and-roles.sql b/code/sql/2.01-00-initialize-schema-and-roles.sql new file mode 100644 index 0000000..0a817d5 --- /dev/null +++ b/code/sql/2.01-00-initialize-schema-and-roles.sql @@ -0,0 +1,38 @@ +/*************************************************************************************** +Note that this script is always run, so everything in it must be idempotent (rerunnable) +IE, use "if not exists" liberally + +Any errors will fail the script +***************************************************************************************/ + +-- This represents Schema as it was in the 2.0 version +-- in 2.0, this file was hand-edited and then hand-run +-- in 3.0, this is auto-run via lambda with placeholders replaced with terraform variables + +CREATE SCHEMA IF NOT EXISTS waze; + +-- create the lambda role +-- NOTE: don't change the placeholder text before provisioning the environment, as we replace it during initial install +-- this one will die if it already exists; no problem. +DO +$do$ +BEGIN + IF NOT EXISTS ( + SELECT -- SELECT list can stay empty for this + FROM pg_catalog.pg_roles + WHERE rolname = 'LAMBDA_ROLE_NAME_PLACEHOLDER') THEN + + CREATE ROLE LAMBDA_ROLE_NAME_PLACEHOLDER LOGIN PASSWORD 'LAMBDA_ROLE_PASSWORD_PLACEHOLDER'; + END IF; +END +$do$; + +-- this one gets it if it has already been created +ALTER ROLE LAMBDA_ROLE_NAME_PLACEHOLDER LOGIN PASSWORD 'LAMBDA_ROLE_PASSWORD_PLACEHOLDER'; + +-- setup permissions for the lambda role +GRANT ALL ON SCHEMA waze TO LAMBDA_ROLE_NAME_PLACEHOLDER; +ALTER DEFAULT PRIVILEGES IN SCHEMA waze GRANT ALL ON TABLES TO LAMBDA_ROLE_NAME_PLACEHOLDER; +ALTER DEFAULT PRIVILEGES IN SCHEMA waze GRANT SELECT, USAGE ON SEQUENCES TO LAMBDA_ROLE_NAME_PLACEHOLDER; +ALTER DEFAULT PRIVILEGES IN SCHEMA waze GRANT EXECUTE ON FUNCTIONS TO LAMBDA_ROLE_NAME_PLACEHOLDER; +ALTER DEFAULT PRIVILEGES IN SCHEMA waze GRANT USAGE ON TYPES TO LAMBDA_ROLE_NAME_PLACEHOLDER; diff --git a/code/sql/2.01-01-create-tables.sql b/code/sql/2.01-01-create-tables.sql new file mode 100644 index 0000000..535a3a0 --- /dev/null +++ b/code/sql/2.01-01-create-tables.sql @@ -0,0 +1,168 @@ +/*************************************************************************************** +Note that this script is always run, so everything in it must be idempotent (rerunnable) +IE, use "if not exists" liberally + +Any errors will fail the script +***************************************************************************************/ + +CREATE TABLE IF NOT EXISTS waze.data_files +( +"id" SERIAL PRIMARY KEY NOT NULL, +"start_time_millis" BIGINT NOT NULL, +"end_time_millis" BIGINT NOT NULL, +"start_time" TIMESTAMP, +"end_time" TIMESTAMP, +"date_created" TIMESTAMP, +"date_updated" TIMESTAMP, +"file_name" TEXT NOT NULL, +"json_hash" VARCHAR(40) NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS "IDX_UNIQUE_json_hash" +ON waze.data_files USING btree +(json_hash COLLATE pg_catalog."default") +TABLESPACE pg_default; + +CREATE TABLE IF NOT EXISTS waze.jams +( + "id" VARCHAR(40) PRIMARY KEY NOT NULL, + "uuid" TEXT NOT NULL, + "pub_millis" BIGINT NOT NULL, + "pub_utc_date" TIMESTAMP, + "start_node" TEXT, + "end_node" TEXT, + "road_type" INTEGER, + "street" TEXT, + "city" TEXT, + "country" TEXT, + "delay" INTEGER, + "speed" float4, + "speed_kmh" float4, + "length" INTEGER, + "turn_type" TEXT, + "level" INTEGER, + "blocking_alert_id" TEXT, + "line" JSONB, + "type" TEXT, + "turn_line" JSONB, + "datafile_id" BIGINT NOT NULL REFERENCES waze.data_files (id) +); + +CREATE TABLE IF NOT EXISTS waze.alerts +( + "id" VARCHAR(40) PRIMARY KEY NOT NULL, + "uuid" TEXT NOT NULL, + "pub_millis" BIGINT NOT NULL, + "pub_utc_date" TIMESTAMP, + "road_type" INTEGER, + "location" JSONB, + "street" TEXT, + "city" TEXT, + "country" TEXT, + "magvar" INTEGER, + "reliability" INTEGER, + "report_description" TEXT, + "report_rating" INTEGER, + "confidence" INTEGER, + "type" TEXT, + "subtype" TEXT, + "report_by_municipality_user" BOOLEAN, + "thumbs_up" INTEGER, + "jam_uuid" TEXT, + "datafile_id" BIGINT NOT NULL REFERENCES waze.data_files (id) +); + +CREATE TABLE IF NOT EXISTS waze.irregularities +( + "id" VARCHAR(40) PRIMARY KEY NOT NULL, + "uuid" TEXT NOT NULL, + "detection_date_millis" BIGINT NOT NULL, + "detection_date" TEXT, + "detection_utc_date" TIMESTAMP, + "update_date_millis" BIGINT NOT NULL, + "update_date" TEXT, + "update_utc_date" TIMESTAMP, + "street" TEXT, + "city" TEXT, + "country" TEXT, + "is_highway" BOOLEAN, + "speed" float4, + "regular_speed" float4, + "delay_seconds" INTEGER, + "seconds" INTEGER, + "length" INTEGER, + "trend" INTEGER, + "type" TEXT, + "severity" float4, + "jam_level" INTEGER, + "drivers_count" INTEGER, + "alerts_count" INTEGER, + "n_thumbs_up" INTEGER, + "n_comments" INTEGER, + "n_images" INTEGER, + "line" JSONB, + "cause_type" TEXT, + "start_node" TEXT, + "end_node" TEXT, + "datafile_id" BIGINT NOT NULL REFERENCES waze.data_files (id) +); + +CREATE TABLE IF NOT EXISTS waze.coordinate_type +( + "id" INTEGER PRIMARY KEY NOT NULL, + "type_name" TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS waze.coordinates +( + "id" VARCHAR(40) PRIMARY KEY NOT NULL, + "latitude" float8 NOT NULL, + "longitude" float8 NOT NULL, + "order" INTEGER NOT NULL, + "jam_id" VARCHAR(40) REFERENCES waze.jams (id), + "irregularity_id" VARCHAR(40) REFERENCES waze.irregularities (id), + "alert_id" VARCHAR(40) REFERENCES waze.alerts (id), + "coordinate_type_id" INTEGER REFERENCES waze.coordinate_type (id) +); + +CREATE TABLE IF NOT EXISTS waze.roads +( + "id" SERIAL PRIMARY KEY NOT NULL, + "value" INTEGER NOT NULL, + "name" VARCHAR(100) NOT NULL +); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * -- SELECT list can stay empty for this + FROM pg_catalog.pg_constraint + WHERE conname = 'roads_unique_combo') THEN + ALTER TABLE waze.roads + ADD CONSTRAINT roads_unique_combo UNIQUE(value, name); + END IF; +END +$$ +; + +CREATE TABLE IF NOT EXISTS waze.alert_types +( + "id" SERIAL PRIMARY KEY NOT NULL, + "type" TEXT NOT NULL, + "subtype" TEXT +); + +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT * -- SELECT list can stay empty for this + FROM pg_catalog.pg_constraint + WHERE conname = 'alert_types_unique_combo') THEN + ALTER TABLE waze.alert_types + ADD CONSTRAINT alert_types_unique_combo UNIQUE("type", "subtype"); + END IF; +END +$$ +; + +-- In version 2.0, performance indexes were NOT created other than IDX_UNIQUE_json_hash diff --git a/code/sql/2.01-02-insert-defaults.sql b/code/sql/2.01-02-insert-defaults.sql new file mode 100644 index 0000000..c29c219 --- /dev/null +++ b/code/sql/2.01-02-insert-defaults.sql @@ -0,0 +1,74 @@ +/*************************************************************************************** +Note that this script is always run, so everything in it must be idempotent (rerunnable) +IE, use if not exists liberally + +Any errors will fail the script +***************************************************************************************/ + +INSERT INTO waze.coordinate_type (id, type_name) VALUES (1, 'Line') ON CONFLICT DO NOTHING; +INSERT INTO waze.coordinate_type (id, type_name) VALUES (2, 'Turn Line') ON CONFLICT DO NOTHING; +INSERT INTO waze.coordinate_type (id, type_name) VALUES (3, 'Location') ON CONFLICT DO NOTHING; + +-- load roads +INSERT INTO waze.roads (value, name) VALUES (1, 'Streets') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (2, 'Primary Street') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (3, 'Freeways') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (4, 'Ramps') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (5, 'Trails') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (6, 'Primary') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (7, 'Secondary') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (8, '4X4 Trails') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (9, 'Walkway') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (10, 'Pedestrian') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (11, 'Exit') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (12, '?') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (13, '?') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (14, '4X4 Trails') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (15, 'Ferry crossing') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (16, 'Stairway') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (17, 'Private road') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (18, 'Railroads') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (19, 'Runway/Taxiway') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (20, 'Parking lot road') ON CONFLICT DO NOTHING; +INSERT INTO waze.roads (value, name) VALUES (21, 'Service road') ON CONFLICT DO NOTHING; + +-- load alert_types +INSERT INTO waze.alert_types (type, subtype) VALUES ('ACCIDENT', 'ACCIDENT_MINOR') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('ACCIDENT', 'ACCIDENT_MAJOR') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('ACCIDENT', 'NO_SUBTYPE') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('JAM', 'JAM_MODERATE_TRAFFIC') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('JAM', 'JAM_HEAVY_TRAFFIC') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('JAM', 'JAM_STAND_STILL_TRAFFIC') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('JAM', 'JAM_LIGHT_TRAFFIC') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('JAM', 'NO_SUBTYPE') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_ROAD') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_SHOULDER') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_WEATHER') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_ROAD_OBJECT') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_ROAD_POT_HOLE') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_ROAD_ROAD_KILL') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_SHOULDER_CAR_STOPPED') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_SHOULDER_ANIMALS') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_SHOULDER_MISSING_SIGN') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_WEATHER_FOG') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_WEATHER_HAIL') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_WEATHER_HEAVY_RAIN') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_WEATHER_HEAVY_SNOW') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_WEATHER_FLOOD') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_WEATHER_MONSOON') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_WEATHER_TORNADO') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_WEATHER_HEAT_WAVE') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_WEATHER_HURRICANE') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_WEATHER_FREEZING_RAIN') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_ROAD_LANE_CLOSED') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_ROAD_OIL') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_ROAD_ICE') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_ROAD_CONSTRUCTION') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_ROAD_CAR_STOPPED') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'HAZARD_ON_ROAD_TRAFFIC_LIGHT_FAULT') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('WEATHERHAZARD/HAZARD', 'NO_SUBTYPE') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('MISC', 'NO_SUBTYPE') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('CONSTRUCTION', 'NO_SUBTYPE') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('ROAD_CLOSED', 'ROAD_CLOSED_HAZARD') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('ROAD_CLOSED', 'ROAD_CLOSED_CONSTRUCTION') ON CONFLICT DO NOTHING; +INSERT INTO waze.alert_types (type, subtype) VALUES ('ROAD_CLOSED', 'ROAD_CLOSED_EVENT') ON CONFLICT DO NOTHING; diff --git a/code/sql/3.0-00-add-readonly-role.sql b/code/sql/3.0-00-add-readonly-role.sql new file mode 100644 index 0000000..f7b7566 --- /dev/null +++ b/code/sql/3.0-00-add-readonly-role.sql @@ -0,0 +1,30 @@ +/*************************************************************************************** +Note that this script is always run, so everything in it must be idempotent (rerunnable) +IE, use "if not exists" liberally + +Any errors will fail the script +***************************************************************************************/ + +-- Create read-only role +-- NOTE: don't change the placeholder text before provisioning the environment, as we replace it during initial install + +DO +$do$ +BEGIN + IF NOT EXISTS ( + SELECT -- SELECT list can stay empty for this + FROM pg_catalog.pg_roles + WHERE rolname = 'READONLY_ROLE_NAME_PLACEHOLDER') THEN + + CREATE ROLE READONLY_ROLE_NAME_PLACEHOLDER LOGIN PASSWORD 'READONLY_ROLE_PASSWORD_PLACEHOLDER'; + END IF; +END +$do$; + +-- this one gets it if it has already been created +ALTER ROLE READONLY_ROLE_NAME_PLACEHOLDER LOGIN PASSWORD 'LAMBDA_ROLE_PASSWORD_PLACEHOLDER'; + +GRANT USAGE ON SCHEMA waze TO READONLY_ROLE_NAME_PLACEHOLDER; +ALTER DEFAULT PRIVILEGES IN SCHEMA waze GRANT SELECT ON SEQUENCES TO READONLY_ROLE_NAME_PLACEHOLDER; +ALTER DEFAULT PRIVILEGES IN SCHEMA waze GRANT SELECT ON TABLES TO READONLY_ROLE_NAME_PLACEHOLDER; + diff --git a/code/sql/3.0-01-add-columns.sql b/code/sql/3.0-01-add-columns.sql new file mode 100644 index 0000000..8138511 --- /dev/null +++ b/code/sql/3.0-01-add-columns.sql @@ -0,0 +1,64 @@ +/*************************************************************************************** +Note that this script is always run, so everything in it must be idempotent (rerunnable) +IE, use "if not exists" liberally + +Any errors will fail the script +***************************************************************************************/ + +CREATE EXTENSION IF NOT EXISTS postgis; + +CREATE TABLE IF NOT EXISTS waze.application_version +( + version_number VARCHAR(30) PRIMARY KEY NOT NULL, + install_date TIMESTAMP NOT NULL +); + +-- all these are nullable. +ALTER TABLE waze.jams ADD COLUMN IF NOT EXISTS ns_direction varchar(3); +ALTER TABLE waze.jams ADD COLUMN IF NOT EXISTS ew_direction varchar(3); +ALTER TABLE waze.jams ADD COLUMN IF NOT EXISTS dayofweek int4 ; +ALTER TABLE waze.jams ADD COLUMN IF NOT EXISTS geom_line geometry(LINESTRING,4326); + +ALTER TABLE waze.alerts ADD COLUMN IF NOT EXISTS type_id INTEGER; +ALTER TABLE waze.alerts ADD COLUMN IF NOT EXISTS dayofweek INTEGER; +ALTER TABLE waze.alerts ADD COLUMN IF NOT EXISTS geom_point geometry(POINT,4326); + +CREATE OR REPLACE FUNCTION waze.line_to_geometry( line jsonb ) +RETURNS GEOMETRY +AS $$ +BEGIN + -- Set the result to be the same SRID as what's used in column definitions + RETURN( + SELECT ST_setsrid( + ST_MakeLine( + ST_MakePoint( + (line -> n ->> 'x')::NUMERIC, + (line -> n ->> 'y')::NUMERIC + ) + ),4326) + FROM generate_series(0, jsonb_array_length(line) - 1) AS n + ); +END +$$ LANGUAGE PLPGSQL IMMUTABLE; + +CREATE OR REPLACE FUNCTION waze.location_to_geometry( location jsonb ) +RETURNS GEOMETRY +AS $$ +BEGIN + -- Set the result to be the same SRID as what's used in column definitions + RETURN( + SELECT ST_setsrid( + ST_MakePoint( + (location ->> 'x')::NUMERIC, + (location ->> 'y')::NUMERIC + ) + ,4326) + ); +END +$$ LANGUAGE PLPGSQL IMMUTABLE; + +-- Version 2.0.1 didn't know about this table, so adding it now +INSERT INTO waze.application_version(version_number, install_date) VALUES('2.01',NOW()) ON CONFLICT DO NOTHING; + +-- and our own version +INSERT INTO waze.application_version(version_number, install_date) VALUES('3.0',NOW()) ON CONFLICT DO NOTHING; diff --git a/code/sql/3.0-02-indexes.sql b/code/sql/3.0-02-indexes.sql new file mode 100644 index 0000000..ae55e6e --- /dev/null +++ b/code/sql/3.0-02-indexes.sql @@ -0,0 +1,29 @@ +/*************************************************************************************** +Note that this script is always run, so everything in it must be idempotent (rerunnable) +IE, use "if not exists" liberally + +Any errors will fail the script +***************************************************************************************/ + +-- cannot create index concurrently in a batch statement, have to do one at a time +CREATE INDEX IF NOT EXISTS jams_pub_utc_date_idx ON waze.jams (pub_utc_date); +CREATE INDEX IF NOT EXISTS alerts_pub_utc_date_idx ON waze.alerts (pub_utc_date); +CREATE INDEX IF NOT EXISTS coordinates_jam_id_idx ON waze.coordinates (jam_id); +CREATE INDEX IF NOT EXISTS coordinates_latitude_idx ON waze.coordinates (latitude); +CREATE INDEX IF NOT EXISTS coordinates_longitude_idx ON waze.coordinates (longitude); +CREATE INDEX IF NOT EXISTS coordinates_order_idx ON waze.coordinates ("order"); +CREATE INDEX IF NOT EXISTS coordinates_coordinate_type_id_idx ON waze.coordinates (coordinate_type_id); +CREATE INDEX IF NOT EXISTS coordinates_alert_id_idx ON waze.coordinates (alert_id); +CREATE INDEX IF NOT EXISTS alerts_type_id_idx ON waze.alerts ("type_id"); +CREATE INDEX IF NOT EXISTS alerts_sub_type_idx ON waze.alerts (subtype); +CREATE INDEX IF NOT EXISTS alerts_type_idx ON waze.alerts ("type"); +CREATE INDEX IF NOT EXISTS jams_uuid_idx ON waze.jams (uuid); +CREATE INDEX IF NOT EXISTS alerts_uuid_idx ON waze.alerts (uuid); +CREATE INDEX IF NOT EXISTS jams_ns_direction_idx ON waze.jams (ns_direction); +CREATE INDEX IF NOT EXISTS jams_ew_direction_idx ON waze.jams (ew_direction); +CREATE INDEX IF NOT EXISTS jams_dayofweek_idx ON waze.jams (dayofweek); +CREATE INDEX IF NOT EXISTS alerts_dayofweek_idx ON waze.alerts (dayofweek); + +-- see indexes.sql which have statements for both drop and create of indexes +-- meant as a utility; a duplicate of what is run automatically + diff --git a/code/sql/indexes.sql b/code/sql/indexes.sql index 1c5f4a8..e81da5f 100644 --- a/code/sql/indexes.sql +++ b/code/sql/indexes.sql @@ -1,42 +1,43 @@ --- Use these scripts to get a list of current indexes, and create or drop them. +-- Use these scripts to get a list of current indexes, and create or drop them. -- select all indexes SELECT * FROM pg_indexes WHERE tablename like '%' and schemaname = 'waze' order by indexname ; -- create indexes (NOTE: THESE ARE CREATED IN THE BASE SCHEMA ALREADY AND MAINTAINED HERE FOR EASY DROP/RECREATE IF NECESSARY) -CREATE INDEX CONCURRENTLY waze.jams_pub_utc_date_idx ON waze.jams (pub_utc_date); -CREATE INDEX CONCURRENTLY waze.alerts_pub_utc_date_idx ON waze.alerts (pub_utc_date); -CREATE INDEX CONCURRENTLY waze.coordinates_jam_id_idx ON waze.coordinates (jam_id); -CREATE INDEX CONCURRENTLY waze.coordinates_latitude_idx ON waze.coordinates (latitude); -CREATE INDEX CONCURRENTLY waze.coordinates_longitude_idx ON waze.coordinates (longitude); -CREATE INDEX CONCURRENTLY waze.coordinates_order_idx ON waze.coordinates (order); -CREATE INDEX CONCURRENTLY waze.coordinates_coordinate_type_id_idx ON waze.coordinates (coordinate_type_id); -CREATE INDEX CONCURRENTLY waze.coordinates_alert_id_idx ON waze.coordinates (alert_id); -CREATE INDEX CONCURRENTLY waze.alerts_type_id_idx ON waze.alerts (type_id); -CREATE INDEX CONCURRENTLY waze.alerts_sub_type_idx ON waze.alerts (sub_type); -CREATE INDEX CONCURRENTLY waze.alerts_type_idx ON waze.alerts (type); -CREATE INDEX CONCURRENTLY waze.jams_uuid_idx ON waze.jams (uuid); -CREATE INDEX CONCURRENTLY waze.alerts_uuid_idx ON waze.alerts (uuid); -CREATE INDEX CONCURRENTLY waze.jams_ns_direction_idx ON waze.jams (ns_direction); -CREATE INDEX CONCURRENTLY waze.jams_ew_direction_idx ON waze.jams (ew_direction); -CREATE INDEX CONCURRENTLY waze.jams_dayofweek_idx ON waze.jams (dayofweek); -CREATE INDEX CONCURRENTLY waze.alerts_dayofweek_idx ON waze.alerts (dayofweek); +-- cannot create index concurrently in a batch statement, have to do one at a time +CREATE INDEX CONCURRENTLY IF NOT EXISTS jams_pub_utc_date_idx ON waze.jams (pub_utc_date); +CREATE INDEX CONCURRENTLY IF NOT EXISTS alerts_pub_utc_date_idx ON waze.alerts (pub_utc_date); +CREATE INDEX CONCURRENTLY IF NOT EXISTS coordinates_jam_id_idx ON waze.coordinates (jam_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS coordinates_latitude_idx ON waze.coordinates (latitude); +CREATE INDEX CONCURRENTLY IF NOT EXISTS coordinates_longitude_idx ON waze.coordinates (longitude); +CREATE INDEX CONCURRENTLY IF NOT EXISTS coordinates_order_idx ON waze.coordinates ("order"); +CREATE INDEX CONCURRENTLY IF NOT EXISTS coordinates_coordinate_type_id_idx ON waze.coordinates (coordinate_type_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS coordinates_alert_id_idx ON waze.coordinates (alert_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS alerts_type_id_idx ON waze.alerts ("type_id"); +CREATE INDEX CONCURRENTLY IF NOT EXISTS alerts_sub_type_idx ON waze.alerts (subtype); +CREATE INDEX CONCURRENTLY IF NOT EXISTS alerts_type_idx ON waze.alerts ("type"); +CREATE INDEX CONCURRENTLY IF NOT EXISTS jams_uuid_idx ON waze.jams (uuid); +CREATE INDEX CONCURRENTLY IF NOT EXISTS alerts_uuid_idx ON waze.alerts (uuid); +CREATE INDEX CONCURRENTLY IF NOT EXISTS jams_ns_direction_idx ON waze.jams (ns_direction); +CREATE INDEX CONCURRENTLY IF NOT EXISTS jams_ew_direction_idx ON waze.jams (ew_direction); +CREATE INDEX CONCURRENTLY IF NOT EXISTS jams_dayofweek_idx ON waze.jams (dayofweek); +CREATE INDEX CONCURRENTLY IF NOT EXISTS alerts_dayofweek_idx ON waze.alerts (dayofweek); -- drop indexes -DROP INDEX CONCURRENTLY waze.jams_pub_utc_date_idx; -DROP INDEX CONCURRENTLY waze.alerts_pub_utc_date_idx; -DROP INDEX CONCURRENTLY waze.coordinates_jam_id_idx; -DROP INDEX CONCURRENTLY waze.coordinates_latitude_idx; -DROP INDEX CONCURRENTLY waze.coordinates_longitude_idx; -DROP INDEX CONCURRENTLY waze.coordinates_order_idx; -DROP INDEX CONCURRENTLY waze.coordinates_coordinate_type_id_idx; -DROP INDEX CONCURRENTLY waze.coordinates_alert_id_idx; -DROP INDEX CONCURRENTLY waze.alerts_type_id_idx; -DROP INDEX CONCURRENTLY waze.alerts_sub_type_idx; -DROP INDEX CONCURRENTLY waze.alerts_type_idx; -DROP INDEX CONCURRENTLY waze.jams_uuid_idx; -DROP INDEX CONCURRENTLY waze.alerts_uuid_idx; -DROP INDEX CONCURRENTLY waze.jams_ns_direction_idx; -DROP INDEX CONCURRENTLY waze.jams_ew_direction_idx; -DROP INDEX CONCURRENTLY waze.jams_dayofweek_idx; -DROP INDEX CONCURRENTLY waze.alerts_dayofweek_idx; +DROP INDEX IF EXISTS waze.jams_pub_utc_date_idx; +DROP INDEX IF EXISTS waze.alerts_pub_utc_date_idx; +DROP INDEX IF EXISTS waze.coordinates_jam_id_idx; +DROP INDEX IF EXISTS waze.coordinates_latitude_idx; +DROP INDEX IF EXISTS waze.coordinates_longitude_idx; +DROP INDEX IF EXISTS waze.coordinates_order_idx; +DROP INDEX IF EXISTS waze.coordinates_coordinate_type_id_idx; +DROP INDEX IF EXISTS waze.coordinates_alert_id_idx; +DROP INDEX IF EXISTS waze.alerts_type_id_idx; +DROP INDEX IF EXISTS waze.alerts_sub_type_idx; +DROP INDEX IF EXISTS waze.alerts_type_idx; +DROP INDEX IF EXISTS waze.jams_uuid_idx; +DROP INDEX IF EXISTS waze.alerts_uuid_idx; +DROP INDEX IF EXISTS waze.jams_ns_direction_idx; +DROP INDEX IF EXISTS waze.jams_ew_direction_idx; +DROP INDEX IF EXISTS waze.jams_dayofweek_idx; +DROP INDEX IF EXISTS waze.alerts_dayofweek_idx; diff --git a/code/sql/initialize-schema-and-roles.sql b/code/sql/initialize-schema-and-roles.sql deleted file mode 100644 index 79bdcf8..0000000 --- a/code/sql/initialize-schema-and-roles.sql +++ /dev/null @@ -1,22 +0,0 @@ --- create the schema -CREATE SCHEMA waze; - --- create the lambda role --- NOTE: don't change the placeholder text before provisioning the environment, as we replace it during initial install -CREATE ROLE LAMBDA_ROLE_NAME_PLACEHOLDER LOGIN PASSWORD 'LAMBDA_ROLE_PASSWORD_PLACEHOLDER'; - --- setup permissions for the lambda role -GRANT ALL ON SCHEMA waze TO LAMBDA_ROLE_NAME_PLACEHOLDER; -ALTER DEFAULT PRIVILEGES IN SCHEMA waze GRANT ALL ON TABLES TO LAMBDA_ROLE_NAME_PLACEHOLDER; -ALTER DEFAULT PRIVILEGES IN SCHEMA waze GRANT SELECT, USAGE ON SEQUENCES TO LAMBDA_ROLE_NAME_PLACEHOLDER; -ALTER DEFAULT PRIVILEGES IN SCHEMA waze GRANT EXECUTE ON FUNCTIONS TO LAMBDA_ROLE_NAME_PLACEHOLDER; -ALTER DEFAULT PRIVILEGES IN SCHEMA waze GRANT USAGE ON TYPES TO LAMBDA_ROLE_NAME_PLACEHOLDER; - - --- Create read-only role --- NOTE: don't change the placeholder text before provisioning the environment, as we replace it during initial install -CREATE ROLE READONLY_ROLE_NAME_PLACEHOLDER LOGIN PASSWORD 'READONLY_ROLE_PASSWORD_PLACEHOLDER'; - -GRANT USAGE ON SCHEMA waze TO READONLY_ROLE_NAME_PLACEHOLDER; -ALTER DEFAULT PRIVILEGES IN SCHEMA waze GRANT SELECT ON SEQUENCES TO READONLY_ROLE_NAME_PLACEHOLDER; -ALTER DEFAULT PRIVILEGES IN SCHEMA waze GRANT SELECT ON TABLES TO READONLY_ROLE_NAME_PLACEHOLDER; diff --git a/infrastructure/terraform/modules/environment/outputs.tf b/infrastructure/terraform/modules/environment/outputs.tf index 1410f30..21d9650 100644 --- a/infrastructure/terraform/modules/environment/outputs.tf +++ b/infrastructure/terraform/modules/environment/outputs.tf @@ -31,4 +31,4 @@ output "data_in_queue_alarm_sns_topic_arn" { output "data_processing_dlq_sns_topic_arn" { value = "${aws_sns_topic.data_processing_dlq_sns_topic.arn}" description = "ARN of the SNS topic that will receive notifications when records are found in the dead letter queue" -} \ No newline at end of file +}