From 10f3d8ef5d5f35daba71dfcd2a40178d26e08340 Mon Sep 17 00:00:00 2001 From: limcross Date: Tue, 8 Nov 2022 23:06:22 -0300 Subject: [PATCH] feat: add support to handle buckets for replication --- src/_get-buckets.js | 58 +++++++++++++++++++ src/_get-multi-region-options.js | 17 +++++- src/_get-replicated-tables.js | 9 ++- src/_update_replication.js | 7 ++- src/index.js | 99 +++++++++++++++++++++++--------- 5 files changed, 158 insertions(+), 32 deletions(-) create mode 100644 src/_get-buckets.js diff --git a/src/_get-buckets.js b/src/_get-buckets.js new file mode 100644 index 0000000..b436b14 --- /dev/null +++ b/src/_get-buckets.js @@ -0,0 +1,58 @@ +// eslint-disable-next-line +let aws = require('aws-sdk') // Assume AWS-SDK is installed via Arc +let { toLogicalID } = require('@architect/utils') +let { updater } = require('@architect/utils') +let getMultiRegionOptions = require('./_get-multi-region-options') +let getArcOptions = require('./_get-arc-options') + +module.exports = async (arc, stage, dryRun) => { + const { primaryRegion, bucketNames } = getMultiRegionOptions(arc) + const { appName, currentRegion } = getArcOptions(arc) + + const update = updater('MultiRegion') + update.start(`Fetching buckets in (${currentRegion})...`) + + let ssmPrimary = new aws.SSM({ region: primaryRegion }) + + let buckets = [] + for (let privacy of [ 'private', 'public' ]) { + for (let bucketName of bucketNames[privacy]) { + let PhysicalBucketName // aka the physical table name + try { + // Arc app physical table names are stored in SSM service discovery + let Name = `/${toLogicalID(appName)}${toLogicalID(stage)}/storage-${privacy}/${bucketName}` + let { Parameter } = await ssmPrimary.getParameter({ Name }).promise() + PhysicalBucketName = Parameter.Value + buckets.push({ + arn: `arn:aws:s3:::${bucketName}`, + logicalName: bucketName, + physicalName: PhysicalBucketName, + privacy: privacy + }) + } + catch (err) { + if (err.name === 'ParameterNotFound') { + const message = `${bucketName} not found on ${currentRegion}` + if (dryRun) { + update.warn(`${message} (Maybe because is a dry-run)`) + } + else { + update.error(message) + throw (err) + } + } + else { + throw (err) + } + } + } + } + + update.done(`Buckets fetched (${currentRegion})`) + + update.status(`Fetched buckets in (${currentRegion})`, buckets.map((bucket) => { + return `${bucket.privacy} ${bucket.privacy == 'public' ? '...' : '..'} ${bucket.logicalName}` + })) + + return buckets +} diff --git a/src/_get-multi-region-options.js b/src/_get-multi-region-options.js index ed6cd3e..b3059f2 100644 --- a/src/_get-multi-region-options.js +++ b/src/_get-multi-region-options.js @@ -1,4 +1,5 @@ -module.exports = (multiRegion) => { +module.exports = (arc) => { + const multiRegion = arc['multi-region'] if (!Array.isArray(multiRegion)) { throw ReferenceError('Invalid multi region params') } @@ -21,5 +22,17 @@ module.exports = (multiRegion) => { throw ReferenceError('Invalid multi region params: Missing replica regions') } - return { primaryRegion, replicaRegions } + let bucketNames = { public: [], private: [] }; + [ 'public', 'private' ].forEach((privacy) => { + if (arc[`storage-${privacy}`]) { + bucketNames[privacy] = bucketNames[privacy].concat(arc[`storage-${privacy}`]) + } + }) + const skipBucketsIndex = multiRegion.findIndex((param) => param['skip-buckets']) + if (skipBucketsIndex >= 0 && Array.isArray(multiRegion[skipBucketsIndex]['skip-buckets'])) { + bucketNames.public = bucketNames.public - multiRegion[skipBucketsIndex]['skip-buckets'] + bucketNames.private = bucketNames.private - multiRegion[skipBucketsIndex]['skip-buckets'] + } + + return { primaryRegion, replicaRegions, bucketNames } } diff --git a/src/_get-replicated-tables.js b/src/_get-replicated-tables.js index 31279dd..9c611ae 100644 --- a/src/_get-replicated-tables.js +++ b/src/_get-replicated-tables.js @@ -2,12 +2,17 @@ let aws = require('aws-sdk') // Assume AWS-SDK is installed via Arc let { toLogicalID } = require('@architect/utils') let { updater } = require('@architect/utils') +let getMultiRegionOptions = require('./_get-multi-region-options') +let getArcOptions = require('./_get-arc-options') + +module.exports = async (arc, stage, dryRun) => { + const { primaryRegion } = getMultiRegionOptions(arc) + const { appName, currentRegion } = getArcOptions(arc) -module.exports = async (arc, stage, dryRun, appName, primaryRegion, currentRegion) => { const update = updater('MultiRegion') update.start(`Fetching replica tables in the replica region (${currentRegion})...`) - let dynamoReplica = new aws.DynamoDB({ region: currentRegion }) // The current region is a replica region + let dynamoReplica = new aws.DynamoDB({ region: currentRegion }) let ssmPrimary = new aws.SSM({ region: primaryRegion }) let tableNames = [] diff --git a/src/_update_replication.js b/src/_update_replication.js index b23fe54..80bc082 100644 --- a/src/_update_replication.js +++ b/src/_update_replication.js @@ -2,8 +2,13 @@ let aws = require('aws-sdk') // Assume AWS-SDK is installed via Arc let { toLogicalID } = require('@architect/utils') let { updater } = require('@architect/utils') +let getMultiRegionOptions = require('./_get-multi-region-options') +let getArcOptions = require('./_get-arc-options') + +module.exports = async (arc, stage, dryRun) => { + const { primaryRegion, replicaRegions } = getMultiRegionOptions(arc) + const { appName, currentRegion } = getArcOptions(arc) -module.exports = async (arc, stage, dryRun, appName, primaryRegion, replicaRegions, currentRegion) => { const update = updater('MultiRegion') const start = Date.now() const done = () => update.done(`Replication updated in ${(Date.now() - start) / 1000} seconds`) diff --git a/src/index.js b/src/index.js index 22cfe32..05a050e 100644 --- a/src/index.js +++ b/src/index.js @@ -3,15 +3,17 @@ let getMultiRegionOptions = require('./_get-multi-region-options') let getArcOptions = require('./_get-arc-options') let updateReplication = require('./_update_replication') let getReplicatedTables = require('./_get-replicated-tables') +let getBuckets = require('./_get-buckets') module.exports = { deploy: { start: async ({ arc, cloudformation, stage, dryRun }) => { + const cfn = cloudformation const multiRegion = arc['multi-region'] - if (!multiRegion) return cloudformation + if (!multiRegion) return cfn - const { primaryRegion, replicaRegions } = getMultiRegionOptions(multiRegion) - const { appName, currentRegion } = getArcOptions(arc) + const { primaryRegion, replicaRegions } = getMultiRegionOptions(arc) + const { currentRegion } = getArcOptions(arc) if (primaryRegion == currentRegion) return @@ -19,32 +21,30 @@ module.exports = { throw Error(`The following region is not included in replica regions: ${currentRegion}`) } - let tables = await getReplicatedTables(arc, stage, dryRun, appName, primaryRegion, currentRegion) - let index = cloudformation.Resources.Role.Properties.Policies + const tables = await getReplicatedTables(arc, stage, dryRun) + let dynamoPolicyIndex = cfn.Resources.Role.Properties.Policies .findIndex(item => item.PolicyName === 'ArcDynamoPolicy') - - // Delete old DynamoDB Policies - cloudformation.Resources.Role.Properties - .Policies[index].PolicyDocument.Statement[0].Resource = [] + let dynamoPolicyDoc = cfn.Resources.Role.Properties.Policies[dynamoPolicyIndex] + .PolicyDocument.Statement[0] tables.forEach(({ arn, logicalName, physicalName }) => { + const ID = toLogicalID(logicalName) // Delete old DynamoDB Tables - let originalResourceTableName = `${toLogicalID(logicalName)}Table` - delete cloudformation.Resources[originalResourceTableName] + const originalResourceName = `${ID}Table` + delete cfn.Resources[originalResourceName] // Delete old SSM Parameters - let originalResourceParamName = `${toLogicalID(logicalName)}Param` - delete cloudformation.Resources[originalResourceParamName] + const originalResourceParamName = `${ID}Param` + delete cfn.Resources[originalResourceParamName] + // Delete old DynamoDB Policies + dynamoPolicyDoc.Resource = dynamoPolicyDoc.Resource.filter((resource) => { + return !resource['Fn::Sub'] || resource['Fn::Sub'][1].tablename.Ref != originalResourceName + }) // Add new DynamoDB Policies - cloudformation.Resources.Role.Properties - .Policies[index].PolicyDocument.Statement[0].Resource.push( - arn, - `${arn}/*`, - `${arn}/stream/*`, - ) + dynamoPolicyDoc.Resource.push(arn, `${arn}/*`, `${arn}/stream/*`) // Add new SSM Parameter for Global Table - let resourceName = `${toLogicalID(logicalName)}GlobalTableParam` - cloudformation.Resources[resourceName] = { + let resourceName = `${ID}GlobalTableParam` + cfn.Resources[resourceName] = { Type: 'AWS::SSM::Parameter', Properties: { Type: 'String', @@ -61,20 +61,65 @@ module.exports = { } }) - return cloudformation + const buckets = await getBuckets(arc, stage, dryRun) + + buckets.forEach(({ arn, logicalName, physicalName, privacy }) => { + const bucketPolicyDoc = cfn.Resources[ + `P${privacy.slice(1)}StorageMacroPolicy` + ].Properties.PolicyDocument.Statement[0] + const ID = toLogicalID(logicalName) + // Delete old Bucket + const originalResourceName = `${ID}Bucket` + delete cfn.Resources[originalResourceName] + // Delete old SSM Parameters + const originalResourceParamName = `${ID}Param` + delete cfn.Resources[originalResourceParamName] + // Delete old Bucket Policies + bucketPolicyDoc.Resource = bucketPolicyDoc.Resource.filter((resource) => { + return !resource['Fn::Sub'] || resource['Fn::Sub'][1].bucket.Ref != originalResourceName + }) + + // Add IAM policy for least-priv runtime access + bucketPolicyDoc.Resource.push(arn, `${arn}/*`) + // Add new SSM Parameter for runtime discovery + let resourceName = `${ID}BucketParam` + cfn.Resources[resourceName] = { + Type: 'AWS::SSM::Parameter', + Properties: { + Type: 'String', + Name: { + 'Fn::Sub': [ + '/${AWS::StackName}/storage-${privacy}/${bucketname}', + { + bucketname: logicalName, + privacy + } + ] + }, + Value: physicalName + } + } + // Replace bucket env var on all Lambda functions + Object.keys(cfn.Resources).forEach((k) => { + let BUCKET = `ARC_STORAGE_PUBLIC_${logicalName.replace(/-/g, '_').toUpperCase()}` + if (cfn.Resources[k].Type === 'AWS::Serverless::Function') { + cfn.Resources[k].Properties.Environment.Variables[BUCKET] = physicalName + } + }) + }) + + return cfn }, end: async ({ arc, cloudformation, stage, dryRun }) => { const multiRegion = arc['multi-region'] if (!multiRegion) return cloudformation - const { primaryRegion, replicaRegions } = getMultiRegionOptions(multiRegion) - const { appName, currentRegion } = getArcOptions(arc) + const { primaryRegion } = getMultiRegionOptions(arc) + const { currentRegion } = getArcOptions(arc) if (primaryRegion != currentRegion) return - await updateReplication( - arc, stage, dryRun, appName, primaryRegion, replicaRegions, currentRegion - ) + await updateReplication(arc, stage, dryRun) return cloudformation }