feat: add support to handle buckets for replication

src/_get-buckets.js

@@ -0,0 +1,58 @@
+// eslint-disable-next-line
+let aws = require('aws-sdk') // Assume AWS-SDK is installed via Arc
+let { toLogicalID } = require('@architect/utils')
+let { updater } = require('@architect/utils')
+let getMultiRegionOptions = require('./_get-multi-region-options')
+let getArcOptions = require('./_get-arc-options')
+
+module.exports = async (arc, stage, dryRun) => {
+  const { primaryRegion, bucketNames } = getMultiRegionOptions(arc)
+  const { appName, currentRegion } = getArcOptions(arc)
+
+  const update = updater('MultiRegion')
+  update.start(`Fetching buckets in (${currentRegion})...`)
+
+  let ssmPrimary = new aws.SSM({ region: primaryRegion })
+
+  let buckets = []
+  for (let privacy of [ 'private', 'public' ]) {
+    for (let bucketName of bucketNames[privacy]) {
+      let PhysicalBucketName // aka the physical table name
+      try {
+        // Arc app physical table names are stored in SSM service discovery
+        let Name = `/${toLogicalID(appName)}${toLogicalID(stage)}/storage-${privacy}/${bucketName}`
+        let { Parameter } = await ssmPrimary.getParameter({ Name }).promise()
+        PhysicalBucketName = Parameter.Value
+        buckets.push({
+          arn: `arn:aws:s3:::${bucketName}`,
+          logicalName: bucketName,
+          physicalName: PhysicalBucketName,
+          privacy: privacy
+        })
+      }
+      catch (err) {
+        if (err.name === 'ParameterNotFound') {
+          const message = `${bucketName} not found on ${currentRegion}`
+          if (dryRun) {
+            update.warn(`${message} (Maybe because is a dry-run)`)
+          }
+          else {
+            update.error(message)
+            throw (err)
+          }
+        }
+        else {
+          throw (err)
+        }
+      }
+    }
+  }
+
+  update.done(`Buckets fetched (${currentRegion})`)
+
+  update.status(`Fetched buckets in (${currentRegion})`, buckets.map((bucket) => {
+    return `${bucket.privacy} ${bucket.privacy == 'public' ? '...' : '..'} ${bucket.logicalName}`
+  }))
+
+  return buckets
+}

src/_get-multi-region-options.js

@@ -1,4 +1,5 @@
-module.exports = (multiRegion) => {
+module.exports = (arc) => {
+  const multiRegion = arc['multi-region']
   if (!Array.isArray(multiRegion)) {
     throw ReferenceError('Invalid multi region params')
   }
@@ -21,5 +22,17 @@ module.exports = (multiRegion) => {
     throw ReferenceError('Invalid multi region params: Missing replica regions')
   }
 
-  return { primaryRegion, replicaRegions }
+  let bucketNames = { public: [], private: [] };
+  [ 'public', 'private' ].forEach((privacy) => {
+    if (arc[`storage-${privacy}`]) {
+      bucketNames[privacy] = bucketNames[privacy].concat(arc[`storage-${privacy}`])
+    }
+  })
+  const skipBucketsIndex = multiRegion.findIndex((param) => param['skip-buckets'])
+  if (skipBucketsIndex >= 0 && Array.isArray(multiRegion[skipBucketsIndex]['skip-buckets'])) {
+    bucketNames.public = bucketNames.public - multiRegion[skipBucketsIndex]['skip-buckets']
+    bucketNames.private = bucketNames.private - multiRegion[skipBucketsIndex]['skip-buckets']
+  }
+
+  return { primaryRegion, replicaRegions, bucketNames }
 }

src/_get-replicated-tables.js

@@ -2,12 +2,17 @@
 let aws = require('aws-sdk') // Assume AWS-SDK is installed via Arc
 let { toLogicalID } = require('@architect/utils')
 let { updater } = require('@architect/utils')
+let getMultiRegionOptions = require('./_get-multi-region-options')
+let getArcOptions = require('./_get-arc-options')
+
+module.exports = async (arc, stage, dryRun) => {
+  const { primaryRegion } = getMultiRegionOptions(arc)
+  const { appName, currentRegion } = getArcOptions(arc)
 
-module.exports = async (arc, stage, dryRun, appName, primaryRegion, currentRegion) => {
   const update = updater('MultiRegion')
   update.start(`Fetching replica tables in the replica region (${currentRegion})...`)
 
-  let dynamoReplica = new aws.DynamoDB({ region: currentRegion }) // The current region is a replica region
+  let dynamoReplica = new aws.DynamoDB({ region: currentRegion })
   let ssmPrimary = new aws.SSM({ region: primaryRegion })
 
   let tableNames = []

src/_update_replication.js

@@ -2,8 +2,13 @@
 let aws = require('aws-sdk') // Assume AWS-SDK is installed via Arc
 let { toLogicalID } = require('@architect/utils')
 let { updater } = require('@architect/utils')
+let getMultiRegionOptions = require('./_get-multi-region-options')
+let getArcOptions = require('./_get-arc-options')
+
+module.exports = async (arc, stage, dryRun) => {
+  const { primaryRegion, replicaRegions } = getMultiRegionOptions(arc)
+  const { appName, currentRegion } = getArcOptions(arc)
 
-module.exports = async (arc, stage, dryRun, appName, primaryRegion, replicaRegions, currentRegion) => {
   const update = updater('MultiRegion')
   const start = Date.now()
   const done = () => update.done(`Replication updated in ${(Date.now() - start) / 1000} seconds`)

src/index.js

@@ -3,48 +3,48 @@ let getMultiRegionOptions = require('./_get-multi-region-options')
 let getArcOptions = require('./_get-arc-options')
 let updateReplication = require('./_update_replication')
 let getReplicatedTables = require('./_get-replicated-tables')
+let getBuckets = require('./_get-buckets')
 
 module.exports = {
   deploy: {
     start: async ({ arc, cloudformation, stage, dryRun }) => {
+      const cfn = cloudformation
       const multiRegion = arc['multi-region']
-      if (!multiRegion) return cloudformation
+      if (!multiRegion) return cfn
 
-      const { primaryRegion, replicaRegions } = getMultiRegionOptions(multiRegion)
-      const { appName, currentRegion } = getArcOptions(arc)
+      const { primaryRegion, replicaRegions } = getMultiRegionOptions(arc)
+      const { currentRegion } = getArcOptions(arc)
 
       if (primaryRegion == currentRegion) return
 
       if (!replicaRegions.includes(currentRegion)) {
         throw Error(`The following region is not included in replica regions: ${currentRegion}`)
       }
 
-      let tables = await getReplicatedTables(arc, stage, dryRun, appName, primaryRegion, currentRegion)
-      let index = cloudformation.Resources.Role.Properties.Policies
+      const tables = await getReplicatedTables(arc, stage, dryRun)
+      let dynamoPolicyIndex = cfn.Resources.Role.Properties.Policies
         .findIndex(item => item.PolicyName === 'ArcDynamoPolicy')
-
-      // Delete old DynamoDB Policies
-      cloudformation.Resources.Role.Properties
-        .Policies[index].PolicyDocument.Statement[0].Resource = []
+      let dynamoPolicyDoc = cfn.Resources.Role.Properties.Policies[dynamoPolicyIndex]
+        .PolicyDocument.Statement[0]
 
       tables.forEach(({ arn, logicalName, physicalName }) => {
+        const ID = toLogicalID(logicalName)
         // Delete old DynamoDB Tables
-        let originalResourceTableName = `${toLogicalID(logicalName)}Table`
-        delete cloudformation.Resources[originalResourceTableName]
+        const originalResourceName = `${ID}Table`
+        delete cfn.Resources[originalResourceName]
         // Delete old SSM Parameters
-        let originalResourceParamName = `${toLogicalID(logicalName)}Param`
-        delete cloudformation.Resources[originalResourceParamName]
+        const originalResourceParamName = `${ID}Param`
+        delete cfn.Resources[originalResourceParamName]
+        // Delete old DynamoDB Policies
+        dynamoPolicyDoc.Resource = dynamoPolicyDoc.Resource.filter((resource) => {
+          return !resource['Fn::Sub'] || resource['Fn::Sub'][1].tablename.Ref != originalResourceName
+        })
 
         // Add new DynamoDB Policies
-        cloudformation.Resources.Role.Properties
-          .Policies[index].PolicyDocument.Statement[0].Resource.push(
-            arn,
-            `${arn}/*`,
-            `${arn}/stream/*`,
-          )
+        dynamoPolicyDoc.Resource.push(arn, `${arn}/*`, `${arn}/stream/*`)
         // Add new SSM Parameter for Global Table
-        let resourceName = `${toLogicalID(logicalName)}GlobalTableParam`
-        cloudformation.Resources[resourceName] = {
+        let resourceName = `${ID}GlobalTableParam`
+        cfn.Resources[resourceName] = {
           Type: 'AWS::SSM::Parameter',
           Properties: {
             Type: 'String',
@@ -61,20 +61,65 @@ module.exports = {
         }
       })
 
-      return cloudformation
+      const buckets = await getBuckets(arc, stage, dryRun)
+
+      buckets.forEach(({ arn, logicalName, physicalName, privacy }) => {
+        const bucketPolicyDoc = cfn.Resources[
+          `P${privacy.slice(1)}StorageMacroPolicy`
+        ].Properties.PolicyDocument.Statement[0]
+        const ID = toLogicalID(logicalName)
+        // Delete old Bucket
+        const originalResourceName = `${ID}Bucket`
+        delete cfn.Resources[originalResourceName]
+        // Delete old SSM Parameters
+        const originalResourceParamName = `${ID}Param`
+        delete cfn.Resources[originalResourceParamName]
+        // Delete old Bucket Policies
+        bucketPolicyDoc.Resource = bucketPolicyDoc.Resource.filter((resource) => {
+          return !resource['Fn::Sub'] || resource['Fn::Sub'][1].bucket.Ref != originalResourceName
+        })
+
+        // Add IAM policy for least-priv runtime access
+        bucketPolicyDoc.Resource.push(arn, `${arn}/*`)
+        // Add new SSM Parameter for runtime discovery
+        let resourceName = `${ID}BucketParam`
+        cfn.Resources[resourceName] = {
+          Type: 'AWS::SSM::Parameter',
+          Properties: {
+            Type: 'String',
+            Name: {
+              'Fn::Sub': [
+                '/${AWS::StackName}/storage-${privacy}/${bucketname}',
+                {
+                  bucketname: logicalName,
+                  privacy
+                }
+              ]
+            },
+            Value: physicalName
+          }
+        }
+        // Replace bucket env var on all Lambda functions
+        Object.keys(cfn.Resources).forEach((k) => {
+          let BUCKET = `ARC_STORAGE_PUBLIC_${logicalName.replace(/-/g, '_').toUpperCase()}`
+          if (cfn.Resources[k].Type === 'AWS::Serverless::Function') {
+            cfn.Resources[k].Properties.Environment.Variables[BUCKET] = physicalName
+          }
+        })
+      })
+
+      return cfn
     },
     end: async ({ arc, cloudformation, stage, dryRun }) => {
       const multiRegion = arc['multi-region']
       if (!multiRegion) return cloudformation
 
-      const { primaryRegion, replicaRegions } = getMultiRegionOptions(multiRegion)
-      const { appName, currentRegion } = getArcOptions(arc)
+      const { primaryRegion } = getMultiRegionOptions(arc)
+      const { currentRegion } = getArcOptions(arc)
 
       if (primaryRegion != currentRegion) return
 
-      await updateReplication(
-        arc, stage, dryRun, appName, primaryRegion, replicaRegions, currentRegion
-      )
+      await updateReplication(arc, stage, dryRun)
 
       return cloudformation
     }