Removed streambot dependency. Added detailed Instructions.

README.md

@@ -185,3 +185,101 @@ Usage: incremental-record-history <tableinfo> <s3url> <recordkey>
 # Read the history of a single record
 $ incremental-record-history us-east-1/my-table s3://dynamodb-backups/incremental '{"id":"abc"}'
 ```
+
+### Instructions to run:
+##### Streambot dependency removed since it is deprecated
+
+The code could be deployed on a lambda for incremental backups to S3 and for replication across DynamoDB tables across regions and accounts. 
+
+Initial setup:
+```
+$ git clone https://github.com/subbuvenk94/dynamodb-replicator.git
+$ cd dynamodb-replicator
+$ npm install
+```
+After the dependent packages are installed, zip the contents of the root directory. Deploy the zip on to AWS Lambda console with the handler as ```index.backup``` or ```index.replicate``` based on your required functionality.
+
+The Lambda console lets us set the environment variables. Refer to this:
+```
+## Backup Functionality
+# Target bucket on S3
+BackupBucket=
+# Prefix for the data inside the bucket (Optional)
+BackupPrefix=
+# Region for the bucket (Optional)
+BackupRegion=
+
+## Replication Functionality
+# Key/Secret for a different AWS account write (Optional)
+ReplicaAccessKeyId=
+ReplicaSecretAccesseKey=
+# Name of replica table
+ReplicaTable=
+# Replica table region (NOTE: We have to manually create this table for the same key)
+ReplicaRegion=
+#Endpoint for replica table (Optional)
+ReplicaEndpoint=
+```
+Enable triggers for the tables on which these operations are made with view type ```New and old images - both the new and the old images of the item```. Note that the 'Triggers' option on DynamoDB tables are available based on region. Create the trigger for the existing Lambda function that we just created.
+
+For tables in which data already exists, the existing data has to be first replicated before we could do the incremental replication. Refer to [diff-tables](#diff-tables) for replication backfill. Similarly, for we can perform [incremental backfill](#incremental-backfill).
+
+The Lambda function would need permissions to perform the incremental backup or replication tasks. The IAM policy that should be created to award this permission could be set as:
+```
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Action": [
+                "logs:CreateLogGroup",
+                "logs:CreateLogStream",
+                "logs:PutLogEvents"
+            ],
+            "Resource": "arn:aws:logs:*:*:*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:ListBucket"
+            ],
+            "Resource": [
+                "arn:aws:s3:::<BACKUP_BUCKET_NAME>"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "s3:GetObject",
+                "s3:PutObject",
+                "s3:DeleteObject"
+            ],
+            "Resource": [
+                "arn:aws:s3:::<BACKUP_BUCKET_NAME>/*"
+            ]
+        },
+        {
+            "Effect": "Allow",
+            "Action": "lambda:InvokeFunction",
+            "Resource": "*"
+        },
+        {
+            "Effect": "Allow",
+            "Action": [
+                "dynamodb:DescribeStream",
+                "dynamodb:GetRecords",
+                "dynamodb:GetShardIterator",
+                "dynamodb:ListStreams",
+                "dynamodb:DescribeTable",
+                "dynamodb:PutItem",
+                "dynamodb:BatchWriteItem",
+                "dynamodb:Scan"
+            ],
+            "Resource": "*"
+        }
+    ]
+}
+```
+NOTE: Edit the BACKUP_BUCKET_NAME to the name of the backup bucket.
+
+After this setup, any changes to the tables should trigger the Lambda which would perform the incremental backup or the table repication. Your changes would be visible in the respective table or bucket.

index.js

@@ -3,16 +3,11 @@ var Dyno = require('dyno');
 var queue = require('queue-async');
 var crypto = require('crypto');
 var https = require('https');
-var streambot = require('streambot');
 
 module.exports.replicate = replicate;
-module.exports.streambotReplicate = streambot(function(event, callback) {
-    replicate(event, {}, callback);
-});
+
 module.exports.backup = incrementalBackup;
-module.exports.streambotBackup = streambot(function(event, callback) {
-    incrementalBackup(event, {}, callback);
-});
+
 module.exports.snapshot = require('./s3-snapshot');
 module.exports.agent = new https.Agent({
     keepAlive: true,
@@ -68,6 +63,7 @@ function replicate(event, context, callback) {
     var params = { RequestItems: {} };
     params.RequestItems[process.env.ReplicaTable] = Object.keys(allRecords).map(function(key) {
         var change = allRecords[key].pop();
+        console.log('change = ', change);
         if (change.eventName === 'INSERT' || change.eventName === 'MODIFY') {
             return {
                 PutRequest: { Item: Dyno.deserialize(JSON.stringify(change.dynamodb.NewImage)) }
@@ -173,7 +169,7 @@ function incrementalBackup(event, context, callback) {
 
     q.awaitAll(function(err) {
         if (err) throw err;
-        callback();
+        callback(null, "Sucessful back up");
     });
 
     function backupRecord(changes, callback) {
@@ -193,6 +189,8 @@ function incrementalBackup(event, context, callback) {
                 };
 
                 var req = change.eventName === 'REMOVE' ? 'deleteObject' : 'putObject';
+                console.log('Event = ', change.eventName);
+                console.log('Change = ', JSON.stringify(change.dynamodb));
                 if (req === 'putObject') params.Body = JSON.stringify(change.dynamodb.NewImage);
 
                 s3[req](params, function(err) {
@@ -223,4 +221,4 @@ function incrementalBackup(event, context, callback) {
             callback();
         });
     }
-}
+}