automation framework initial commit

Blogs/automation-framework/LambdaCode/ArtifactCleanup.py

@@ -0,0 +1,45 @@
+"""
+Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Information Block.
+
+Cleans up S3 versions and removes objects so bucket can be deleted
+API Triggers: Lambda invoke Custom Resource
+Services: S3
+Python 3.6 - AWS Lambda - Last Modified 07/01/2019
+"""
+
+import boto3
+import cfnresponse
+import json
+
+def lambda_handler(event, context):
+  try:
+    bucketcfn=event['ResourceProperties']['DestBucket']
+    responseData = {}
+
+    if event['RequestType'] == 'Create':
+      print('Create stack operation nothing will be done')
+    elif event['RequestType'] == 'Delete':
+      s3 = boto3.resource('s3')
+      bucket = s3.Bucket(bucketcfn)
+      bucket.object_versions.all().delete()
+      print('Delete stack in progress the bucket is emptied')
+    elif event['RequestType'] == 'Update':
+      print('Update stack')
+    cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, '')
+  except Exception as e:
+    print(e)
+    cfnresponse.send(event, context, cfnresponse.FAILURE, {}, '')

Blogs/automation-framework/LambdaCode/CheckDmsEndpoints.py

@@ -0,0 +1,73 @@
+"""
+Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Information Block.
+
+Checks the DMS endpoint during creation to see if they were successful
+API Triggers: Lambda invoke Custom Resource
+Services: DMS
+Python 3.6 - AWS Lambda - Last Modified 07/01/2019
+"""
+
+import boto3
+import json
+import cfnresponse
+
+def lambda_handler(event, context):
+   rep_arn = event['ResourceProperties']['ReplicationInstanceArn']
+   source_arn = event['ResourceProperties']['SourceArn']
+   target_arn = event['ResourceProperties']['TargetArn']
+   responseData = {}
+
+   try:
+       if event['RequestType'] == 'Delete':
+           print('Nothing will be done on DeleteStack call')
+       else:
+           print('This is a %s event' %(event['RequestType']))
+           source_status = dms_status(source_arn,rep_arn)
+           target_status = dms_status(target_arn,rep_arn)
+           if 'successful' in source_status:
+               print('Source endpoint was successfully tested')
+               if 'successful' in target_status:
+                   print('Target endpoint was successfully tested')
+               else:
+                   print('Target endpoint was not tested. Please test connection with replication instance')
+           else:
+               print('Source endpoint was not tested. Please test connection with replication instance.')
+       cfnresponse.send(event, context, cfnresponse.SUCCESS, responseData, '')
+   except Exception as e:
+       print(e)
+       cfnresponse.send(event, context, cfnresponse.FAILURE, {}, '')
+
+def dms_status(endpoint, rep_inst):
+    try:
+        dms_client = boto3.client('dms')
+        response = dms_client.describe_connections(
+        Filters=[
+            {
+                'Name': 'endpoint-arn',
+                'Values': [endpoint]
+            },
+            {
+                'Name': 'replication-instance-arn',
+                'Values': [rep_inst]
+            }
+            ]
+        )
+        e = response['Connections'][0]['Status']
+    except Exception as e:
+        print('Error occured with replication instance: %s and endpoint: %s' %(endpoint,rep_inst))
+        print ('Exception is %s' %(e))
+    return e

Blogs/automation-framework/LambdaCode/CheckDmsReplicationStatus.py

@@ -0,0 +1,101 @@
+"""
+Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+Permission is hereby granted, free of charge, to any person obtaining a copy of this
+software and associated documentation files (the "Software"), to deal in the Software
+without restriction, including without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A
+PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Information Block.
+
+Checks the messages from SNS to see if its from DMS or CodePipeline
+If CodePipeline it retrieves execution token and stores in SSM parameter store
+If DMS then based on task status approves or rejects the pipeline
+API Triggers: Lambda invoke Custom Resource
+Services: DMS, CodePipeline, SNS
+Python 3.6 - AWS Lambda - Last Modified 07/01/2019
+"""
+import boto3
+import json
+import os
+ssm = boto3.client('ssm')
+sns = boto3.client('sns')
+codepipeline = boto3.client('codepipeline')
+ssm_parameter = os.environ['codepipeline_token']
+pipeline_name = os.environ['pipeline_name']
+task_name = os.environ['dms_task']
+topic = os.environ['notify_topic']
+def lambda_handler(event, context):
+    print(event)
+    str_subject = event['Records'][0]['Sns']['Subject']
+    if 'APPROVAL NEEDED' in str_subject:
+        print('This is a Codepipeline approval action')
+        str_sns = event['Records'][0]['Sns']['Message']
+        sns_msg = json.loads(str_sns)
+        pipeline = sns_msg['approval']['pipelineName']
+        stage = sns_msg['approval']['stageName']
+        action = sns_msg['approval']['actionName']
+        token = sns_msg['approval']['token']
+        approve_param ="pipelineName='%s',stageName='%s',actionName='%s',token='%s'" % ( pipeline , stage , action , token)
+        print(approve_param)
+        response = ssm.put_parameter(Name=ssm_parameter,
+            Value=approve_param,
+            Type='String',
+            Overwrite=True
+            )
+    elif 'DMS' in str_subject:
+        print('This is a message from DMS')
+        str_sns = event['Records'][0]['Sns']['Message']
+        if 'attempt' in str_sns:
+            print(str_sns)
+            print('Event notification nothing will be done')
+        else:
+            sns_msg = json.loads(str_sns)
+            print(sns_msg['Event Message'])
+            dms_status = sns_msg['Event Message']
+            if 'STOPPED_AFTER_FULL_LOAD' in dms_status:
+                print('DMS task replication is stopped after full load, proceeding to put an approval in Codepipeline')
+                result_pipeline('Approved')
+            elif 'started' in dms_status:
+                print('Lambda will do nothing at this step as the task is started')
+            elif 'Create' in dms_status:
+                print('Lambda will do nothing at this step as the task is created')
+            elif 'FAIL' in dms_status.upper():
+                status = 'DMS task failed. Please check the task'
+                print(status)
+                subj = 'Status Update on DMS Task ' + os.environ['dms_task']
+                sns.publish(TopicArn = topic, Message = status, Subject = subj)
+                result_pipeline('Rejected')
+            else:
+                status = 'DMS task did not stop or errored out after full load. Please check the task'
+                print(status)
+                subj = 'Status Update on DMS Task ' + os.environ['dms_task']
+                sns.publish(TopicArn = topic, Message = status, Subject = subj)
+                result_pipeline('Rejected')
+
+    else:
+        print('This message is from neither Codepipeline Approval or DMS event. Nothing will be done')
+
+
+def result_pipeline(event):
+    print('Getting Codepipeline parameters from SSM to put a %s' %(event))
+    codepipeline_params = ssm.get_parameter(Name=ssm_parameter)['Parameter']['Value'].split("'")
+    print(codepipeline_params)
+    result_reponse = codepipeline.put_approval_result(
+        pipelineName=codepipeline_params[1],
+        stageName=codepipeline_params[3],
+        actionName=codepipeline_params[5],
+        result={
+            'summary': event,
+            'status': event
+        },
+        token=codepipeline_params[7]
+    )
+    print(result_reponse)

Blogs/automation-framework/README.md

@@ -0,0 +1,55 @@
+# Automation Framework for AWS Database Migration Service
+
+This code allows you to create an automated framework to migrate relational databases.
+
+## Steps:
+The steps below are an example of how the setup was taken place. The exact commands will not work and need to be changed as per your requirements.
+1. Make sure you have a source or create source Oracle database in RDS as per the command:
+```
+aws rds create-db-instance --db-instance-identifier sourcedb06 --engine oracle-ee --engine-version 12.1.0.2.v16 --storage-type gp2 --allocated-storage 100 --db-instance-class db.t2.large --master-username master --master-user-password Passw0rd --vpc-security-group-ids sg-3bbe8340
+```
+2. If you created a brand new source you can populate source based on https://github.com/aws-samples/aws-database-migration-samples/tree/master/oracle/sampledb/v1 . Your local client must have sqlplus installed.
+```
+git clone git clone https://github.com/aws-samples/aws-database-migration-samples.git
+cd aws-database-migration-samples/oracle/sampledb/v1
+echo "exit;" >> install-rds.sql
+sqlplus $DB_USER/$DB_PASSWORD@$DB_HOST/$DB_NAME @install-rds.sql
+wget https://raw.githubusercontent.com/nagmesh/aws-database-migration-samples/master/oracle/sampledb/v1/schema/fix.sql
+sqlplus $DB_USER/$DB_PASSWORD@$DB_HOST/$DB_NAME @fix.sql
+```
+3. Create target Aurora Postgres RDS as per the command:
+```
+aws rds create-db-cluster --db-cluster-identifier targetdb06 --engine aurora-postgresql --engine-version 10.7 --master-username master --master-user-password Passw0rd --vpc-security-group-ids sg-3bbe8340
+
+aws rds create-db-instance --db-instance-identifier targetdb06 --db-cluster-identifier targetdb06 --db-instance-class db.t3.medium --engine aurora-postgresql --engine-version 10.7
+```
+4. Make sure you have a replication instance or create a DMS replication instance using the command below:
+```
+aws dms create-replication-instance --replication-instance-identifier sample-inst-01 --vpc-security-group-ids sg-3bbe8340 --replication-instance-class dms.t2.medium
+````
+5. Create the DMS endpoint for source and target
+```
+aws dms create-endpoint --endpoint-identifier sourcedb06 --endpoint-type source --engine-name oracle --username master --password Passw0rd --port 1521 --server-name sourcedb06.cmbi8axdpp1y.us-west-2.rds.amazonaws.com --database-name ORCL
+
+aws dms create-endpoint --endpoint-identifier targetdb06 --endpoint-type target --engine-name aurora-postgresql --username master --password Passw0rd --port 5432 --server-name targetdb06.cmbi8axdpp1y.us-west-2.rds.amazonaws.com --database-name postgres
+```
+6. Test the source database connection to replication instance:
+```
+aws dms test-connection --endpoint-arn $(aws dms describe-endpoints --filters Name=endpoint-id,Values=sourcedb06 --query 'Endpoints[0].EndpointArn' --output text) --replication-instance-arn $(aws dms describe-replication-instances --filters Name=replication-instance-id,Values=sample-inst-01 --query 'ReplicationInstances[0].ReplicationInstanceArn' --output text)
+
+aws dms wait test-connection-succeeds --filters Name=endpoint-arn,Values=$(aws dms describe-endpoints --filters Name=endpoint-id,Values=sourcedb06 --query 'Endpoints[0].EndpointArn' --output text),Name=replication-instance-arn,Values=$(aws dms describe-replication-instances --filters Name=replication-instance-id,Values=sample-inst-01 --query 'ReplicationInstances[0].ReplicationInstanceArn' --output text)
+
+aws dms test-connection --endpoint-arn $(aws dms describe-endpoints --filters Name=endpoint-id,Values=targetdb06 --query 'Endpoints[0].EndpointArn' --output text) --replication-instance-arn $(aws dms describe-replication-instances --filters Name=replication-instance-id,Values=sample-inst-01 --query 'ReplicationInstances[0].ReplicationInstanceArn' --output text)
+
+aws dms wait test-connection-succeeds --filters Name=endpoint-arn,Values=$(aws dms describe-endpoints --filters Name=endpoint-id,Values=targetdb06 --query 'Endpoints[0].EndpointArn' --output text),Name=replication-instance-arn,Values=$(aws dms describe-replication-instances --filters Name=replication-instance-id,Values=sample-inst-01 --query 'ReplicationInstances[0].ReplicationInstanceArn' --output text)
+```
+7. Verify that the output of the commands below are successful
+```
+aws dms describe-connections --filters Name=endpoint-arn,Values=$(aws dms describe-endpoints --filters Name=endpoint-id,Values=sourcedb06 --query 'Endpoints[0].EndpointArn' --output text),Name=replication-instance-arn,Values=$(aws dms describe-replication-instances --filters Name=replication-instance-id,Values=sample-inst-01 --query 'ReplicationInstances[0].ReplicationInstanceArn' --output text) --query 'Connections[0].Status' --output text
+
+aws dms describe-connections --filters Name=endpoint-arn,Values=$(aws dms describe-endpoints --filters Name=endpoint-id,Values=targetdb06 --query 'Endpoints[0].EndpointArn' --output text),Name=replication-instance-arn,Values=$(aws dms describe-replication-instances --filters Name=replication-instance-id,Values=sample-inst-01 --query 'ReplicationInstances[0].ReplicationInstanceArn' --output text) --query 'Connections[0].Status' --output text
+```
+8. Upload the S3Files/test.zip to an S3 bucket
+9. Launch the template at CloudFormationTemplate/automation-migration.yml using the security group, replication arn and endpoints arns that you created above.
+
+To further confgure this you can change the S3Files/setup-target-schema.yml to change the actions in SetupTarget Stage and S3Files/pre-CDC-build.yml to change actions in the pre-CDC state.

Blogs/automation-framework/S3Files/create-empty-schema.sql

@@ -0,0 +1,2 @@
+\i create-table.sql
+\i create-index.sql