forked from aws-samples/aws-big-data-blog-dmscdc-walkthrough
-
Notifications
You must be signed in to change notification settings - Fork 0
/
DMSCDC_CloudTemplate_Source.yaml
221 lines (221 loc) · 6.6 KB
/
DMSCDC_CloudTemplate_Source.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
AWSTemplateFormatVersion: 2010-09-09
Parameters:
DBEngine:
Description: Choose the source DB engine to stream changes.
Type: String
AllowedValues:
- mysql
- oracle
- postgres
- mariadb
- aurora
- aurora-postgresql
- db2
- sybase
- sqlserver
DBServer:
Description: Hostname of the source DB.
Type: String
DBPort:
Description: Port of the source DB.
Type: Number
DBUser:
Description: Username for the source DB.
Type: String
DBPassword:
Description: Password for the source DB.
Type: String
NoEcho: true
DBName:
Description: Name of the source DB.
Type: String
DMSReplicationInstanceArn:
Description: Enter the ARN for the DMS Replication Instance which has been configured with network connectivity to your Source DB.
Type: String
DMSTableFilter:
Description: Choose the DMS table filter criteria.
Type: String
Default: "%"
AllowedPattern: ^[a-zA-Z0-9-_%]*$
ConstraintDescription: Must only contain upper/lower-case letters, numbers, -, _, or %.
DMSSchemaFilter:
Description: Choose the DMS schema filter criteria.
Type: String
Default: "%"
AllowedPattern: ^[a-zA-Z0-9-_%]*$
ConstraintDescription: Must only contain upper/lower-case letters, numbers, -, _, or %.
DMSBucket:
Description: Name of the Bucket to land the DMS data.
Type: String
DMSFolder:
Description: Prefix within bucket to store DMS data. Please omit the trailing /
Type: String
LakeBucket:
Description: Name of the Bucket to land the processed data.
Type: String
LakeFolder:
Description: Prefix within bucket to store processed data. Please omit the trailing /
Type: String
LakeDBName:
Description: Name to be used in analytical tools such as Athena and Redshift to reference these tables within the data lake.
Type: String
CDCSchedule:
Description: "CDC Execution Schedule Cron Expression. Default is every hour on the hour. For more info: https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html"
Type: String
Default: cron(0 0-23 * * ? *)
CrawlerSchedule:
Description: "Crawler Execution Schedule Cron Expression. Default is daily at 12:30 AM GMT: https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html"
Type: String
Default: cron(30 0 * * ? *)
Metadata:
AWS::CloudFormation::Interface:
ParameterGroups:
-
Label:
default: "DMS Source Database Configuration"
Parameters:
- DBEngine
- DBServer
- DBPort
- DBUser
- DBPassword
- DBName
-
Label:
default: "DMS Task Configuration"
Parameters:
- DMSReplicationInstanceArn
- DMSTableFilter
- DMSSchemaFilter
- DMSBucket
- DMSFolder
-
Label:
default: "Data Lake Configuration"
Parameters:
- LakeBucket
- LakeFolder
- LakeDBName
- CDCSchedule
- CrawlerSchedule
Resources:
DMSSourceEndpoint:
Type: AWS::DMS::Endpoint
Properties:
EndpointType: source
EngineName:
Ref: DBEngine
ServerName:
Ref: DBServer
Port:
Ref: DBPort
Username:
Ref: DBUser
Password:
Ref: DBPassword
DatabaseName:
Ref: DBName
DMSRawBucket:
Type: AWS::S3::Bucket
Properties:
BucketName:
Ref: DMSBucket
DMSTargetEndpoint:
Type: AWS::DMS::Endpoint
Properties:
EndpointType: target
EngineName: s3
ExtraConnectionAttributes : "dataFormat=parquet;parquetTimestampInMillisecond=true"
S3Settings :
BucketFolder:
Ref: DMSFolder
BucketName:
Ref: DMSBucket
ServiceAccessRoleArn: !Sub 'arn:aws:iam::${AWS::AccountId}:role/DMSCDC_Execution_Role'
DMSReplicationTask:
Type: AWS::DMS::ReplicationTask
Properties:
MigrationType: full-load-and-cdc
ReplicationInstanceArn:
Ref: DMSReplicationInstanceArn
SourceEndpointArn:
Ref: DMSSourceEndpoint
TableMappings:
Fn::Sub:
- "{\"rules\": [{\"rule-type\": \"selection\", \"rule-id\": \"1\", \"rule-action\": \"include\", \"object-locator\": {\"schema-name\": \"${schema}\", \"table-name\": \"${tables}\"}, \"rule-name\": \"1\"}]}"
- {schema: !Ref DMSSchemaFilter, tables: !Ref DMSTableFilter }
TargetEndpointArn:
Ref: DMSTargetEndpoint
ReplicationTaskSettings: "{\"Logging\": {\"EnableLogging\": true }}"
DependsOn:
- DMSTargetEndpoint
- DMSSourceEndpoint
- DMSRawBucket
DataLakeBucket:
Type: 'AWS::S3::Bucket'
Properties:
BucketName:
Ref: LakeBucket
Trigger:
Type: AWS::Glue::Trigger
Properties:
Type: SCHEDULED
Name: !Sub
- "DMSCDC-Trigger-${dbname}"
- {dbname: !Ref LakeDBName}
Schedule:
Ref: CDCSchedule
Actions:
- JobName: DMSCDC_Controller
Arguments:
'--bucket': !Ref DMSBucket
'--prefix': !Sub
- '${prefix}/'
- prefix: !Ref DMSFolder
'--out_bucket': !Ref LakeBucket
'--out_prefix': !Sub
- '${out_prefix}/'
- out_prefix: !Ref LakeFolder
GlueCrawler:
Type: AWS::Glue::Crawler
Properties:
Role: !Sub 'arn:aws:iam::${AWS::AccountId}:role/DMSCDC_Execution_Role'
Schedule:
ScheduleExpression:
Ref: CrawlerSchedule
DatabaseName:
Ref: LakeDBName
Targets:
S3Targets:
-
Path:
Fn::Join :
- "/"
-
- Ref : LakeBucket
- Ref : LakeFolder
InitDMS:
Type: Custom::InitDMS
DependsOn:
- DMSReplicationTask
Properties:
ServiceToken: !Sub 'arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:DMSCDC_InitDMS'
taskArn: !Ref DMSReplicationTask
sourceArn: !Ref DMSSourceEndpoint
targetArn: !Ref DMSTargetEndpoint
instanceArn: !Ref DMSReplicationInstanceArn
dmsBucket: !Ref DMSBucket
InitController:
Type: Custom::InitController
DependsOn:
- InitDMS
- Trigger
Properties:
ServiceToken: !Sub 'arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:DMSCDC_InitController'
dmsBucket: !Ref DMSBucket
dmsPath: !Sub
- "${prefix}/"
- { prefix: !Ref DMSFolder}
lakeBucket: !Ref LakeBucket
lakePath: !Ref LakeFolder
lakeDBName: !Ref LakeDBName