From bc9c2bec5f4890f7cb0341d8c8f68d8b8011b25f Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 23 May 2020 12:46:18 -0400 Subject: [PATCH 01/13] Suppress readonly properties from DMS ReplicationTaskSettings --- aws/diff_suppress_funcs.go | 25 +++ aws/diff_suppress_funcs_test.go | 195 +++++++++++++++++++++++ aws/resource_aws_dms_replication_task.go | 2 +- 3 files changed, 221 insertions(+), 1 deletion(-) diff --git a/aws/diff_suppress_funcs.go b/aws/diff_suppress_funcs.go index 4d3a82aa766..6fcfb9318d7 100644 --- a/aws/diff_suppress_funcs.go +++ b/aws/diff_suppress_funcs.go @@ -137,3 +137,28 @@ func suppressEquivalentTime(k, old, new string, d *schema.ResourceData) bool { return oldTime.Equal(newTime) } + +func suppressDmsReplicationTaskSettingsDiffs(k, old, new string, d *schema.ResourceData) bool { + var oldData map[string]interface{} + if err := json.Unmarshal([]byte(old), &oldData); err != nil { + return false + } + + controlTablesSettings, ok := oldData["ControlTablesSettings"].(map[string]interface{}) + if ok { + delete(controlTablesSettings, "historyTimeslotInMinutes") + } + + logging, ok := oldData["Logging"].(map[string]interface{}) + if ok { + delete(logging, "CloudWatchLogGroup") + delete(logging, "CloudWatchLogStream") + } + + cleanedOld, err := json.Marshal(oldData) + if err != nil { + return false + } + + return suppressEquivalentJsonDiffs(k, string(cleanedOld), new, d) +} diff --git a/aws/diff_suppress_funcs_test.go b/aws/diff_suppress_funcs_test.go index 3096217fd9d..2a903bec241 100644 --- a/aws/diff_suppress_funcs_test.go +++ b/aws/diff_suppress_funcs_test.go @@ -264,3 +264,198 @@ Outputs: } } } + +func TestSuppressRoute53ZoneNameWithTrailingDot(t *testing.T) { + testCases := []struct { + old string + new string + equivalent bool + }{ + { + old: "example.com", + new: "example.com", + equivalent: true, + }, + { + old: "example.com.", + new: "example.com.", + equivalent: true, + }, + { + old: "example.com.", + new: "example.com", + equivalent: true, + }, + { + old: "example.com", + new: "example.com.", + equivalent: true, + }, + { + old: ".", + new: "", + equivalent: false, + }, + { + old: "", + new: ".", + equivalent: false, + }, + { + old: ".", + new: ".", + equivalent: true, + }, + } + + for i, tc := range testCases { + value := suppressRoute53ZoneNameWithTrailingDot("test_property", tc.old, tc.new, nil) + + if tc.equivalent && !value { + t.Fatalf("expected test case %d to be equivalent", i) + } + + if !tc.equivalent && value { + t.Fatalf("expected test case %d to not be equivalent", i) + } + } +} + +func TestSuppressDmsReplicationTaskSettingsDiffs(t *testing.T) { + testCases := []struct { + description string + equivalent bool + old string + new string + }{ + { + description: `JSON change`, + equivalent: false, + old: ` +{ + "ControlTablesSettings": { + "ControlSchema": "dms_control", + "HistoryTimeslotInMinutes": 10, + "HistoryTableEnabled": true, + "SuspendedTablesTableEnabled": true, + "StatusTableEnabled": false + } +} +`, + new: ` +{ + "ControlTablesSettings": { + "ControlSchema": "dms_control", + "HistoryTimeslotInMinutes": 5, + "HistoryTableEnabled": true, + "SuspendedTablesTableEnabled": true, + "StatusTableEnabled": false + } +} +`, + }, + { + description: `historyTimeslotInMinutes ignored`, + equivalent: true, + old: ` +{ + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "ControlSchema": "dms_control", + "HistoryTimeslotInMinutes": 5, + "HistoryTableEnabled": true, + "SuspendedTablesTableEnabled": true, + "StatusTableEnabled": false + } +} +`, + new: ` +{ + "ControlTablesSettings": { + "ControlSchema": "dms_control", + "HistoryTimeslotInMinutes": 5, + "HistoryTableEnabled": true, + "SuspendedTablesTableEnabled": true, + "StatusTableEnabled": false + } +} +`, + }, + { + description: `CloudWatchLogGroup, CloudWatchLogStream ignored`, + equivalent: true, + old: ` +{ + "Logging": { + "EnableLogging": true, + "LogComponents": [ + { + "Id": "SOURCE_UNLOAD", + "Severity": "LOGGER_SEVERITY_DEFAULT" + }, + { + "Id": "TARGET_LOAD", + "Severity": "LOGGER_SEVERITY_DEFAULT" + }, + { + "Id": "SOURCE_CAPTURE", + "Severity": "LOGGER_SEVERITY_DEFAULT" + }, + { + "Id": "TARGET_APPLY", + "Severity": "LOGGER_SEVERITY_DEFAULT" + }, + { + "Id": "TASK_MANAGER", + "Severity": "LOGGER_SEVERITY_DEFAULT" + } + ], + "CloudWatchLogGroup": "dms-tasks-task-name", + "CloudWatchLogStream": "dms-task-ABCDEF123456789" + } +} +`, + new: ` +{ + "Logging": { + "EnableLogging": true, + "LogComponents": [ + { + "Id": "SOURCE_UNLOAD", + "Severity": "LOGGER_SEVERITY_DEFAULT" + }, + { + "Id": "TARGET_LOAD", + "Severity": "LOGGER_SEVERITY_DEFAULT" + }, + { + "Id": "SOURCE_CAPTURE", + "Severity": "LOGGER_SEVERITY_DEFAULT" + }, + { + "Id": "TARGET_APPLY", + "Severity": "LOGGER_SEVERITY_DEFAULT" + }, + { + "Id": "TASK_MANAGER", + "Severity": "LOGGER_SEVERITY_DEFAULT" + } + ] + } +} +`, + }, + } + + for _, tc := range testCases { + value := suppressDmsReplicationTaskSettingsDiffs("test_property", tc.old, tc.new, nil) + + if tc.equivalent && !value { + t.Fatalf("expected test case (%s) to be equivalent", tc.description) + } + + if !tc.equivalent && value { + t.Fatalf("expected test case (%s) to not be equivalent", tc.description) + } + } +} diff --git a/aws/resource_aws_dms_replication_task.go b/aws/resource_aws_dms_replication_task.go index ac1e9b05d92..a48806d2155 100644 --- a/aws/resource_aws_dms_replication_task.go +++ b/aws/resource_aws_dms_replication_task.go @@ -61,7 +61,7 @@ func resourceAwsDmsReplicationTask() *schema.Resource { Type: schema.TypeString, Optional: true, ValidateFunc: validation.StringIsJSON, - DiffSuppressFunc: suppressEquivalentJsonDiffs, + DiffSuppressFunc: suppressDmsReplicationTaskSettingsDiffs, }, "source_endpoint_arn": { Type: schema.TypeString, From 35b33af5cc66f736a4978d8db28b2a5489090097 Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 23 May 2020 13:44:12 -0400 Subject: [PATCH 02/13] Move logic from diff suppression to state --- aws/diff_suppress_funcs.go | 25 ---- aws/diff_suppress_funcs_test.go | 139 ----------------------- aws/resource_aws_dms_replication_task.go | 37 +++++- 3 files changed, 34 insertions(+), 167 deletions(-) diff --git a/aws/diff_suppress_funcs.go b/aws/diff_suppress_funcs.go index 6fcfb9318d7..4d3a82aa766 100644 --- a/aws/diff_suppress_funcs.go +++ b/aws/diff_suppress_funcs.go @@ -137,28 +137,3 @@ func suppressEquivalentTime(k, old, new string, d *schema.ResourceData) bool { return oldTime.Equal(newTime) } - -func suppressDmsReplicationTaskSettingsDiffs(k, old, new string, d *schema.ResourceData) bool { - var oldData map[string]interface{} - if err := json.Unmarshal([]byte(old), &oldData); err != nil { - return false - } - - controlTablesSettings, ok := oldData["ControlTablesSettings"].(map[string]interface{}) - if ok { - delete(controlTablesSettings, "historyTimeslotInMinutes") - } - - logging, ok := oldData["Logging"].(map[string]interface{}) - if ok { - delete(logging, "CloudWatchLogGroup") - delete(logging, "CloudWatchLogStream") - } - - cleanedOld, err := json.Marshal(oldData) - if err != nil { - return false - } - - return suppressEquivalentJsonDiffs(k, string(cleanedOld), new, d) -} diff --git a/aws/diff_suppress_funcs_test.go b/aws/diff_suppress_funcs_test.go index 2a903bec241..12c6a1a1af8 100644 --- a/aws/diff_suppress_funcs_test.go +++ b/aws/diff_suppress_funcs_test.go @@ -320,142 +320,3 @@ func TestSuppressRoute53ZoneNameWithTrailingDot(t *testing.T) { } } } - -func TestSuppressDmsReplicationTaskSettingsDiffs(t *testing.T) { - testCases := []struct { - description string - equivalent bool - old string - new string - }{ - { - description: `JSON change`, - equivalent: false, - old: ` -{ - "ControlTablesSettings": { - "ControlSchema": "dms_control", - "HistoryTimeslotInMinutes": 10, - "HistoryTableEnabled": true, - "SuspendedTablesTableEnabled": true, - "StatusTableEnabled": false - } -} -`, - new: ` -{ - "ControlTablesSettings": { - "ControlSchema": "dms_control", - "HistoryTimeslotInMinutes": 5, - "HistoryTableEnabled": true, - "SuspendedTablesTableEnabled": true, - "StatusTableEnabled": false - } -} -`, - }, - { - description: `historyTimeslotInMinutes ignored`, - equivalent: true, - old: ` -{ - "ControlTablesSettings": { - "historyTimeslotInMinutes": 5, - "ControlSchema": "dms_control", - "HistoryTimeslotInMinutes": 5, - "HistoryTableEnabled": true, - "SuspendedTablesTableEnabled": true, - "StatusTableEnabled": false - } -} -`, - new: ` -{ - "ControlTablesSettings": { - "ControlSchema": "dms_control", - "HistoryTimeslotInMinutes": 5, - "HistoryTableEnabled": true, - "SuspendedTablesTableEnabled": true, - "StatusTableEnabled": false - } -} -`, - }, - { - description: `CloudWatchLogGroup, CloudWatchLogStream ignored`, - equivalent: true, - old: ` -{ - "Logging": { - "EnableLogging": true, - "LogComponents": [ - { - "Id": "SOURCE_UNLOAD", - "Severity": "LOGGER_SEVERITY_DEFAULT" - }, - { - "Id": "TARGET_LOAD", - "Severity": "LOGGER_SEVERITY_DEFAULT" - }, - { - "Id": "SOURCE_CAPTURE", - "Severity": "LOGGER_SEVERITY_DEFAULT" - }, - { - "Id": "TARGET_APPLY", - "Severity": "LOGGER_SEVERITY_DEFAULT" - }, - { - "Id": "TASK_MANAGER", - "Severity": "LOGGER_SEVERITY_DEFAULT" - } - ], - "CloudWatchLogGroup": "dms-tasks-task-name", - "CloudWatchLogStream": "dms-task-ABCDEF123456789" - } -} -`, - new: ` -{ - "Logging": { - "EnableLogging": true, - "LogComponents": [ - { - "Id": "SOURCE_UNLOAD", - "Severity": "LOGGER_SEVERITY_DEFAULT" - }, - { - "Id": "TARGET_LOAD", - "Severity": "LOGGER_SEVERITY_DEFAULT" - }, - { - "Id": "SOURCE_CAPTURE", - "Severity": "LOGGER_SEVERITY_DEFAULT" - }, - { - "Id": "TARGET_APPLY", - "Severity": "LOGGER_SEVERITY_DEFAULT" - }, - { - "Id": "TASK_MANAGER", - "Severity": "LOGGER_SEVERITY_DEFAULT" - } - ] - } -} -`, - }, - } - - for _, tc := range testCases { - value := suppressDmsReplicationTaskSettingsDiffs("test_property", tc.old, tc.new, nil) - - if tc.equivalent && !value { - t.Fatalf("expected test case (%s) to be equivalent", tc.description) - } - - if !tc.equivalent && value { - t.Fatalf("expected test case (%s) to not be equivalent", tc.description) - } - } -} diff --git a/aws/resource_aws_dms_replication_task.go b/aws/resource_aws_dms_replication_task.go index a48806d2155..3b09c9f8e6a 100644 --- a/aws/resource_aws_dms_replication_task.go +++ b/aws/resource_aws_dms_replication_task.go @@ -1,6 +1,7 @@ package aws import ( + "encoding/json" "fmt" "log" "strconv" @@ -61,7 +62,7 @@ func resourceAwsDmsReplicationTask() *schema.Resource { Type: schema.TypeString, Optional: true, ValidateFunc: validation.StringIsJSON, - DiffSuppressFunc: suppressDmsReplicationTaskSettingsDiffs, + DiffSuppressFunc: suppressEquivalentJsonDiffs, }, "source_endpoint_arn": { Type: schema.TypeString, @@ -283,16 +284,20 @@ func resourceAwsDmsReplicationTaskDelete(d *schema.ResourceData, meta interface{ func resourceAwsDmsReplicationTaskSetState(d *schema.ResourceData, task *dms.ReplicationTask) error { d.SetId(aws.StringValue(task.ReplicationTaskIdentifier)) - d.Set("migration_type", task.MigrationType) d.Set("replication_instance_arn", task.ReplicationInstanceArn) d.Set("replication_task_arn", task.ReplicationTaskArn) d.Set("replication_task_id", task.ReplicationTaskIdentifier) - d.Set("replication_task_settings", task.ReplicationTaskSettings) d.Set("source_endpoint_arn", task.SourceEndpointArn) d.Set("table_mappings", task.TableMappings) d.Set("target_endpoint_arn", task.TargetEndpointArn) + cleanedReplicationTaskSettings, err := cleanReadOnlyReplicationTaskSettings(*task.ReplicationTaskSettings) + if err != nil { + return err + } + d.Set("replication_task_settings", cleanedReplicationTaskSettings) + return nil } @@ -328,3 +333,29 @@ func resourceAwsDmsReplicationTaskStateRefreshFunc( return v, *v.ReplicationTasks[0].Status, nil } } + +func cleanReadOnlyReplicationTaskSettings(settings string) (*string, error) { + var settingsData map[string]interface{} + if err := json.Unmarshal([]byte(settings), &settingsData); err != nil { + return nil, err + } + + controlTablesSettings, ok := settingsData["ControlTablesSettings"].(map[string]interface{}) + if ok { + delete(controlTablesSettings, "historyTimeslotInMinutes") + } + + logging, ok := settingsData["Logging"].(map[string]interface{}) + if ok { + delete(logging, "CloudWatchLogGroup") + delete(logging, "CloudWatchLogStream") + } + + cleanedSettings, err := json.Marshal(settingsData) + if err != nil { + return nil, err + } + + cleanedSettingsString := string(cleanedSettings) + return &cleanedSettingsString, nil +} From 1229f9ab7d0f726828c126d1b2abeb62ae28dcbe Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Sat, 23 May 2020 21:37:18 -0400 Subject: [PATCH 03/13] Improve DMS acceptance test --- aws/resource_aws_dms_replication_task_test.go | 81 ++++++++++++++++++- 1 file changed, 79 insertions(+), 2 deletions(-) diff --git a/aws/resource_aws_dms_replication_task_test.go b/aws/resource_aws_dms_replication_task_test.go index 62ac0facf4c..03c9f312315 100644 --- a/aws/resource_aws_dms_replication_task_test.go +++ b/aws/resource_aws_dms_replication_task_test.go @@ -28,6 +28,11 @@ func TestAccAWSDmsReplicationTask_basic(t *testing.T) { resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), ), }, + { + Config: dmsReplicationTaskConfig(randId), + PlanOnly: true, + ExpectNonEmptyPlan: false, + }, { ResourceName: resourceName, ImportState: true, @@ -109,12 +114,48 @@ data "aws_partition" "current" {} data "aws_region" "current" {} +data "aws_iam_policy_document" "dms_assume_role_policy_document" { + statement { + actions = ["sts:AssumeRole"] + + principals { + identifiers = ["dms.amazonaws.com"] + type = "Service" + } + } +} + +resource "aws_iam_role" "dms_vpc_role" { + assume_role_policy = data.aws_iam_policy_document.dms_assume_role_policy_document.json + name = "dms-vpc-role" +} + +resource "aws_iam_role" "dms_cloudwatch_logs_role" { + assume_role_policy = data.aws_iam_policy_document.dms_assume_role_policy_document.json + name = "dms-cloudwatch-logs-role" +} + +resource "aws_iam_role_policy_attachment" "dms_vpc_access_policy" { + role = aws_iam_role.dms_vpc_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonDMSVPCManagementRole" +} + +resource "aws_iam_role_policy_attachment" "dms_cloudwatch_logs_access_policy" { + role = aws_iam_role.dms_cloudwatch_logs_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonDMSCloudWatchLogsRole" +} + resource "aws_vpc" "dms_vpc" { cidr_block = "10.1.0.0/16" tags = { Name = "terraform-testacc-dms-replication-task-%[1]s" } + + depends_on = [ + aws_iam_role_policy_attachment.dms_vpc_access_policy, + aws_iam_role_policy_attachment.dms_cloudwatch_logs_access_policy, + ] } resource "aws_subnet" "dms_subnet_1" { @@ -183,7 +224,7 @@ resource "aws_dms_replication_task" "dms_replication_task" { migration_type = "full-load" replication_instance_arn = aws_dms_replication_instance.dms_replication_instance.replication_instance_arn replication_task_id = "tf-test-dms-replication-task-%[1]s" - replication_task_settings = "{\"TargetMetadata\":{\"TargetSchema\":\"\",\"SupportLobs\":true,\"FullLobMode\":false,\"LobChunkSize\":0,\"LimitedSizeLobMode\":true,\"LobMaxSize\":32,\"LoadMaxFileSize\":0,\"ParallelLoadThreads\":0,\"BatchApplyEnabled\":false},\"FullLoadSettings\":{\"FullLoadEnabled\":true,\"ApplyChangesEnabled\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"CreatePkAfterFullLoad\":false,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"ResumeEnabled\":false,\"ResumeMinTableSize\":100000,\"ResumeOnlyClusteredPKTables\":true,\"MaxFullLoadSubTasks\":8,\"TransactionConsistencyTimeout\":600,\"CommitRate\":10000},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}],\"CloudWatchLogGroup\":null,\"CloudWatchLogStream\":null},\"ControlTablesSettings\":{\"historyTimeslotInMinutes\":5,\"ControlSchema\":\"\",\"HistoryTimeslotInMinutes\":5,\"HistoryTableEnabled\":false,\"SuspendedTablesTableEnabled\":false,\"StatusTableEnabled\":false},\"StreamBufferSettings\":{\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8,\"CtrlStreamBufferSizeInMB\":5},\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true,\"HandleSourceTableAltered\":true},\"ErrorBehavior\":{\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorEscalationCount\":0,\"TableErrorPolicy\":\"SUSPEND_TABLE\",\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorEscalationCount\":0,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorEscalationCount\":0,\"FullLoadIgnoreConflicts\":true},\"ChangeProcessingTuning\":{\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMin\":1,\"BatchApplyTimeoutMax\":30,\"BatchApplyMemoryLimit\":500,\"BatchSplitSize\":0,\"MinTransactionSize\":1000,\"CommitTimeout\":1,\"MemoryLimitTotal\":1024,\"MemoryKeepTime\":60,\"StatementCacheSize\":50}}" + replication_task_settings = "{\"BeforeImageSettings\":null,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false}}" source_endpoint_arn = aws_dms_endpoint.dms_endpoint_source.endpoint_arn table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" @@ -204,12 +245,48 @@ data "aws_partition" "current" {} data "aws_region" "current" {} +data "aws_iam_policy_document" "dms_assume_role_policy_document" { + statement { + actions = ["sts:AssumeRole"] + + principals { + identifiers = ["dms.amazonaws.com"] + type = "Service" + } + } +} + +resource "aws_iam_role" "dms_vpc_role" { + assume_role_policy = data.aws_iam_policy_document.dms_assume_role_policy_document.json + name = "dms-vpc-role" +} + +resource "aws_iam_role" "dms_cloudwatch_logs_role" { + assume_role_policy = data.aws_iam_policy_document.dms_assume_role_policy_document.json + name = "dms-cloudwatch-logs-role" +} + +resource "aws_iam_role_policy_attachment" "dms_vpc_access_policy" { + role = aws_iam_role.dms_vpc_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonDMSVPCManagementRole" +} + +resource "aws_iam_role_policy_attachment" "dms_cloudwatch_logs_access_policy" { + role = aws_iam_role.dms_cloudwatch_logs_role.name + policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonDMSCloudWatchLogsRole" +} + resource "aws_vpc" "dms_vpc" { cidr_block = "10.1.0.0/16" tags = { Name = "terraform-testacc-dms-replication-task-%[1]s" } + + depends_on = [ + aws_iam_role_policy_attachment.dms_vpc_access_policy, + aws_iam_role_policy_attachment.dms_cloudwatch_logs_access_policy, + ] } resource "aws_subnet" "dms_subnet_1" { @@ -278,7 +355,7 @@ resource "aws_dms_replication_task" "dms_replication_task" { migration_type = "full-load" replication_instance_arn = aws_dms_replication_instance.dms_replication_instance.replication_instance_arn replication_task_id = "tf-test-dms-replication-task-%[1]s" - replication_task_settings = "{\"TargetMetadata\":{\"TargetSchema\":\"\",\"SupportLobs\":true,\"FullLobMode\":false,\"LobChunkSize\":0,\"LimitedSizeLobMode\":true,\"LobMaxSize\":32,\"LoadMaxFileSize\":0,\"ParallelLoadThreads\":0,\"BatchApplyEnabled\":false},\"FullLoadSettings\":{\"FullLoadEnabled\":true,\"ApplyChangesEnabled\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"CreatePkAfterFullLoad\":false,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"ResumeEnabled\":false,\"ResumeMinTableSize\":100000,\"ResumeOnlyClusteredPKTables\":true,\"MaxFullLoadSubTasks\":7,\"TransactionConsistencyTimeout\":600,\"CommitRate\":10000},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}],\"CloudWatchLogGroup\":null,\"CloudWatchLogStream\":null},\"ControlTablesSettings\":{\"historyTimeslotInMinutes\":5,\"ControlSchema\":\"\",\"HistoryTimeslotInMinutes\":5,\"HistoryTableEnabled\":false,\"SuspendedTablesTableEnabled\":false,\"StatusTableEnabled\":false},\"StreamBufferSettings\":{\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8,\"CtrlStreamBufferSizeInMB\":5},\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true,\"HandleSourceTableAltered\":true},\"ErrorBehavior\":{\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorEscalationCount\":0,\"TableErrorPolicy\":\"SUSPEND_TABLE\",\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorEscalationCount\":0,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorEscalationCount\":0,\"FullLoadIgnoreConflicts\":true},\"ChangeProcessingTuning\":{\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMin\":1,\"BatchApplyTimeoutMax\":30,\"BatchApplyMemoryLimit\":500,\"BatchSplitSize\":0,\"MinTransactionSize\":1000,\"CommitTimeout\":1,\"MemoryLimitTotal\":1024,\"MemoryKeepTime\":60,\"StatementCacheSize\":50}}" + replication_task_settings = "{\"BeforeImageSettings\":null,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false}}" source_endpoint_arn = aws_dms_endpoint.dms_endpoint_source.endpoint_arn table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" From f02b20999d1e4e403e1fb4f88c4c18addd8bba3e Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Wed, 4 Nov 2020 20:27:08 -0500 Subject: [PATCH 04/13] Run terrafmt --- aws/resource_aws_dms_replication_task_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/resource_aws_dms_replication_task_test.go b/aws/resource_aws_dms_replication_task_test.go index 03c9f312315..c271908802b 100644 --- a/aws/resource_aws_dms_replication_task_test.go +++ b/aws/resource_aws_dms_replication_task_test.go @@ -127,12 +127,12 @@ data "aws_iam_policy_document" "dms_assume_role_policy_document" { resource "aws_iam_role" "dms_vpc_role" { assume_role_policy = data.aws_iam_policy_document.dms_assume_role_policy_document.json - name = "dms-vpc-role" + name = "dms-vpc-role" } resource "aws_iam_role" "dms_cloudwatch_logs_role" { assume_role_policy = data.aws_iam_policy_document.dms_assume_role_policy_document.json - name = "dms-cloudwatch-logs-role" + name = "dms-cloudwatch-logs-role" } resource "aws_iam_role_policy_attachment" "dms_vpc_access_policy" { @@ -258,12 +258,12 @@ data "aws_iam_policy_document" "dms_assume_role_policy_document" { resource "aws_iam_role" "dms_vpc_role" { assume_role_policy = data.aws_iam_policy_document.dms_assume_role_policy_document.json - name = "dms-vpc-role" + name = "dms-vpc-role" } resource "aws_iam_role" "dms_cloudwatch_logs_role" { assume_role_policy = data.aws_iam_policy_document.dms_assume_role_policy_document.json - name = "dms-cloudwatch-logs-role" + name = "dms-cloudwatch-logs-role" } resource "aws_iam_role_policy_attachment" "dms_vpc_access_policy" { From 13820ec2e6a0171620ea570688f7bedcaaca2f1e Mon Sep 17 00:00:00 2001 From: Alex Burgel Date: Wed, 4 Nov 2020 20:32:07 -0500 Subject: [PATCH 05/13] AWS partition linter --- aws/resource_aws_dms_replication_task_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/aws/resource_aws_dms_replication_task_test.go b/aws/resource_aws_dms_replication_task_test.go index c271908802b..3c8e5ec0fa1 100644 --- a/aws/resource_aws_dms_replication_task_test.go +++ b/aws/resource_aws_dms_replication_task_test.go @@ -137,12 +137,12 @@ resource "aws_iam_role" "dms_cloudwatch_logs_role" { resource "aws_iam_role_policy_attachment" "dms_vpc_access_policy" { role = aws_iam_role.dms_vpc_role.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonDMSVPCManagementRole" + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonDMSVPCManagementRole" } resource "aws_iam_role_policy_attachment" "dms_cloudwatch_logs_access_policy" { role = aws_iam_role.dms_cloudwatch_logs_role.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonDMSCloudWatchLogsRole" + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonDMSCloudWatchLogsRole" } resource "aws_vpc" "dms_vpc" { @@ -268,12 +268,12 @@ resource "aws_iam_role" "dms_cloudwatch_logs_role" { resource "aws_iam_role_policy_attachment" "dms_vpc_access_policy" { role = aws_iam_role.dms_vpc_role.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonDMSVPCManagementRole" + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonDMSVPCManagementRole" } resource "aws_iam_role_policy_attachment" "dms_cloudwatch_logs_access_policy" { role = aws_iam_role.dms_cloudwatch_logs_role.name - policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonDMSCloudWatchLogsRole" + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonDMSCloudWatchLogsRole" } resource "aws_vpc" "dms_vpc" { From 2cf0ea5e22176a7b924cff6778e68659811a7e57 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Thu, 1 Apr 2021 19:23:11 -0400 Subject: [PATCH 06/13] r/dms_replication_task: Avoid hardcoded DNS suffix --- aws/resource_aws_dms_replication_task_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws/resource_aws_dms_replication_task_test.go b/aws/resource_aws_dms_replication_task_test.go index 3c8e5ec0fa1..63b2f5216e1 100644 --- a/aws/resource_aws_dms_replication_task_test.go +++ b/aws/resource_aws_dms_replication_task_test.go @@ -119,7 +119,7 @@ data "aws_iam_policy_document" "dms_assume_role_policy_document" { actions = ["sts:AssumeRole"] principals { - identifiers = ["dms.amazonaws.com"] + identifiers = ["dms.${data.aws_partition.current.dns_suffix}"] type = "Service" } } @@ -250,7 +250,7 @@ data "aws_iam_policy_document" "dms_assume_role_policy_document" { actions = ["sts:AssumeRole"] principals { - identifiers = ["dms.amazonaws.com"] + identifiers = ["dms.${data.aws_partition.current.dns_suffix}"] type = "Service" } } From 531f9ef0b991d96c8d86aceee6f8269160b488ec Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Thu, 1 Apr 2021 19:29:54 -0400 Subject: [PATCH 07/13] r/dms_replication_task: Remove errant rebase fodder --- aws/diff_suppress_funcs_test.go | 56 --------------------------------- 1 file changed, 56 deletions(-) diff --git a/aws/diff_suppress_funcs_test.go b/aws/diff_suppress_funcs_test.go index 12c6a1a1af8..3096217fd9d 100644 --- a/aws/diff_suppress_funcs_test.go +++ b/aws/diff_suppress_funcs_test.go @@ -264,59 +264,3 @@ Outputs: } } } - -func TestSuppressRoute53ZoneNameWithTrailingDot(t *testing.T) { - testCases := []struct { - old string - new string - equivalent bool - }{ - { - old: "example.com", - new: "example.com", - equivalent: true, - }, - { - old: "example.com.", - new: "example.com.", - equivalent: true, - }, - { - old: "example.com.", - new: "example.com", - equivalent: true, - }, - { - old: "example.com", - new: "example.com.", - equivalent: true, - }, - { - old: ".", - new: "", - equivalent: false, - }, - { - old: "", - new: ".", - equivalent: false, - }, - { - old: ".", - new: ".", - equivalent: true, - }, - } - - for i, tc := range testCases { - value := suppressRoute53ZoneNameWithTrailingDot("test_property", tc.old, tc.new, nil) - - if tc.equivalent && !value { - t.Fatalf("expected test case %d to be equivalent", i) - } - - if !tc.equivalent && value { - t.Fatalf("expected test case %d to not be equivalent", i) - } - } -} From 9030219fd5ba0cdbf029b7b90a049d60a45d6fca Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Thu, 1 Apr 2021 20:15:57 -0400 Subject: [PATCH 08/13] r/dms_replication_task: Simplify, cleanup names --- aws/resource_aws_dms_replication_task.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws/resource_aws_dms_replication_task.go b/aws/resource_aws_dms_replication_task.go index 3b09c9f8e6a..b4f4d628ffc 100644 --- a/aws/resource_aws_dms_replication_task.go +++ b/aws/resource_aws_dms_replication_task.go @@ -292,11 +292,11 @@ func resourceAwsDmsReplicationTaskSetState(d *schema.ResourceData, task *dms.Rep d.Set("table_mappings", task.TableMappings) d.Set("target_endpoint_arn", task.TargetEndpointArn) - cleanedReplicationTaskSettings, err := cleanReadOnlyReplicationTaskSettings(*task.ReplicationTaskSettings) + settings, err := dmsReplicationTaskRemoveReadOnlySettings(*task.ReplicationTaskSettings) if err != nil { return err } - d.Set("replication_task_settings", cleanedReplicationTaskSettings) + d.Set("replication_task_settings", settings) return nil } @@ -334,7 +334,7 @@ func resourceAwsDmsReplicationTaskStateRefreshFunc( } } -func cleanReadOnlyReplicationTaskSettings(settings string) (*string, error) { +func dmsReplicationTaskRemoveReadOnlySettings(settings string) (*string, error) { var settingsData map[string]interface{} if err := json.Unmarshal([]byte(settings), &settingsData); err != nil { return nil, err From 6474fe898e08c526f76898ab66fb3cb4bb81a6ff Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Thu, 1 Apr 2021 20:16:27 -0400 Subject: [PATCH 09/13] tests/dms_replication_task: Reduce duplication of HCL --- aws/resource_aws_dms_replication_task_test.go | 251 +++--------------- 1 file changed, 44 insertions(+), 207 deletions(-) diff --git a/aws/resource_aws_dms_replication_task_test.go b/aws/resource_aws_dms_replication_task_test.go index 63b2f5216e1..e57baa72009 100644 --- a/aws/resource_aws_dms_replication_task_test.go +++ b/aws/resource_aws_dms_replication_task_test.go @@ -12,8 +12,17 @@ import ( ) func TestAccAWSDmsReplicationTask_basic(t *testing.T) { - resourceName := "aws_dms_replication_task.dms_replication_task" - randId := acctest.RandString(8) + rName := acctest.RandomWithPrefix("tf-acc-test") + resourceName := "aws_dms_replication_task.test" + + tags := ` + Update = "to-update" + Remove = "to-remove" +` + updatedTags := ` + Update = "updated" + Add = "added" +` resource.ParallelTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, @@ -22,14 +31,14 @@ func TestAccAWSDmsReplicationTask_basic(t *testing.T) { CheckDestroy: dmsReplicationTaskDestroy, Steps: []resource.TestStep{ { - Config: dmsReplicationTaskConfig(randId), + Config: dmsReplicationTaskConfig(rName, tags), Check: resource.ComposeTestCheckFunc( checkDmsReplicationTaskExists(resourceName), resource.TestCheckResourceAttrSet(resourceName, "replication_task_arn"), ), }, { - Config: dmsReplicationTaskConfig(randId), + Config: dmsReplicationTaskConfig(rName, tags), PlanOnly: true, ExpectNonEmptyPlan: false, }, @@ -39,7 +48,7 @@ func TestAccAWSDmsReplicationTask_basic(t *testing.T) { ImportStateVerify: true, }, { - Config: dmsReplicationTaskConfigUpdate(randId), + Config: dmsReplicationTaskConfig(rName, updatedTags), Check: resource.ComposeTestCheckFunc( checkDmsReplicationTaskExists(resourceName), ), @@ -108,83 +117,43 @@ func dmsReplicationTaskDestroy(s *terraform.State) error { return nil } -func dmsReplicationTaskConfig(randId string) string { +func dmsReplicationTaskConfig(rName, tags string) string { return composeConfig(testAccAvailableAZsNoOptInConfig(), fmt.Sprintf(` data "aws_partition" "current" {} data "aws_region" "current" {} -data "aws_iam_policy_document" "dms_assume_role_policy_document" { - statement { - actions = ["sts:AssumeRole"] - - principals { - identifiers = ["dms.${data.aws_partition.current.dns_suffix}"] - type = "Service" - } - } -} - -resource "aws_iam_role" "dms_vpc_role" { - assume_role_policy = data.aws_iam_policy_document.dms_assume_role_policy_document.json - name = "dms-vpc-role" -} - -resource "aws_iam_role" "dms_cloudwatch_logs_role" { - assume_role_policy = data.aws_iam_policy_document.dms_assume_role_policy_document.json - name = "dms-cloudwatch-logs-role" -} - -resource "aws_iam_role_policy_attachment" "dms_vpc_access_policy" { - role = aws_iam_role.dms_vpc_role.name - policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonDMSVPCManagementRole" -} - -resource "aws_iam_role_policy_attachment" "dms_cloudwatch_logs_access_policy" { - role = aws_iam_role.dms_cloudwatch_logs_role.name - policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonDMSCloudWatchLogsRole" -} - -resource "aws_vpc" "dms_vpc" { +resource "aws_vpc" "test" { cidr_block = "10.1.0.0/16" tags = { - Name = "terraform-testacc-dms-replication-task-%[1]s" + Name = %[1]q } - - depends_on = [ - aws_iam_role_policy_attachment.dms_vpc_access_policy, - aws_iam_role_policy_attachment.dms_cloudwatch_logs_access_policy, - ] } -resource "aws_subnet" "dms_subnet_1" { +resource "aws_subnet" "test1" { cidr_block = "10.1.1.0/24" availability_zone = data.aws_availability_zones.available.names[0] - vpc_id = aws_vpc.dms_vpc.id + vpc_id = aws_vpc.test.id tags = { - Name = "tf-acc-dms-replication-task-1-%[1]s" + Name = %[1]q } - - depends_on = [aws_vpc.dms_vpc] } -resource "aws_subnet" "dms_subnet_2" { +resource "aws_subnet" "test2" { cidr_block = "10.1.2.0/24" availability_zone = data.aws_availability_zones.available.names[1] - vpc_id = aws_vpc.dms_vpc.id + vpc_id = aws_vpc.test.id tags = { - Name = "tf-acc-dms-replication-task-2-%[1]s" + Name = "%[1]s-2" } - - depends_on = [aws_vpc.dms_vpc] } -resource "aws_dms_endpoint" "dms_endpoint_source" { - database_name = "tf-test-dms-db" - endpoint_id = "tf-test-dms-endpoint-source-%[1]s" +resource "aws_dms_endpoint" "source" { + database_name = %[1]q + endpoint_id = "%[1]s-source" endpoint_type = "source" engine_name = "aurora" server_name = "tf-test-cluster.cluster-xxxxxxx.${data.aws_region.current.name}.rds.${data.aws_partition.current.dns_suffix}" @@ -193,9 +162,9 @@ resource "aws_dms_endpoint" "dms_endpoint_source" { password = "tftest" } -resource "aws_dms_endpoint" "dms_endpoint_target" { - database_name = "tf-test-dms-db" - endpoint_id = "tf-test-dms-endpoint-target-%[1]s" +resource "aws_dms_endpoint" "target" { + database_name = %[1]q + endpoint_id = "%[1]s-target" endpoint_type = "target" engine_name = "aurora" server_name = "tf-test-cluster.cluster-xxxxxxx.${data.aws_region.current.name}.rds.${data.aws_partition.current.dns_suffix}" @@ -204,168 +173,36 @@ resource "aws_dms_endpoint" "dms_endpoint_target" { password = "tftest" } -resource "aws_dms_replication_subnet_group" "dms_replication_subnet_group" { - replication_subnet_group_id = "tf-test-dms-replication-subnet-group-%[1]s" +resource "aws_dms_replication_subnet_group" "test" { + replication_subnet_group_id = %[1]q replication_subnet_group_description = "terraform test for replication subnet group" - subnet_ids = [aws_subnet.dms_subnet_1.id, aws_subnet.dms_subnet_2.id] + subnet_ids = [aws_subnet.test1.id, aws_subnet.test2.id] } -resource "aws_dms_replication_instance" "dms_replication_instance" { +resource "aws_dms_replication_instance" "test" { allocated_storage = 5 auto_minor_version_upgrade = true replication_instance_class = "dms.c4.large" - replication_instance_id = "tf-test-dms-replication-instance-%[1]s" + replication_instance_id = %[1]q preferred_maintenance_window = "sun:00:30-sun:02:30" publicly_accessible = false - replication_subnet_group_id = aws_dms_replication_subnet_group.dms_replication_subnet_group.replication_subnet_group_id + replication_subnet_group_id = aws_dms_replication_subnet_group.test.replication_subnet_group_id } -resource "aws_dms_replication_task" "dms_replication_task" { +resource "aws_dms_replication_task" "test" { migration_type = "full-load" - replication_instance_arn = aws_dms_replication_instance.dms_replication_instance.replication_instance_arn - replication_task_id = "tf-test-dms-replication-task-%[1]s" - replication_task_settings = "{\"BeforeImageSettings\":null,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false}}" - source_endpoint_arn = aws_dms_endpoint.dms_endpoint_source.endpoint_arn + replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn + replication_task_id = %[1]q + replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":\"false\",\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false}}" + source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" tags = { - Name = "tf-test-dms-replication-task-%[1]s" - Update = "to-update" - Remove = "to-remove" - } - - target_endpoint_arn = aws_dms_endpoint.dms_endpoint_target.endpoint_arn -} -`, randId)) -} - -func dmsReplicationTaskConfigUpdate(randId string) string { - return composeConfig(testAccAvailableAZsNoOptInConfig(), fmt.Sprintf(` -data "aws_partition" "current" {} - -data "aws_region" "current" {} - -data "aws_iam_policy_document" "dms_assume_role_policy_document" { - statement { - actions = ["sts:AssumeRole"] - - principals { - identifiers = ["dms.${data.aws_partition.current.dns_suffix}"] - type = "Service" - } - } -} - -resource "aws_iam_role" "dms_vpc_role" { - assume_role_policy = data.aws_iam_policy_document.dms_assume_role_policy_document.json - name = "dms-vpc-role" -} - -resource "aws_iam_role" "dms_cloudwatch_logs_role" { - assume_role_policy = data.aws_iam_policy_document.dms_assume_role_policy_document.json - name = "dms-cloudwatch-logs-role" -} - -resource "aws_iam_role_policy_attachment" "dms_vpc_access_policy" { - role = aws_iam_role.dms_vpc_role.name - policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonDMSVPCManagementRole" -} - -resource "aws_iam_role_policy_attachment" "dms_cloudwatch_logs_access_policy" { - role = aws_iam_role.dms_cloudwatch_logs_role.name - policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonDMSCloudWatchLogsRole" -} - -resource "aws_vpc" "dms_vpc" { - cidr_block = "10.1.0.0/16" - - tags = { - Name = "terraform-testacc-dms-replication-task-%[1]s" - } - - depends_on = [ - aws_iam_role_policy_attachment.dms_vpc_access_policy, - aws_iam_role_policy_attachment.dms_cloudwatch_logs_access_policy, - ] -} - -resource "aws_subnet" "dms_subnet_1" { - cidr_block = "10.1.1.0/24" - availability_zone = data.aws_availability_zones.available.names[0] - vpc_id = aws_vpc.dms_vpc.id - - tags = { - Name = "tf-acc-dms-replication-task-1-%[1]s" - } - - depends_on = [aws_vpc.dms_vpc] -} - -resource "aws_subnet" "dms_subnet_2" { - cidr_block = "10.1.2.0/24" - availability_zone = data.aws_availability_zones.available.names[1] - vpc_id = aws_vpc.dms_vpc.id - - tags = { - Name = "tf-acc-dms-replication-task-2-%[1]s" - } - - depends_on = [aws_vpc.dms_vpc] -} - -resource "aws_dms_endpoint" "dms_endpoint_source" { - database_name = "tf-test-dms-db" - endpoint_id = "tf-test-dms-endpoint-source-%[1]s" - endpoint_type = "source" - engine_name = "aurora" - server_name = "tf-test-cluster.cluster-xxxxxxx.${data.aws_region.current.name}.rds.${data.aws_partition.current.dns_suffix}" - port = 3306 - username = "tftest" - password = "tftest" -} - -resource "aws_dms_endpoint" "dms_endpoint_target" { - database_name = "tf-test-dms-db" - endpoint_id = "tf-test-dms-endpoint-target-%[1]s" - endpoint_type = "target" - engine_name = "aurora" - server_name = "tf-test-cluster.cluster-xxxxxxx.${data.aws_region.current.name}.rds.${data.aws_partition.current.dns_suffix}" - port = 3306 - username = "tftest" - password = "tftest" -} - -resource "aws_dms_replication_subnet_group" "dms_replication_subnet_group" { - replication_subnet_group_id = "tf-test-dms-replication-subnet-group-%[1]s" - replication_subnet_group_description = "terraform test for replication subnet group" - subnet_ids = [aws_subnet.dms_subnet_1.id, aws_subnet.dms_subnet_2.id] -} - -resource "aws_dms_replication_instance" "dms_replication_instance" { - allocated_storage = 5 - auto_minor_version_upgrade = true - replication_instance_class = "dms.c4.large" - replication_instance_id = "tf-test-dms-replication-instance-%[1]s" - preferred_maintenance_window = "sun:00:30-sun:02:30" - publicly_accessible = false - replication_subnet_group_id = aws_dms_replication_subnet_group.dms_replication_subnet_group.replication_subnet_group_id -} - -resource "aws_dms_replication_task" "dms_replication_task" { - migration_type = "full-load" - replication_instance_arn = aws_dms_replication_instance.dms_replication_instance.replication_instance_arn - replication_task_id = "tf-test-dms-replication-task-%[1]s" - replication_task_settings = "{\"BeforeImageSettings\":null,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false}}" - source_endpoint_arn = aws_dms_endpoint.dms_endpoint_source.endpoint_arn - table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" - - tags = { - Name = "tf-test-dms-replication-task-%[1]s" - Update = "updated" - Add = "added" + Name = %[1]q +%[2]s } - target_endpoint_arn = aws_dms_endpoint.dms_endpoint_target.endpoint_arn + target_endpoint_arn = aws_dms_endpoint.target.endpoint_arn } -`, randId)) +`, rName, tags)) } From 32d55018c5b4d0478acdf288589468230a6fd631 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Thu, 1 Apr 2021 20:20:10 -0400 Subject: [PATCH 10/13] tests/dms_replication_task: Lint --- aws/resource_aws_dms_replication_task_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/resource_aws_dms_replication_task_test.go b/aws/resource_aws_dms_replication_task_test.go index e57baa72009..a9f6d5c4a60 100644 --- a/aws/resource_aws_dms_replication_task_test.go +++ b/aws/resource_aws_dms_replication_task_test.go @@ -198,7 +198,7 @@ resource "aws_dms_replication_task" "test" { table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" tags = { - Name = %[1]q + Name = %[1]q %[2]s } From d281a506f996c7ea43c33b46666d03bf6a343f24 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Thu, 1 Apr 2021 20:20:52 -0400 Subject: [PATCH 11/13] tests/dms_replication_task: Remove quotes --- aws/resource_aws_dms_replication_task_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws/resource_aws_dms_replication_task_test.go b/aws/resource_aws_dms_replication_task_test.go index a9f6d5c4a60..d8996bf5da8 100644 --- a/aws/resource_aws_dms_replication_task_test.go +++ b/aws/resource_aws_dms_replication_task_test.go @@ -193,7 +193,7 @@ resource "aws_dms_replication_task" "test" { migration_type = "full-load" replication_instance_arn = aws_dms_replication_instance.test.replication_instance_arn replication_task_id = %[1]q - replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":\"false\",\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false}}" + replication_task_settings = "{\"BeforeImageSettings\":null,\"FailTaskWhenCleanTaskResourceFailed\":false,\"ChangeProcessingDdlHandlingPolicy\":{\"HandleSourceTableAltered\":true,\"HandleSourceTableDropped\":true,\"HandleSourceTableTruncated\":true},\"ChangeProcessingTuning\":{\"BatchApplyMemoryLimit\":500,\"BatchApplyPreserveTransaction\":true,\"BatchApplyTimeoutMax\":30,\"BatchApplyTimeoutMin\":1,\"BatchSplitSize\":0,\"CommitTimeout\":1,\"MemoryKeepTime\":60,\"MemoryLimitTotal\":1024,\"MinTransactionSize\":1000,\"StatementCacheSize\":50},\"CharacterSetSettings\":null,\"ControlTablesSettings\":{\"ControlSchema\":\"\",\"FullLoadExceptionTableEnabled\":false,\"HistoryTableEnabled\":false,\"HistoryTimeslotInMinutes\":5,\"StatusTableEnabled\":false,\"SuspendedTablesTableEnabled\":false},\"ErrorBehavior\":{\"ApplyErrorDeletePolicy\":\"IGNORE_RECORD\",\"ApplyErrorEscalationCount\":0,\"ApplyErrorEscalationPolicy\":\"LOG_ERROR\",\"ApplyErrorFailOnTruncationDdl\":false,\"ApplyErrorInsertPolicy\":\"LOG_ERROR\",\"ApplyErrorUpdatePolicy\":\"LOG_ERROR\",\"DataErrorEscalationCount\":0,\"DataErrorEscalationPolicy\":\"SUSPEND_TABLE\",\"DataErrorPolicy\":\"LOG_ERROR\",\"DataTruncationErrorPolicy\":\"LOG_ERROR\",\"FailOnNoTablesCaptured\":false,\"FailOnTransactionConsistencyBreached\":false,\"FullLoadIgnoreConflicts\":true,\"RecoverableErrorCount\":-1,\"RecoverableErrorInterval\":5,\"RecoverableErrorStopRetryAfterThrottlingMax\":false,\"RecoverableErrorThrottling\":true,\"RecoverableErrorThrottlingMax\":1800,\"TableErrorEscalationCount\":0,\"TableErrorEscalationPolicy\":\"STOP_TASK\",\"TableErrorPolicy\":\"SUSPEND_TABLE\"},\"FullLoadSettings\":{\"CommitRate\":10000,\"CreatePkAfterFullLoad\":false,\"MaxFullLoadSubTasks\":8,\"StopTaskCachedChangesApplied\":false,\"StopTaskCachedChangesNotApplied\":false,\"TargetTablePrepMode\":\"DROP_AND_CREATE\",\"TransactionConsistencyTimeout\":600},\"Logging\":{\"EnableLogging\":false,\"LogComponents\":[{\"Id\":\"TRANSFORMATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_UNLOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"IO\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_LOAD\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"PERFORMANCE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SOURCE_CAPTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"SORTER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"REST_SERVER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"VALIDATOR_EXT\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TARGET_APPLY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TASK_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"TABLES_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"METADATA_MANAGER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_FACTORY\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMON\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"ADDONS\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"DATA_STRUCTURE\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"COMMUNICATION\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"},{\"Id\":\"FILE_TRANSFER\",\"Severity\":\"LOGGER_SEVERITY_DEFAULT\"}]},\"LoopbackPreventionSettings\":null,\"PostProcessingRules\":null,\"StreamBufferSettings\":{\"CtrlStreamBufferSizeInMB\":5,\"StreamBufferCount\":3,\"StreamBufferSizeInMB\":8},\"TargetMetadata\":{\"BatchApplyEnabled\":false,\"FullLobMode\":false,\"InlineLobMaxSize\":0,\"LimitedSizeLobMode\":true,\"LoadMaxFileSize\":0,\"LobChunkSize\":0,\"LobMaxSize\":32,\"ParallelApplyBufferSize\":0,\"ParallelApplyQueuesPerThread\":0,\"ParallelApplyThreads\":0,\"ParallelLoadBufferSize\":0,\"ParallelLoadQueuesPerThread\":0,\"ParallelLoadThreads\":0,\"SupportLobs\":true,\"TargetSchema\":\"\",\"TaskRecoveryTableEnabled\":false}}" source_endpoint_arn = aws_dms_endpoint.source.endpoint_arn table_mappings = "{\"rules\":[{\"rule-type\":\"selection\",\"rule-id\":\"1\",\"rule-name\":\"1\",\"object-locator\":{\"schema-name\":\"%%\",\"table-name\":\"%%\"},\"rule-action\":\"include\"}]}" From af2c65ceba7fad9b9cd3cb54288bdbe1da74a4d1 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Thu, 1 Apr 2021 20:27:40 -0400 Subject: [PATCH 12/13] r/dms_replication_task: Add changelog --- .changelog/13476.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/13476.txt diff --git a/.changelog/13476.txt b/.changelog/13476.txt new file mode 100644 index 00000000000..7117af8ecce --- /dev/null +++ b/.changelog/13476.txt @@ -0,0 +1,3 @@ +```release-note:bug +resource/aws_dms_replication_task: Handle read only attributes in `replication_task_settings` to avoid unnecessary diffs. +``` \ No newline at end of file From 58f25253a5819785af0687bd240c576a58878f85 Mon Sep 17 00:00:00 2001 From: Dirk Avery Date: Thu, 1 Apr 2021 20:30:03 -0400 Subject: [PATCH 13/13] r/dms_replication_task: Minor cleanups --- .changelog/13476.txt | 2 +- aws/resource_aws_dms_replication_task.go | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.changelog/13476.txt b/.changelog/13476.txt index 7117af8ecce..16af42ba2bf 100644 --- a/.changelog/13476.txt +++ b/.changelog/13476.txt @@ -1,3 +1,3 @@ ```release-note:bug -resource/aws_dms_replication_task: Handle read only attributes in `replication_task_settings` to avoid unnecessary diffs. +resource/aws_dms_replication_task: Handle read-only attributes in `replication_task_settings` to avoid unnecessary diffs. ``` \ No newline at end of file diff --git a/aws/resource_aws_dms_replication_task.go b/aws/resource_aws_dms_replication_task.go index b4f4d628ffc..dc946b38618 100644 --- a/aws/resource_aws_dms_replication_task.go +++ b/aws/resource_aws_dms_replication_task.go @@ -284,6 +284,7 @@ func resourceAwsDmsReplicationTaskDelete(d *schema.ResourceData, meta interface{ func resourceAwsDmsReplicationTaskSetState(d *schema.ResourceData, task *dms.ReplicationTask) error { d.SetId(aws.StringValue(task.ReplicationTaskIdentifier)) + d.Set("migration_type", task.MigrationType) d.Set("replication_instance_arn", task.ReplicationInstanceArn) d.Set("replication_task_arn", task.ReplicationTaskArn)