From 43aebaadbbf57ff0d1e94692fbe9c27df78fdd9a Mon Sep 17 00:00:00 2001 From: Hari Chintala Date: Wed, 31 May 2023 09:30:49 +0100 Subject: [PATCH] DPR-197-1, Rebased to MAIN --- .../digital-prison-reporting/main.tf | 31 ++- .../config/nomis-replication-settings.json | 204 ++++++++++++++---- .../modules/dms/main.tf | 6 +- 3 files changed, 191 insertions(+), 50 deletions(-) diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf index 33f9dd6e435..a50cd98e71c 100644 --- a/terraform/environments/digital-prison-reporting/main.tf +++ b/terraform/environments/digital-prison-reporting/main.tf @@ -557,6 +557,7 @@ module "dms_nomis_ingestor" { short_name = "nomis" migration_type = "full-load-and-cdc" replication_instance_version = "3.4.6" + replication_instance_class = "dms.t3.medium" subnet_ids = [data.aws_subnet.data_subnets_a.id, data.aws_subnet.data_subnets_b.id, data.aws_subnet.data_subnets_c.id] vpc_role_dependency = [aws_iam_role.dmsvpcrole] @@ -684,4 +685,32 @@ module "s3_application_tf_state" { Resource_Type = "S3 Bucket" } ) -} \ No newline at end of file +} + +# Dynamo Tab for Application TF State +module "dynamo_tab_application_tf_state" { + source = "./modules/dynamo_tables" + create_table = true + autoscaling_enabled = false + name = "${local.project}-terraform-state-${local.environment}" + + hash_key = "LockID" # Hash + range_key = "" # Sort + table_class = "STANDARD" + ttl_enabled = false + + attributes = [ + { + name = "LockID" + type = "S" + } + ] + + tags = merge( + local.all_tags, + { + Name = "${local.project}-terraform-state-${local.environment}" + Resource_Type = "Dynamo Table" + } + ) +} diff --git a/terraform/environments/digital-prison-reporting/modules/dms/config/nomis-replication-settings.json b/terraform/environments/digital-prison-reporting/modules/dms/config/nomis-replication-settings.json index 02bae8d9a96..b5db6ecd692 100644 --- a/terraform/environments/digital-prison-reporting/modules/dms/config/nomis-replication-settings.json +++ b/terraform/environments/digital-prison-reporting/modules/dms/config/nomis-replication-settings.json @@ -1,66 +1,178 @@ { - "FullLoadSettings": { - "TargetTablePrepMode": "DO_NOTHING", - "CreatePkAfterFullLoad": false, - "StopTaskCachedChangesApplied": false, - "StopTaskCachedChangesNotApplied": false, - "MaxFullLoadSubTasks": 8, - "TransactionConsistencyTimeout": 600, - "CommitRate": 10000 - }, - "ControlTablesSettings": { - "historyTimeslotInMinutes": 5, - "ControlSchema": "", - "HistoryTimeslotInMinutes": 5, - "HistoryTableEnabled": false, - "SuspendedTablesTableEnabled": false, - "StatusTableEnabled": false + "Logging": { + "EnableLogging": false, + "EnableLogContext": false, + "LogComponents": [ + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TRANSFORMATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_UNLOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "IO" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_LOAD" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "PERFORMANCE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SOURCE_CAPTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "SORTER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "REST_SERVER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "VALIDATOR_EXT" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TARGET_APPLY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TASK_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "TABLES_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "METADATA_MANAGER" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_FACTORY" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMON" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "ADDONS" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "DATA_STRUCTURE" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "COMMUNICATION" + }, + { + "Severity": "LOGGER_SEVERITY_DEFAULT", + "Id": "FILE_TRANSFER" + } + ], + "CloudWatchLogGroup": null, + "CloudWatchLogStream": null }, "StreamBufferSettings": { "StreamBufferCount": 3, - "StreamBufferSizeInMB": 8, - "CtrlStreamBufferSizeInMB": 5 - }, - "ChangeProcessingDdlHandlingPolicy": { - "HandleSourceTableDropped": true, - "HandleSourceTableTruncated": true, - "HandleSourceTableAltered": true + "CtrlStreamBufferSizeInMB": 5, + "StreamBufferSizeInMB": 8 }, "ErrorBehavior": { - "DataErrorPolicy": "LOG_ERROR", - "DataTruncationErrorPolicy": "LOG_ERROR", + "FailOnNoTablesCaptured": false, + "ApplyErrorUpdatePolicy": "LOG_ERROR", + "FailOnTransactionConsistencyBreached": false, + "RecoverableErrorThrottlingMax": 1800, "DataErrorEscalationPolicy": "SUSPEND_TABLE", + "ApplyErrorEscalationCount": 0, + "RecoverableErrorStopRetryAfterThrottlingMax": true, + "RecoverableErrorThrottling": true, + "ApplyErrorFailOnTruncationDdl": false, + "DataTruncationErrorPolicy": "LOG_ERROR", + "ApplyErrorInsertPolicy": "LOG_ERROR", + "EventErrorPolicy": "IGNORE", + "ApplyErrorEscalationPolicy": "LOG_ERROR", + "RecoverableErrorCount": -1, "DataErrorEscalationCount": 0, - "TableErrorPolicy": "SUSPEND_TABLE", "TableErrorEscalationPolicy": "STOP_TASK", - "TableErrorEscalationCount": 0, - "RecoverableErrorCount": -1, "RecoverableErrorInterval": 5, - "RecoverableErrorThrottling": true, - "RecoverableErrorThrottlingMax": 1800, "ApplyErrorDeletePolicy": "IGNORE_RECORD", - "ApplyErrorInsertPolicy": "LOG_ERROR", - "ApplyErrorUpdatePolicy": "LOG_ERROR", - "ApplyErrorEscalationPolicy": "LOG_ERROR", - "ApplyErrorEscalationCount": 0, - "ApplyErrorFailOnTruncationDdl": false, + "TableErrorEscalationCount": 0, "FullLoadIgnoreConflicts": true, - "FailOnTransactionConsistencyBreached": false, - "FailOnNoTablesCaptured": false + "DataErrorPolicy": "LOG_ERROR", + "TableErrorPolicy": "SUSPEND_TABLE" + }, + "TTSettings": { + "TTS3Settings": null, + "TTRecordSettings": null, + "EnableTT": false + }, + "FullLoadSettings": { + "CommitRate": 10000, + "StopTaskCachedChangesApplied": false, + "StopTaskCachedChangesNotApplied": false, + "MaxFullLoadSubTasks": 8, + "TransactionConsistencyTimeout": 600, + "CreatePkAfterFullLoad": false, + "TargetTablePrepMode": "DO_NOTHING" }, + "TargetMetadata": { + "ParallelApplyBufferSize": 0, + "ParallelApplyQueuesPerThread": 0, + "ParallelApplyThreads": 0, + "TargetSchema": "", + "InlineLobMaxSize": 0, + "ParallelLoadQueuesPerThread": 0, + "SupportLobs": true, + "LobChunkSize": 64, + "TaskRecoveryTableEnabled": false, + "ParallelLoadThreads": 8, + "LobMaxSize": 32, + "BatchApplyEnabled": false, + "FullLobMode": false, + "LimitedSizeLobMode": true, + "LoadMaxFileSize": 0, + "ParallelLoadBufferSize": 1000 + }, + "BeforeImageSettings": null, + "ControlTablesSettings": { + "historyTimeslotInMinutes": 5, + "HistoryTimeslotInMinutes": 5, + "StatusTableEnabled": false, + "SuspendedTablesTableEnabled": false, + "HistoryTableEnabled": false, + "ControlSchema": "", + "FullLoadExceptionTableEnabled": false + }, + "LoopbackPreventionSettings": null, + "CharacterSetSettings": null, + "FailTaskWhenCleanTaskResourceFailed": false, "ChangeProcessingTuning": { + "StatementCacheSize": 50, + "CommitTimeout": 1, "BatchApplyPreserveTransaction": true, "BatchApplyTimeoutMin": 1, - "BatchApplyTimeoutMax": 30, - "BatchApplyMemoryLimit": 500, "BatchSplitSize": 0, + "BatchApplyTimeoutMax": 30, "MinTransactionSize": 1000, - "CommitTimeout": 1, - "MemoryLimitTotal": 1024, "MemoryKeepTime": 60, - "StatementCacheSize": 50 + "BatchApplyMemoryLimit": 500, + "MemoryLimitTotal": 1024 }, - "PostProcessingRules": null, - "CharacterSetSettings": null, - "LoopbackPreventionSettings": null -} + "ChangeProcessingDdlHandlingPolicy": { + "HandleSourceTableDropped": true, + "HandleSourceTableTruncated": true, + "HandleSourceTableAltered": true + }, + "PostProcessingRules": null +} \ No newline at end of file diff --git a/terraform/environments/digital-prison-reporting/modules/dms/main.tf b/terraform/environments/digital-prison-reporting/modules/dms/main.tf index 4f641e3853a..79dd8e04d93 100644 --- a/terraform/environments/digital-prison-reporting/modules/dms/main.tf +++ b/terraform/environments/digital-prison-reporting/modules/dms/main.tf @@ -43,9 +43,9 @@ resource "aws_dms_replication_task" "dms-replication" { table_mappings = data.template_file.table-mappings.rendered replication_task_settings = file("${path.module}/config/${var.short_name}-replication-settings.json") - lifecycle { - ignore_changes = [replication_task_settings] - } +# lifecycle { +# ignore_changes = [replication_task_settings] +# } } # Create an endpoint for the source database