Merge pull request #2388 from ministryofjustice/DPR-197-1

DPR-197-1, Rebased to MAIN
ministryofjustice · May 31, 2023 · f6c124f · f6c124f
2 parents 429c4d0 + 43aebaa
commit f6c124f
Show file tree

Hide file tree

Showing 3 changed files with 191 additions and 50 deletions.
diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf
@@ -557,6 +557,7 @@ module "dms_nomis_ingestor" {
   short_name            = "nomis"
   migration_type        = "full-load-and-cdc"
   replication_instance_version  = "3.4.6"
+  replication_instance_class = "dms.t3.medium"
   subnet_ids            = [data.aws_subnet.data_subnets_a.id, data.aws_subnet.data_subnets_b.id, data.aws_subnet.data_subnets_c.id]
 
   vpc_role_dependency        = [aws_iam_role.dmsvpcrole]
@@ -684,4 +685,32 @@ module "s3_application_tf_state" {
       Resource_Type = "S3 Bucket"
     }
   )
-}
+}
+
+# Dynamo Tab for Application TF State 
+module "dynamo_tab_application_tf_state" {
+  source              = "./modules/dynamo_tables"
+  create_table        = true
+  autoscaling_enabled = false
+  name                = "${local.project}-terraform-state-${local.environment}"
+
+  hash_key    = "LockID"   # Hash
+  range_key   = ""         # Sort
+  table_class = "STANDARD"
+  ttl_enabled = false
+
+  attributes = [
+    {
+      name = "LockID"
+      type = "S"
+    }
+  ]
+
+  tags = merge(
+    local.all_tags,
+    {
+      Name          = "${local.project}-terraform-state-${local.environment}"
+      Resource_Type = "Dynamo Table"
+    }
+  )
+}
diff --git a/.../environments/digital-prison-reporting/modules/dms/config/nomis-replication-settings.json b/.../environments/digital-prison-reporting/modules/dms/config/nomis-replication-settings.json
@@ -1,66 +1,178 @@
 {
-  "FullLoadSettings": {
-    "TargetTablePrepMode": "DO_NOTHING",
-    "CreatePkAfterFullLoad": false,
-    "StopTaskCachedChangesApplied": false,
-    "StopTaskCachedChangesNotApplied": false,
-    "MaxFullLoadSubTasks": 8,
-    "TransactionConsistencyTimeout": 600,
-    "CommitRate": 10000
-  },
-  "ControlTablesSettings": {
-    "historyTimeslotInMinutes": 5,
-    "ControlSchema": "",
-    "HistoryTimeslotInMinutes": 5,
-    "HistoryTableEnabled": false,
-    "SuspendedTablesTableEnabled": false,
-    "StatusTableEnabled": false
+  "Logging": {
+    "EnableLogging": false,
+    "EnableLogContext": false,
+    "LogComponents": [
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "TRANSFORMATION"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "SOURCE_UNLOAD"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "IO"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "TARGET_LOAD"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "PERFORMANCE"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "SOURCE_CAPTURE"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "SORTER"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "REST_SERVER"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "VALIDATOR_EXT"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "TARGET_APPLY"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "TASK_MANAGER"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "TABLES_MANAGER"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "METADATA_MANAGER"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "FILE_FACTORY"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "COMMON"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "ADDONS"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "DATA_STRUCTURE"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "COMMUNICATION"
+      },
+      {
+        "Severity": "LOGGER_SEVERITY_DEFAULT",
+        "Id": "FILE_TRANSFER"
+      }
+    ],
+    "CloudWatchLogGroup": null,
+    "CloudWatchLogStream": null
   },
   "StreamBufferSettings": {
     "StreamBufferCount": 3,
-    "StreamBufferSizeInMB": 8,
-    "CtrlStreamBufferSizeInMB": 5
-  },
-  "ChangeProcessingDdlHandlingPolicy": {
-    "HandleSourceTableDropped": true,
-    "HandleSourceTableTruncated": true,
-    "HandleSourceTableAltered": true
+    "CtrlStreamBufferSizeInMB": 5,
+    "StreamBufferSizeInMB": 8
   },
   "ErrorBehavior": {
-    "DataErrorPolicy": "LOG_ERROR",
-    "DataTruncationErrorPolicy": "LOG_ERROR",
+    "FailOnNoTablesCaptured": false,
+    "ApplyErrorUpdatePolicy": "LOG_ERROR",
+    "FailOnTransactionConsistencyBreached": false,
+    "RecoverableErrorThrottlingMax": 1800,
     "DataErrorEscalationPolicy": "SUSPEND_TABLE",
+    "ApplyErrorEscalationCount": 0,
+    "RecoverableErrorStopRetryAfterThrottlingMax": true,
+    "RecoverableErrorThrottling": true,
+    "ApplyErrorFailOnTruncationDdl": false,
+    "DataTruncationErrorPolicy": "LOG_ERROR",
+    "ApplyErrorInsertPolicy": "LOG_ERROR",
+    "EventErrorPolicy": "IGNORE",
+    "ApplyErrorEscalationPolicy": "LOG_ERROR",
+    "RecoverableErrorCount": -1,
     "DataErrorEscalationCount": 0,
-    "TableErrorPolicy": "SUSPEND_TABLE",
     "TableErrorEscalationPolicy": "STOP_TASK",
-    "TableErrorEscalationCount": 0,
-    "RecoverableErrorCount": -1,
     "RecoverableErrorInterval": 5,
-    "RecoverableErrorThrottling": true,
-    "RecoverableErrorThrottlingMax": 1800,
     "ApplyErrorDeletePolicy": "IGNORE_RECORD",
-    "ApplyErrorInsertPolicy": "LOG_ERROR",
-    "ApplyErrorUpdatePolicy": "LOG_ERROR",
-    "ApplyErrorEscalationPolicy": "LOG_ERROR",
-    "ApplyErrorEscalationCount": 0,
-    "ApplyErrorFailOnTruncationDdl": false,
+    "TableErrorEscalationCount": 0,
     "FullLoadIgnoreConflicts": true,
-    "FailOnTransactionConsistencyBreached": false,
-    "FailOnNoTablesCaptured": false
+    "DataErrorPolicy": "LOG_ERROR",
+    "TableErrorPolicy": "SUSPEND_TABLE"
+  },
+  "TTSettings": {
+    "TTS3Settings": null,
+    "TTRecordSettings": null,
+    "EnableTT": false
+  },
+  "FullLoadSettings": {
+    "CommitRate": 10000,
+    "StopTaskCachedChangesApplied": false,
+    "StopTaskCachedChangesNotApplied": false,
+    "MaxFullLoadSubTasks": 8,
+    "TransactionConsistencyTimeout": 600,
+    "CreatePkAfterFullLoad": false,
+    "TargetTablePrepMode": "DO_NOTHING"
   },
+  "TargetMetadata": {
+    "ParallelApplyBufferSize": 0,
+    "ParallelApplyQueuesPerThread": 0,
+    "ParallelApplyThreads": 0,
+    "TargetSchema": "",
+    "InlineLobMaxSize": 0,
+    "ParallelLoadQueuesPerThread": 0,
+    "SupportLobs": true,
+    "LobChunkSize": 64,
+    "TaskRecoveryTableEnabled": false,
+    "ParallelLoadThreads": 8,
+    "LobMaxSize": 32,
+    "BatchApplyEnabled": false,
+    "FullLobMode": false,
+    "LimitedSizeLobMode": true,
+    "LoadMaxFileSize": 0,
+    "ParallelLoadBufferSize": 1000
+  },
+  "BeforeImageSettings": null,
+  "ControlTablesSettings": {
+    "historyTimeslotInMinutes": 5,
+    "HistoryTimeslotInMinutes": 5,
+    "StatusTableEnabled": false,
+    "SuspendedTablesTableEnabled": false,
+    "HistoryTableEnabled": false,
+    "ControlSchema": "",
+    "FullLoadExceptionTableEnabled": false
+  },
+  "LoopbackPreventionSettings": null,
+  "CharacterSetSettings": null,
+  "FailTaskWhenCleanTaskResourceFailed": false,
   "ChangeProcessingTuning": {
+    "StatementCacheSize": 50,
+    "CommitTimeout": 1,
     "BatchApplyPreserveTransaction": true,
     "BatchApplyTimeoutMin": 1,
-    "BatchApplyTimeoutMax": 30,
-    "BatchApplyMemoryLimit": 500,
     "BatchSplitSize": 0,
+    "BatchApplyTimeoutMax": 30,
     "MinTransactionSize": 1000,
-    "CommitTimeout": 1,
-    "MemoryLimitTotal": 1024,
     "MemoryKeepTime": 60,
-    "StatementCacheSize": 50
+    "BatchApplyMemoryLimit": 500,
+    "MemoryLimitTotal": 1024
   },
-  "PostProcessingRules": null,
-  "CharacterSetSettings": null,
-  "LoopbackPreventionSettings": null
-}
+  "ChangeProcessingDdlHandlingPolicy": {
+    "HandleSourceTableDropped": true,
+    "HandleSourceTableTruncated": true,
+    "HandleSourceTableAltered": true
+  },
+  "PostProcessingRules": null
+}
diff --git a/terraform/environments/digital-prison-reporting/modules/dms/main.tf b/terraform/environments/digital-prison-reporting/modules/dms/main.tf
@@ -43,9 +43,9 @@ resource "aws_dms_replication_task" "dms-replication" {
   table_mappings            = data.template_file.table-mappings.rendered
   replication_task_settings = file("${path.module}/config/${var.short_name}-replication-settings.json")
 
-  lifecycle {
-    ignore_changes = [replication_task_settings]
-  }
+#  lifecycle {
+#    ignore_changes = [replication_task_settings]
+#  }
 }
 
 # Create an endpoint for the source database