Merge branch 'main' into performance-hub/ssm-ami-param

.gitignore

@@ -22,4 +22,6 @@ out/
 .tfsec/
 
 # vim
-**/*.swp
+**/*.swp
+
+terraform/modules/**/*.zip

terraform/environments/analytical-platform-compute/iam-policies.tf

@@ -280,3 +280,22 @@ module "analytical_platform_lake_formation_share_policy" {
 
   policy = data.aws_iam_policy_document.analytical_platform_share_policy.json
 }
+
+data "aws_iam_policy_document" "s3_server_access_logs_policy" {
+  #checkov:skip=CKV_AWS_356:resource "*" limited by condition
+  statement {
+    sid       = "S3ServerAccessLogsPolicy"
+    effect    = "Allow"
+    actions   = ["s3:PutObject"]
+    resources = ["*"]
+    principals {
+      type        = "Service"
+      identifiers = ["logging.s3.amazonaws.com"]
+    }
+    condition {
+      test     = "StringEquals"
+      variable = "aws:SourceAccount"
+      values   = [data.aws_caller_identity.current.account_id]
+    }
+  }
+}

terraform/environments/analytical-platform-compute/kms-keys.tf

@@ -275,6 +275,66 @@ module "mlflow_s3_kms" {
   tags = local.tags
 }
 
+module "mojap_derived_tables_replication_s3_kms" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
+
+  source  = "terraform-aws-modules/kms/aws"
+  version = "3.1.1"
+
+  aliases               = ["s3/mojap-derived-tables-replication"]
+  description           = "mojap-derived-tables-replication S3 KMS key"
+  enable_default_policy = true
+
+  deletion_window_in_days = 7
+
+  tags = local.tags
+}
+
+module "mojap_compute_logs_s3_kms" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
+
+  source  = "terraform-aws-modules/kms/aws"
+  version = "3.1.1"
+
+  aliases               = ["s3/mojap-compute-logs"]
+  description           = "mojap-compute-logs S3 KMS key"
+  enable_default_policy = true
+
+  deletion_window_in_days = 7
+
+  tags = local.tags
+
+  key_statements = [
+    {
+      sid = "AllowS3Logging"
+      effect = "Allow" 
+      actions = [
+        "kms:Encrypt",
+        "kms:Decrypt",
+        "kms:GenerateDataKey",
+        "kms:GenerateDataKeyWithoutPlaintext",
+        "kms:DescribeKey"
+      ]
+      resources = ["*"]
+      principals = [
+        {
+          type        = "Service"
+          identifiers = ["logging.s3.amazonaws.com"]
+        }
+      ]
+      conditions = [
+        {
+          test     = "StringEquals"
+          variable = "kms:ViaService"
+          values   = ["logging.s3.amazonaws.com"]
+        }
+      ]
+    }
+  ]
+}
+
 module "common_secrets_manager_kms" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions

terraform/environments/analytical-platform-compute/s3-buckets.tf

@@ -21,3 +21,70 @@ module "mlflow_bucket" {
 
   tags = local.tags
 }
+
+module "mojap_derived_tables_replication_bucket" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
+
+  source  = "terraform-aws-modules/s3-bucket/aws"
+  version = "4.2.1"
+
+  bucket = "mojap-compute-${local.environment}-derived-tables-replication"
+
+  force_destroy = true
+
+  object_lock_enabled = false
+
+  versioning = {
+    status = "Disabled"
+  }
+
+  server_side_encryption_configuration = {
+    rule = {
+      bucket_key_enabled = true
+      apply_server_side_encryption_by_default = {
+        kms_master_key_id = module.mojap_derived_tables_replication_s3_kms.key_arn
+        sse_algorithm     = "aws:kms"
+      }
+    }
+  }
+
+  logging = {
+    target_bucket = module.mojap_compute_logs_bucket.s3_bucket_id
+    target_prefix = "mojap-derived-tables-replication/"
+  }
+
+  tags = local.tags
+}
+
+module "mojap_compute_logs_bucket" {
+  #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
+  #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
+
+  source  = "terraform-aws-modules/s3-bucket/aws"
+  version = "4.2.1"
+
+  bucket = "mojap-compute-${local.environment}-logs"
+
+  force_destroy = false
+
+  policy = data.aws_iam_policy_document.s3_server_access_logs_policy.json
+
+  object_lock_enabled = false
+
+  versioning = {
+    status = "Disabled"
+  }
+
+  server_side_encryption_configuration = {
+    rule = {
+      bucket_key_enabled = true
+      apply_server_side_encryption_by_default = {
+        kms_master_key_id = module.mojap_compute_logs_s3_kms.key_arn
+        sse_algorithm     = "aws:kms"
+      }
+    }
+  }
+
+  tags = local.tags
+}

terraform/environments/contract-work-administration/app_servers.tf

@@ -62,7 +62,7 @@ do
 done
 
 echo "Updating /etc/rc.local file"
-cat <<EOT > etc/rc.local
+cat <<EOT > /etc/rc.local
 #!/bin/sh
 #
 # This script will be executed *after* all the other init scripts.
@@ -116,8 +116,8 @@ sed -i 's/${local.application_data.accounts[local.environment].old_domain_name}/
 
 ## Remove SSH key allowed
 echo "Removing old SSH key"
-sed -i '/development-general$/d' /home/ec2-user/.ssh/authorized_keys
-sed -i '/development-general$/d' /root/.ssh/authorized_keys
+sed -i '/.*-general$/d' /home/ec2-user/.ssh/authorized_keys
+sed -i '/.*-general$/d' /root/.ssh/authorized_keys
 sed -i '/testimage$/d' /root/.ssh/authorized_keys
 
 ## Add custom metric script
@@ -181,7 +181,7 @@ resource "aws_instance" "app1" {
   iam_instance_profile        = aws_iam_instance_profile.cwa.id
   key_name                    = aws_key_pair.cwa.key_name
   user_data_base64            = base64encode(local.app_userdata)
-  user_data_replace_on_change = true
+  user_data_replace_on_change = false
   metadata_options {
     http_tokens = "optional"
   }

terraform/environments/contract-work-administration/application_variables.json

@@ -43,9 +43,9 @@
       "database_diskspace_threshold": "95",
       "database_read_write_ops_threshold": "1100000",
       "database_oradata_queue_length_threshold": "3",
-      "old_mail_server_url": "mail.aws.dev.legalservices.gov.uk",
+      "old_mail_server_url": "mail.aws.tst.legalservices.gov.uk",
       "laa_mail_relay_url": "laa-mail.laa-development.modernisation-platform.service.justice.gov.uk",
-      "old_domain_name": "dev.legalservices.gov.uk",
+      "old_domain_name": "tst.legalservices.gov.uk",
       "app_disk_space_alert_threshold": "92",
       "lz_account_id": "013163512034"
     },

terraform/environments/contract-work-administration/concurrent_manager.tf

@@ -63,7 +63,7 @@ do
 done
 
 echo "Updating /etc/rc.local file"
-cat <<EOT > etc/rc.local
+cat <<EOT > /etc/rc.local
 #!/bin/sh
 #
 # This script will be executed *after* all the other init scripts.
@@ -116,8 +116,8 @@ sed -i 's/${local.application_data.accounts[local.environment].old_domain_name}/
 
 ## Remove SSH key allowed
 echo "Removing old SSH key"
-sed -i '/development-general$/d' /home/ec2-user/.ssh/authorized_keys
-sed -i '/development-general$/d' /root/.ssh/authorized_keys
+sed -i '/.*-general$/d' /home/ec2-user/.ssh/authorized_keys
+sed -i '/.*-general$/d' /root/.ssh/authorized_keys
 sed -i '/testimage$/d' /root/.ssh/authorized_keys
 
 ## Add custom metric script
@@ -158,7 +158,7 @@ resource "aws_instance" "concurrent_manager" {
   iam_instance_profile        = aws_iam_instance_profile.cwa.id
   key_name                    = aws_key_pair.cwa.key_name
   user_data_base64            = base64encode(local.cm_userdata)
-  user_data_replace_on_change = true
+  user_data_replace_on_change = false
   metadata_options {
     http_tokens = "optional"
   }

terraform/environments/contract-work-administration/database.tf

@@ -99,6 +99,8 @@ echo "Adding disk space script"
 chmod 766 /home/oracle/scripts/disk_space.sh
 sed -i "s/SLACK_ALERT_URL/$SLACK_ALERT_URL/g" /home/oracle/scripts/disk_space.sh
 
+sed -i "/^mail.*tablespace.warning$/c\mailx -s \"\$ORACLE_SID on \$\{hostname\}: ${upper(local.application_data.accounts[local.environment].env_short)} CWA Tablespace Warning\" $SLACK_ALERT_URL < /tmp/tablespace.warning" /home/oracle/scripts/tablespace1.sh
+
 echo "Setting up AWS EBS backup"
 INSTANCE_ID=$(curl http://169.254.169.254/latest/meta-data/instance-id)
 cat <<EOT > /home/oracle/scripts/aws_ebs_backup.sh
@@ -113,7 +115,9 @@ chmod 744 /home/oracle/scripts/aws_ebs_backup.sh
 
 echo "Setting up cron jobs"
 su oracle -c "crontab -l > /home/oracle/oraclecrontab.txt"
+sed -i '/disk_space.sh/d' /home/oracle/oraclecrontab.txt
 echo "00 02 * * * /home/oracle/scripts/aws_ebs_backup.sh > /tmp/aws_ebs_backup.log" >> /home/oracle/oraclecrontab.txt
+echo "0,30 08-17 * * 1-5 /home/oracle/scripts/disk_space.sh ${upper(local.application_data.accounts[local.environment].env_short)} ${local.application_data.accounts[local.environment].app_disk_space_alert_threshold} >/tmp/disk_space.trc 2>&1" >> /home/oracle/oraclecrontab.txt
 
 chown oracle:oinstall /home/oracle/oraclecrontab.txt
 chmod 744 /home/oracle/oraclecrontab.txt
@@ -124,8 +128,8 @@ ln -s /bin/mail /bin/mailx
 
 ## Remove SSH key allowed
 echo "Removing old SSH key"
-sed -i '/development-general$/d' /home/ec2-user/.ssh/authorized_keys
-sed -i '/development-general$/d' /root/.ssh/authorized_keys
+sed -i '/.*-general$/d' /home/ec2-user/.ssh/authorized_keys
+sed -i '/.*-general$/d' /root/.ssh/authorized_keys
 sed -i '/testimage$/d' /root/.ssh/authorized_keys
 
 ## Add custom metric script

terraform/environments/contract-work-administration/scripts/disk-space-alert.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 if [ $# -ne 2 ]; then
- echo "1st parameter is ENV, 2nd parameter is  % usage. 3rd parameter is the Slack URL to alert to"
+ echo "1st parameter is ENV, 2nd parameter is  % usage"
 else
 
 

terraform/environments/corporate-information-system/ec2.tf

@@ -27,7 +27,7 @@ resource "aws_instance" "cis_db_instance" {
   }
 
   metadata_options {
-    http_tokens = "optional"
+    http_tokens = "required"
   }
 
   tags = merge(

terraform/environments/corporate-information-system/locals.tf

@@ -14,6 +14,16 @@ sed -i 's/#ClientAliveInterval.*/ClientAliveInterval 1200/' /etc/ssh/sshd_config
 sed -i 's/#ClientAliveCountMax.*/ClientAliveCountMax 3/' /etc/ssh/sshd_config
 service sshd restart 
 
+# Add TCP keepalive time to sysctl.conf ---> keepalive solution
+echo "net.ipv4.tcp_keepalive_time = 300" >> /etc/sysctl.conf
+sysctl -p
+
+# Add SQLNET.EXPIRE_TIME to sqlnet.ora ---> keepalive solution
+echo "SQLNET.EXPIRE_TIME = 5" >> /oracle/software/product/10.2.0/network/admin/sqlnet.ora
+
+# Modify tnsnames.ora to insert (ENABLE=broken) ---> keepalive solution
+sed -i '/(DESCRIPTION =/a\\  (ENABLE=broken)' /oracle/software/product/10.2.0/network/admin/tnsnames.ora
+
 # Changes to oracle files
 sed -i 's|cis.*legalservices.gov.uk:8080|${local.application_name_short}.${data.aws_route53_zone.external.name}:8080|' /home/batman/bin/dkj-shell-funcs
 sed -i 's|cis.*legalservices.gov.uk|${local.application_name_short}.${data.aws_route53_zone.external.name}|' /oracle/software/product/10.2.0/network/admin/listener.ora

terraform/environments/delius-core/fargate_graceful_retirement.tf

@@ -0,0 +1,7 @@
+module "fargate_graceful_retirement" {
+  count                   = local.environment == "development" ? 1 : 0
+  source                  = "../../modules/fargate_graceful_retirement"
+  restart_time            = "22:00"
+  restart_day_of_the_week = "WEDNESDAY"
+  debug_logging           =  true
+}

terraform/environments/delius-core/locals_development.tf

@@ -38,6 +38,7 @@ locals {
     instance_policies = {
       "business_unit_kms_key_access" = aws_iam_policy.business_unit_kms_key_access
     }
+    inline_ebs             = false
     primary_instance_count = 1
     standby_count          = 2
     ebs_volumes = {

terraform/environments/delius-core/locals_preproduction.tf

@@ -37,6 +37,7 @@ locals {
     instance_policies = {
       "business_unit_kms_key_access" = aws_iam_policy.business_unit_kms_key_access
     }
+    inline_ebs             = true
     primary_instance_count = 0
     standby_count          = 0
     ebs_volumes = {

terraform/environments/delius-core/locals_stage.tf

@@ -32,8 +32,11 @@ locals {
 
 
   db_config_stage = {
-    instance_type          = "r7i.2xlarge"
-    ami_name_regex         = "^delius_core_ol_8_5_oracle_db_19c_patch_2024-06-04T11-24-58.162Z"
+    instance_type  = "r7i.2xlarge"
+    ami_name_regex = "^delius_core_ol_8_5_oracle_db_19c_patch_2024-06-04T11-24-58.162Z"
+
+    inline_ebs = true
+
     primary_instance_count = 1
     standby_count          = 0
 

terraform/environments/delius-core/locals_test.tf

@@ -38,6 +38,9 @@ locals {
     instance_policies = {
       "business_unit_kms_key_access" = aws_iam_policy.business_unit_kms_key_access
     }
+
+    inline_ebs = true
+
     primary_instance_count = 1
     standby_count          = 0
     ebs_volumes = {

terraform/environments/delius-core/modules/components/dms/cloudwatch-alarms.tf

@@ -70,8 +70,8 @@ resource "aws_cloudwatch_metric_alarm" "dms_cdc_latency_source" {
   statistic           = "Average"
   metric_name         = "CDCLatencySource"
   comparison_operator = "GreaterThanThreshold"
-  threshold           = 10
-  evaluation_periods  = 2
+  threshold           = 15
+  evaluation_periods  = 3
   period              = 30
   actions_enabled     = true
   alarm_actions       = [aws_sns_topic.dms_alerting.arn]
@@ -92,8 +92,8 @@ resource "aws_cloudwatch_metric_alarm" "dms_cdc_latency_target" {
   statistic           = "Average"
   metric_name         = "CDCLatencyTarget"
   comparison_operator = "GreaterThanThreshold"
-  threshold           = 10
-  evaluation_periods  = 2
+  threshold           = 15
+  evaluation_periods  = 3
   period              = 30
   actions_enabled     = true
   alarm_actions       = [aws_sns_topic.dms_alerting.arn]

terraform/environments/delius-core/modules/components/oracle_db_instance/instance.tf

@@ -8,7 +8,7 @@ locals {
     metadata_endpoint_enabled    = var.metadata_options.http_endpoint
     metadata_options_http_tokens = var.metadata_options.http_tokens
     monitoring                   = var.monitoring
-    ebs_block_device_inline      = true
+    ebs_block_device_inline      = false
     vpc_security_group_ids       = var.security_group_ids
     private_dns_name_options = {
       enable_resource_name_dns_aaaa_record = false

terraform/environments/delius-core/modules/components/oracle_db_instance/variables.tf

@@ -176,3 +176,9 @@ variable "sns_topic_arn" {
   description = "The ARN of the SNS topic"
   type        = string
 }
+
+variable "inline_ebs" {
+  default     = true
+  type        = bool
+  description = "Whether to create EBS volumes inline with the instance"
+}