Merge branch 'delius-nextcloud/NIT-1251/Nexctloud_RDS' of https://git…

…hub.com/ministryofjustice/modernisation-platform-environments into delius-nextcloud/NIT-1251/Nexctloud_RDS
ministryofjustice · May 9, 2024 · 44c9103 · 44c9103
2 parents 46edde8 + 92d1f80
commit 44c9103
Show file tree

Hide file tree

Showing 13 changed files with 148 additions and 20 deletions.
diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml
@@ -81,7 +81,7 @@ jobs:
           fetch-depth: 0
       - name: Run Checkov action
         id: checkov
-        uses: bridgecrewio/checkov-action@bcc0f53ec84a48d130b4647803567ede80fe52e6 # v12.2734.0
+        uses: bridgecrewio/checkov-action@65ca23ae4ebac97e587bdb6ef94b708d7b9cc6ea # v12.2736.0
         with:
           directory: ./
           framework: terraform

diff --git a/.github/workflows/nuke-redeploy.yml b/.github/workflows/nuke-redeploy.yml
@@ -65,7 +65,7 @@ jobs:
           aws-region: ${{ env.AWS_REGION }}
 
       - name: Load and Configure Terraform
-        uses: hashicorp/setup-terraform@97f030cf6dc0b4f5e0da352c7bca9cca34579800 # v3.1.0
+        uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
         with:
           terraform_version: "~1"
           terraform_wrapper: false

diff --git a/.github/workflows/reusable_terraform_plan_apply.yml b/.github/workflows/reusable_terraform_plan_apply.yml
@@ -107,7 +107,7 @@ jobs:
           aws-region: "eu-west-2"
 
       - name: Setup Terraform
-        uses: hashicorp/setup-terraform@97f030cf6dc0b4f5e0da352c7bca9cca34579800  # v3.1.0
+        uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8  # v3.1.1
         with:
           terraform_version: "${{ inputs.terraform_version }}"
           terraform_wrapper: false
@@ -286,7 +286,7 @@ jobs:
           aws-region: "eu-west-2"
 
       - name: Setup Terraform
-        uses: hashicorp/setup-terraform@97f030cf6dc0b4f5e0da352c7bca9cca34579800  # v3.1.0
+        uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8  # v3.1.1
         with:
           terraform_version: "${{ inputs.terraform_version }}"
           terraform_wrapper: false

diff --git a/.github/workflows/reusable_terraform_plan_apply_test.yml b/.github/workflows/reusable_terraform_plan_apply_test.yml
@@ -106,7 +106,7 @@ jobs:
           aws-region: "eu-west-2"
 
       - name: Setup Terraform
-        uses: hashicorp/setup-terraform@97f030cf6dc0b4f5e0da352c7bca9cca34579800 # v3.1.0
+        uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
         with:
           terraform_version: "${{ inputs.terraform_version }}"
           terraform_wrapper: false
@@ -251,7 +251,7 @@ jobs:
           aws-region: "eu-west-2"
 
       - name: Setup Terraform
-        uses: hashicorp/setup-terraform@97f030cf6dc0b4f5e0da352c7bca9cca34579800 # v3.1.0
+        uses: hashicorp/setup-terraform@651471c36a6092792c552e8b1bef71e592b462d8 # v3.1.1
         with:
           terraform_version: "${{ inputs.terraform_version }}"
           terraform_wrapper: false

diff --git a/terraform/environments/analytical-platform-ingestion/environment-configuration.tf b/terraform/environments/analytical-platform-ingestion/environment-configuration.tf
@@ -13,9 +13,9 @@ locals {
       observability_platform = "development"
 
       /* Image Versions */
-      scan_image_version     = "0.0.4"
-      transfer_image_version = "0.0.4"
-      notify_image_version   = "0.0.8"
+      scan_image_version     = "0.0.5"
+      transfer_image_version = "0.0.8"
+      notify_image_version   = "0.0.9"
 
       /* Target Buckets */
       target_buckets = ["mojap-land-dev"]
@@ -44,9 +44,9 @@ locals {
       observability_platform = "production"
 
       /* Image Versions */
-      scan_image_version     = "0.0.4"
-      transfer_image_version = "0.0.4"
-      notify_image_version   = "0.0.8"
+      scan_image_version     = "0.0.5"
+      transfer_image_version = "0.0.8"
+      notify_image_version   = "0.0.9"
 
       /* Target Buckets */
       target_buckets = ["mojap-land"]

diff --git a/terraform/environments/analytical-platform-ingestion/security-groups.tf b/terraform/environments/analytical-platform-ingestion/security-groups.tf
@@ -13,6 +13,7 @@ resource "aws_security_group" "transfer_server" {
   vpc_id      = module.vpc.vpc_id
 }
 
+#tfsec:ignore:avd-aws-0104 - The security group is attached to the resource
 module "definition_upload_lambda_security_group" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
 

diff --git a/terraform/environments/digital-prison-reporting/files/log4j2.properties b/terraform/environments/digital-prison-reporting/files/log4j2.properties
@@ -0,0 +1,46 @@
+status = warn
+
+appenders = console
+
+appender.console.type = Console
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss.SSS} %p %c: %msg%n
+
+# The log level for everything that isn't specified below
+rootLogger.level = WARN
+rootLogger.appenderRefs = stdout
+rootLogger.appenderRef.stdout.ref = console
+
+# Our code's log level can be configured separately
+logger.our-code.name = uk.gov.justice.digital
+logger.our-code.level = INFO
+logger.our-code.additivity = false
+logger.our-code.appenderRef.console.ref = console
+
+# Suppress some potentially particularly verbose libraries
+
+logger.AbstractLifeCycle.name = org.spark-project.jetty.util.component.AbstractLifeCycle
+logger.AbstractLifeCycle.level = ERROR
+logger.AbstractLifeCycle.additivity = false
+logger.AbstractLifeCycle.appenderRef.console.ref = console
+
+logger.parquetfull.name = org.apache.parquet
+logger.parquetfull.level = ERROR
+logger.parquetfull.additivity = false
+logger.parquetfull.appenderRef.console.ref = console
+
+logger.parquet.name = parquet
+logger.parquet.level = ERROR
+logger.parquet.additivity = false
+logger.parquet.appenderRef.console.ref = console
+
+logger.RetryingHMSHandler.name = org.apache.hadoop.hive.metastore.RetryingHMSHandler
+logger.RetryingHMSHandler.level = FATAL
+logger.RetryingHMSHandler.additivity = false
+logger.RetryingHMSHandler.appenderRef.console.ref = console
+
+logger.FunctionRegistry.name = org.apache.hadoop.hive.ql.exec.FunctionRegistry
+logger.FunctionRegistry.level = ERROR
+logger.FunctionRegistry.additivity = false
+logger.FunctionRegistry.appenderRef.console.ref = console
diff --git a/terraform/environments/digital-prison-reporting/locals.tf b/terraform/environments/digital-prison-reporting/locals.tf
@@ -329,7 +329,7 @@ locals {
   all_tags = merge(
     local.tags,
     {
-      Name = "${local.application_name}"
+      Name = local.application_name
     }
   )
 }
diff --git a/terraform/environments/digital-prison-reporting/main.tf b/terraform/environments/digital-prison-reporting/main.tf
@@ -4,7 +4,15 @@
 ## Glue Job, Reporting Hub
 ## Glue Cloud Platform Ingestion Job (Load, Reload, CDC)
 locals {
-  glue_avro_registry = split("/", module.glue_registry_avro.registry_name)
+  glue_avro_registry           = split("/", module.glue_registry_avro.registry_name)
+  shared_log4j_properties_path = "s3://${aws_s3_object.glue_job_shared_custom_log4j_properties.bucket}/${aws_s3_object.glue_job_shared_custom_log4j_properties.key}"
+}
+
+resource "aws_s3_object" "glue_job_shared_custom_log4j_properties" {
+  bucket = module.s3_glue_job_bucket.bucket_id
+  key    = "logging/misc-jobs/log4j2.properties"
+  source = "files/log4j2.properties"
+  etag   = filemd5("files/log4j2.properties")
 }
 
 module "glue_reporting_hub_job" {
@@ -45,6 +53,7 @@ module "glue_reporting_hub_job" {
 
   arguments = {
     "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--extra-files"                         = local.shared_log4j_properties_path
     "--job-bookmark-option"                 = "job-bookmark-disable"
     "--class"                               = "uk.gov.justice.digital.job.DataHubJob"
     "--dpr.kinesis.stream.arn"              = module.kinesis_stream_ingestor.kinesis_stream_arn
@@ -110,6 +119,7 @@ module "glue_reporting_hub_batch_job" {
 
   arguments = {
     "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--extra-files"                         = local.shared_log4j_properties_path
     "--class"                               = "uk.gov.justice.digital.job.DataHubBatchJob"
     "--datalake-formats"                    = "delta"
     "--dpr.aws.region"                      = local.account_region
@@ -163,6 +173,7 @@ module "glue_reporting_hub_cdc_job" {
 
   arguments = {
     "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--extra-files"                         = local.shared_log4j_properties_path
     "--job-bookmark-option"                 = "job-bookmark-disable"
     "--class"                               = "uk.gov.justice.digital.job.DataHubCdcJob"
     "--datalake-formats"                    = "delta"
@@ -224,6 +235,7 @@ module "glue_hive_table_creation_job" {
 
   arguments = {
     "--extra-jars"                = local.glue_jobs_latest_jar_location
+    "--extra-files"               = local.shared_log4j_properties_path
     "--class"                     = "uk.gov.justice.digital.job.HiveTableCreationJob"
     "--dpr.aws.region"            = local.account_region
     "--dpr.config.s3.bucket"      = module.s3_glue_job_bucket.bucket_id,
@@ -289,6 +301,7 @@ module "glue_s3_file_transfer_job" {
 
   arguments = {
     "--extra-jars"                            = local.glue_jobs_latest_jar_location
+    "--extra-files"                           = local.shared_log4j_properties_path
     "--class"                                 = "uk.gov.justice.digital.job.S3FileTransferJob"
     "--dpr.aws.region"                        = local.account_region
     "--dpr.config.s3.bucket"                  = module.s3_glue_job_bucket.bucket_id,
@@ -355,6 +368,7 @@ module "glue_switch_prisons_hive_data_location_job" {
 
   arguments = {
     "--extra-jars"                = local.glue_jobs_latest_jar_location
+    "--extra-files"               = local.shared_log4j_properties_path
     "--class"                     = "uk.gov.justice.digital.job.SwitchHiveTableJob"
     "--dpr.aws.region"            = local.account_region
     "--dpr.config.s3.bucket"      = module.s3_glue_job_bucket.bucket_id,
@@ -414,6 +428,7 @@ module "glue_s3_data_deletion_job" {
 
   arguments = {
     "--extra-jars"                     = local.glue_jobs_latest_jar_location
+    "--extra-files"                    = local.shared_log4j_properties_path
     "--class"                          = "uk.gov.justice.digital.job.S3DataDeletionJob"
     "--dpr.aws.region"                 = local.account_region
     "--dpr.config.s3.bucket"           = module.s3_glue_job_bucket.bucket_id,
@@ -467,6 +482,7 @@ module "glue_stop_glue_instance_job" {
 
   arguments = {
     "--extra-jars"     = local.glue_jobs_latest_jar_location
+    "--extra-files"    = local.shared_log4j_properties_path
     "--class"          = "uk.gov.justice.digital.job.StopGlueInstanceJob"
     "--dpr.aws.region" = local.account_region
     "--dpr.log.level"  = local.glue_job_common_log_level
@@ -800,7 +816,7 @@ module "s3_artifacts_store" {
 
   # Dynamic, supports multiple notifications blocks
   bucket_notifications = {
-    "lambda_function_arn" = "${module.domain_builder_flyway_Lambda.lambda_function}"
+    "lambda_function_arn" = module.domain_builder_flyway_Lambda.lambda_function
     "events"              = ["s3:ObjectCreated:*"]
     "filter_prefix"       = "build-artifacts/domain-builder/jars/"
     "filter_suffix"       = ".jar"
@@ -1020,7 +1036,7 @@ module "datamart" {
   tags = merge(
     local.all_tags,
     {
-      Name          = "${local.redshift_cluster_name}"
+      Name          = local.redshift_cluster_name
       Resource_Type = "Redshift Cluster"
     }
   )

diff --git a/terraform/environments/digital-prison-reporting/maintenance_jobs.tf b/terraform/environments/digital-prison-reporting/maintenance_jobs.tf
@@ -63,6 +63,7 @@ module "glue_compact_raw_job" {
 
   arguments = {
     "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--extra-files"                         = local.shared_log4j_properties_path
     "--class"                               = local.compact_job_class
     "--dpr.maintenance.root.path"           = local.raw_zone_nomis_path
     "--datalake-formats"                    = "delta"
@@ -108,6 +109,7 @@ module "glue_compact_structured_job" {
 
   arguments = {
     "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--extra-files"                         = local.shared_log4j_properties_path
     "--class"                               = local.compact_job_class
     "--dpr.maintenance.root.path"           = local.structured_zone_nomis_path
     "--datalake-formats"                    = "delta"
@@ -153,6 +155,7 @@ module "glue_compact_curated_job" {
 
   arguments = {
     "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--extra-files"                         = local.shared_log4j_properties_path
     "--class"                               = local.compact_job_class
     "--dpr.maintenance.root.path"           = local.curated_zone_nomis_path
     "--datalake-formats"                    = "delta"
@@ -198,6 +201,7 @@ module "glue_compact_domain_job" {
 
   arguments = {
     "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--extra-files"                         = local.shared_log4j_properties_path
     "--class"                               = local.compact_job_class
     "--dpr.maintenance.root.path"           = local.domain_zone_root_path
     "--datalake-formats"                    = "delta"
@@ -244,6 +248,7 @@ module "glue_retention_raw_job" {
 
   arguments = {
     "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--extra-files"                         = local.shared_log4j_properties_path
     "--class"                               = local.retention_job_class
     "--dpr.maintenance.root.path"           = local.raw_zone_nomis_path
     "--datalake-formats"                    = "delta"
@@ -289,6 +294,7 @@ module "glue_retention_structured_job" {
 
   arguments = {
     "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--extra-files"                         = local.shared_log4j_properties_path
     "--class"                               = local.retention_job_class
     "--dpr.maintenance.root.path"           = local.structured_zone_nomis_path
     "--datalake-formats"                    = "delta"
@@ -334,6 +340,7 @@ module "glue_retention_curated_job" {
 
   arguments = {
     "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--extra-files"                         = local.shared_log4j_properties_path
     "--class"                               = local.retention_job_class
     "--dpr.maintenance.root.path"           = local.curated_zone_nomis_path
     "--datalake-formats"                    = "delta"
@@ -379,6 +386,7 @@ module "glue_retention_domain_job" {
 
   arguments = {
     "--extra-jars"                          = local.glue_jobs_latest_jar_location
+    "--extra-files"                         = local.shared_log4j_properties_path
     "--class"                               = local.retention_job_class
     "--dpr.maintenance.root.path"           = local.domain_zone_root_path
     "--datalake-formats"                    = "delta"

diff --git a/terraform/environments/digital-prison-reporting/temporary_domain_refresh_jobs.tf b/terraform/environments/digital-prison-reporting/temporary_domain_refresh_jobs.tf
@@ -46,6 +46,7 @@ module "glue_temp_refresh_job_establishment_establishment" {
 
   arguments = {
     "--extra-jars"                   = "s3://${local.project}-artifact-store-${local.environment}/build-artifacts/digital-prison-reporting-jobs/jars/digital-prison-reporting-jobs-${local.artifact_version}.jar"
+    "--extra-files"                  = local.shared_log4j_properties_path
     "--class"                        = "uk.gov.justice.digital.job.DomainRefreshJob"
     "--datalake-formats"             = "delta"
     "--dpr.aws.dynamodb.endpointUrl" = "https://dynamodb.${local.account_region}.amazonaws.com"
@@ -102,6 +103,7 @@ module "glue_temp_refresh_job_establishment_living_unit" {
 
   arguments = {
     "--extra-jars"                   = "s3://${local.project}-artifact-store-${local.environment}/build-artifacts/digital-prison-reporting-jobs/jars/digital-prison-reporting-jobs-${local.artifact_version}.jar"
+    "--extra-files"                  = local.shared_log4j_properties_path
     "--class"                        = "uk.gov.justice.digital.job.DomainRefreshJob"
     "--datalake-formats"             = "delta"
     "--dpr.aws.dynamodb.endpointUrl" = "https://dynamodb.${local.account_region}.amazonaws.com"
@@ -159,6 +161,7 @@ module "glue_temp_refresh_job_movements_movements" {
 
   arguments = {
     "--extra-jars"                   = "s3://${local.project}-artifact-store-${local.environment}/build-artifacts/digital-prison-reporting-jobs/jars/digital-prison-reporting-jobs-${local.artifact_version}.jar"
+    "--extra-files"                  = local.shared_log4j_properties_path
     "--class"                        = "uk.gov.justice.digital.job.DomainRefreshJob"
     "--datalake-formats"             = "delta"
     "--dpr.aws.dynamodb.endpointUrl" = "https://dynamodb.${local.account_region}.amazonaws.com"
@@ -215,6 +218,7 @@ module "glue_temp_refresh_job_prisoner_prisoner" {
 
   arguments = {
     "--extra-jars"                   = "s3://${local.project}-artifact-store-${local.environment}/build-artifacts/digital-prison-reporting-jobs/jars/digital-prison-reporting-jobs-${local.artifact_version}.jar"
+    "--extra-files"                  = local.shared_log4j_properties_path
     "--class"                        = "uk.gov.justice.digital.job.DomainRefreshJob"
     "--datalake-formats"             = "delta"
     "--dpr.aws.dynamodb.endpointUrl" = "https://dynamodb.${local.account_region}.amazonaws.com"