Merge branch 'main' into standard-lifecycle-v1

ministryofjustice · Dec 3, 2024 · 2c5836d · 2c5836d
2 parents 2fe3b65 + 8d2f502
commit 2c5836d
Show file tree

Hide file tree

Showing 103 changed files with 2,569 additions and 1,573 deletions.
diff --git a/.github/workflows/awsnuke.yml b/.github/workflows/awsnuke.yml
@@ -133,11 +133,11 @@ jobs:
       - name: Slack failure notification
         uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
+          webhook-type: incoming-webhook
           payload: |
             {"blocks":[{"type": "section","text": {"type": "mrkdwn","text": ":no_entry: Failed GitHub Action:"}},{"type": "section","fields":[{"type": "mrkdwn","text": "*Workflow:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"},{"type": "mrkdwn","text": "*Job:*\n${{ github.job }}"},{"type": "mrkdwn","text": "*Repo:*\n${{ github.repository }}"}]}]}
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
         if: ${{ failure() }}
     env:
       ACCOUNT_NAME: ${{ matrix.nuke_accts }}
@@ -217,11 +217,11 @@ jobs:
       - name: Slack failure notification
         uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
+          webhook-type: incoming-webhook
           payload: |
             {"blocks":[{"type": "section","text": {"type": "mrkdwn","text": ":no_entry: Failed GitHub Action:"}},{"type": "section","fields":[{"type": "mrkdwn","text": "*Workflow:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"},{"type": "mrkdwn","text": "*Job:*\n${{ github.job }}"},{"type": "mrkdwn","text": "*Repo:*\n${{ github.repository }}"}]}]}
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
         if: ${{ failure() }}
     env:
       AWS_ACCESS_KEY_ID:  ${{ secrets.TESTING_AWS_ACCESS_KEY_ID }}

diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml
@@ -81,7 +81,7 @@ jobs:
           fetch-depth: 0
       - name: Run Checkov action
         id: checkov
-        uses: bridgecrewio/checkov-action@5ae57a8860ce0657cb09591f5b8b8d9ead999a68 # v12.2920.0
+        uses: bridgecrewio/checkov-action@b8f970b660bc01f598fc2f108eabd9e8dee728f8 # v12.2924.0
         with:
           directory: ./
           framework: terraform

diff --git a/.github/workflows/nuke-redeploy.yml b/.github/workflows/nuke-redeploy.yml
@@ -93,11 +93,11 @@ jobs:
       - name: Slack failure notification
         uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0
         with:
+          webhook-type: incoming-webhook
           payload: |
             {"blocks":[{"type": "section","text": {"type": "mrkdwn","text": ":no_entry: Failed GitHub Action:"}},{"type": "section","fields":[{"type": "mrkdwn","text": "*Workflow:*\n<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>"},{"type": "mrkdwn","text": "*Job:*\n${{ github.job }}"},{"type": "mrkdwn","text": "*Repo:*\n${{ github.repository }}"}]}]}
         env:
           SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
-          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
         if: ${{ failure() }}
 
     env:

diff --git a/terraform/environments/analytical-platform-compute/environment-configuration.tf b/terraform/environments/analytical-platform-compute/environment-configuration.tf
@@ -43,6 +43,7 @@ locals {
 
       /* UI */
       ui_hostname = "development.analytical-platform.service.justice.gov.uk"
+
     }
     test = {
       /* VPC */
@@ -131,6 +132,11 @@ locals {
 
       /* UI */
       ui_hostname = "analytical-platform.service.justice.gov.uk"
+
+      /* LF Domain Tags */
+      cadet_lf_tags = {
+        domain = ["bold", "civil", "courts", "general", "criminal_history", "development_sandpit", "electronic_monitoring", "finance", "interventions", "opg", "performance", "risk", "people", "prison", "probation", "victims", "victims_case_management"] # extracted from bucket paths
+      }
     }
   }
 }
diff --git a/terraform/environments/analytical-platform-compute/helm-charts-applications.tf b/terraform/environments/analytical-platform-compute/helm-charts-applications.tf
diff --git a/terraform/environments/analytical-platform-compute/iam-policies.tf b/terraform/environments/analytical-platform-compute/iam-policies.tf
@@ -350,7 +350,7 @@ module "data_production_mojap_derived_bucket_lake_formation_policy" {
   tags = local.tags
 }
 
-data "aws_iam_policy_document" "analytical_platform_cadet_runner_compute_policy" {
+data "aws_iam_policy_document" "copy_apdp_cadet_metadata_to_compute_policy" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
   statement {
@@ -396,6 +396,7 @@ data "aws_iam_policy_document" "analytical_platform_cadet_runner_compute_policy"
     ]
   }
   statement {
+    sid    = "GlueFetchMetadataAccess"
     effect = "Allow"
     actions = [
       "glue:GetTable",
@@ -404,18 +405,52 @@ data "aws_iam_policy_document" "analytical_platform_cadet_runner_compute_policy"
     ]
     resources = ["arn:aws:glue:eu-west-2:${data.aws_caller_identity.current.account_id}:*"]
   }
+  statement {
+    sid    = "AthenaQueryBucketAccess"
+    effect = "Allow"
+    actions = [
+      "s3:GetBucketLocation",
+      "s3:GetObject",
+      "s3:ListBucket",
+      "s3:ListBucketMultipartUploads",
+      "s3:ListMultipartUploadParts",
+      "s3:AbortMultipartUpload",
+      "s3:PutObject"
+    ]
+    resources = [
+      module.mojap_compute_athena_query_results_bucket_eu_west_2.s3_bucket_arn,
+      "${module.mojap_compute_athena_query_results_bucket_eu_west_2.s3_bucket_arn}/*"
+    ]
+  }
+  statement {
+    sid    = "AlterLFTags"
+    effect = "Allow"
+    actions = [
+      "lakeformation:ListLFTags",
+      "lakeformation:GetLFTag",
+      "lakeformation:CreateLFTag",
+      "lakeformation:UpdateLFTag",
+      "lakeformation:AddLFTagsToResource",
+      "lakeformation:RemoveLFTagsFromResource",
+      "lakeformation:GetResourceLFTags",
+      "lakeformation:SearchTablesByLFTags",
+      "lakeformation:SearchDatabasesByLFTags",
+    ]
+    resources = ["*"]
+  }
+
 }
 
-module "analytical_platform_cadet_runner_compute_policy" {
+module "copy_apdp_cadet_metadata_to_compute_policy" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
 
   source  = "terraform-aws-modules/iam/aws//modules/iam-policy"
   version = "5.48.0"
 
-  name_prefix = "analytical-platform-cadet-runner-compute-policy"
+  name_prefix = "copy-apdp-cadet-metadata-to-compute-"
 
-  policy = data.aws_iam_policy_document.analytical_platform_cadet_runner_compute_policy.json
+  policy = data.aws_iam_policy_document.copy_apdp_cadet_metadata_to_compute_policy.json
 
   tags = local.tags
 }
diff --git a/terraform/environments/analytical-platform-compute/iam-roles.tf b/terraform/environments/analytical-platform-compute/iam-roles.tf
@@ -374,7 +374,7 @@ module "lake_formation_to_data_production_mojap_derived_tables_role" {
   tags = local.tags
 }
 
-module "analytical_platform_cadet_runner" {
+module "copy_apdp_cadet_metadata_to_compute_assumable_role" {
   #checkov:skip=CKV_TF_1:Module registry does not support commit hashes for versions
   #checkov:skip=CKV_TF_2:Module registry does not support tags for versions
   source  = "terraform-aws-modules/iam/aws//modules/iam-assumable-role"
@@ -384,9 +384,13 @@ module "analytical_platform_cadet_runner" {
   trusted_role_arns      = ["arn:aws:iam::${local.environment_management.account_ids["analytical-platform-data-production"]}:role/create-a-derived-table"]
   create_role            = true
   role_requires_mfa      = false
-  role_name              = "analytical-platform-cadet-runner-assumable"
+  role_name              = "copy-apdp-cadet-metadata-to-compute"
 
-  custom_role_policy_arns = [module.analytical_platform_cadet_runner_compute_policy.arn]
+  custom_role_policy_arns = [module.copy_apdp_cadet_metadata_to_compute_policy.arn]
   # number_of_custom_role_policy_arns = 1
+}
 
+moved {
+  from = module.analytical_platform_cadet_runner
+  to   = module.copy_apdp_cadet_metadata_to_compute_assumable_role
 }
diff --git a/terraform/environments/analytical-platform-compute/lakeformation-data-lake-settings.tf b/terraform/environments/analytical-platform-compute/lakeformation-data-lake-settings.tf
@@ -5,7 +5,8 @@ resource "aws_lakeformation_data_lake_settings" "london" {
     module.analytical_platform_ui_service_role.iam_role_arn,
     module.analytical_platform_data_eng_dba_service_role.iam_role_arn,
     "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-reserved/sso.amazonaws.com/${data.aws_region.current.name}/${one(data.aws_iam_roles.data_engineering_sso_role.names)}",
-    "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-reserved/sso.amazonaws.com/${data.aws_region.current.name}/${one(data.aws_iam_roles.eks_sso_access_role.names)}"
+    "arn:aws:iam::${data.aws_caller_identity.current.account_id}:role/aws-reserved/sso.amazonaws.com/${data.aws_region.current.name}/${one(data.aws_iam_roles.eks_sso_access_role.names)}",
+    module.copy_apdp_cadet_metadata_to_compute_assumable_role.iam_role_arn
   ]
 }
 

diff --git a/terraform/environments/analytical-platform-compute/lakeformation-permissions.tf b/terraform/environments/analytical-platform-compute/lakeformation-permissions.tf
@@ -0,0 +1,64 @@
+
+resource "aws_lakeformation_lf_tag" "source" {
+  count  = terraform.workspace == "analytical-platform-compute-production" ? 1 : 0
+  key    = "source"
+  values = ["create-a-derived-table"]
+}
+
+resource "aws_lakeformation_permissions" "cadet_all_data" {
+  for_each = (terraform.workspace == "analytical-platform-compute-production" ?
+  toset(["TABLE", "DATABASE"]) : toset([]))
+
+  principal   = module.copy_apdp_cadet_metadata_to_compute_assumable_role.iam_role_arn
+  permissions = ["ALL"] # https://docs.aws.amazon.com/lake-formation/latest/dg/lf-permissions-reference.html
+
+  lf_tag_policy {
+    resource_type = each.value
+    expression {
+      key    = "source"
+      values = ["create-a-derived-table"]
+    }
+  }
+}
+
+resource "aws_lakeformation_lf_tag" "domain" {
+  for_each = try(local.environment_configuration.cadet_lf_tags, {})
+  key = each.key
+  values = each.value
+}
+
+resource "aws_lakeformation_permissions" "cadet_domain_database_data" {
+  for_each = try(local.environment_configuration.cadet_lf_tags, {})
+
+  principal   = module.copy_apdp_cadet_metadata_to_compute_assumable_role.iam_role_arn
+  permissions = ["ALL"] # https://docs.aws.amazon.com/lake-formation/latest/dg/lf-permissions-reference.html
+
+  lf_tag_policy {
+    resource_type = "DATABASE"
+    expression {
+      key    = each.key
+      values = each.value
+    }
+  }
+}
+
+resource "aws_lakeformation_permissions" "cadet_domain_table_data" {
+  for_each = try(local.environment_configuration.cadet_lf_tags, {})
+
+  principal   = module.copy_apdp_cadet_metadata_to_compute_assumable_role.iam_role_arn
+  permissions = ["ALL"] # https://docs.aws.amazon.com/lake-formation/latest/dg/lf-permissions-reference.html
+
+  lf_tag_policy {
+    resource_type = "TABLE"
+    expression {
+      key    = each.key
+      values = each.value
+    }
+  }
+}
+
+import {
+  for_each = try(local.environment_configuration.cadet_lf_tags, {})
+  to = aws_lakeformation_lf_tag.domain[each.key]
+  id = "${local.environment_management.account_ids[terraform.workspace]}:${each.key}"
+}
diff --git a/...ents/analytical-platform-compute/src/helm/values/amazon-prometheus-proxy/values.yml.tftpl b/...ents/analytical-platform-compute/src/helm/values/amazon-prometheus-proxy/values.yml.tftpl
@@ -14,6 +14,10 @@ alertmanager:
 grafana:
   enabled: false
 
+kube-state-metrics:
+  extraArgs:
+    - --metric-labels-allowlist=pods=[*]
+
 prometheus:
   agentMode: true
   serviceAccount:

diff --git a/terraform/environments/apex/application_variables.json b/terraform/environments/apex/application_variables.json
@@ -153,17 +153,17 @@
       "lz_account_id": "484221692666"
     },
     "production": {
-      "ec2amiid": "ami-0fd11105aa6dcd77d",
+      "ec2amiid": "ami-0484bb2dca8e69b20",
       "ec2instancetype": "t3.xlarge",
       "workspace_cidr": "10.200.16.0/20",
       "u01_orahome_size": "20",
-      "u01_orahome_snapshot": "snap-0ff8db461fc1cc4af",
+      "u01_orahome_snapshot": "snap-0f65cdb141493bd72",
       "u02_oradata_size": "100",
-      "u02_oradata_snapshot": "snap-00aee055837c0f329",
+      "u02_oradata_snapshot": "snap-0497aeb9f2f4029e3",
       "u03_redo_size": "50",
-      "u03_redo_snapshot": "snap-07292192ef32185e2",
+      "u03_redo_snapshot": "snap-0f0890c80d8c4efe9",
       "u04_arch_size": "50",
-      "u04_arch_snapshot": "snap-0d419d52da8066a22",
+      "u04_arch_snapshot": "snap-07f384ea32af67b25",
       "container_instance_type": "linux",
       "instance_type": "m5.large",
       "key_name": "",

diff --git a/terraform/environments/apex/ec2.tf b/terraform/environments/apex/ec2.tf
@@ -65,14 +65,14 @@ resource "aws_vpc_security_group_ingress_rule" "db_ecs" {
   to_port                      = 1521
 }
 
-# resource "aws_vpc_security_group_ingress_rule" "db_mp_vpc" {
-#   security_group_id = aws_security_group.database.id
-#   description       = "Allow MP VPC (OAS) to access database instance"
-#   cidr_ipv4         = data.aws_vpc.shared.cidr_block
-#   from_port         = 1521
-#   ip_protocol       = "tcp"
-#   to_port           = 1521
-# }
+resource "aws_vpc_security_group_ingress_rule" "db_mp_vpc" {
+  security_group_id = aws_security_group.database.id
+  description       = "Allow MP VPC (OAS) to access database instance"
+  cidr_ipv4         = data.aws_vpc.shared.cidr_block
+  from_port         = 1521
+  ip_protocol       = "tcp"
+  to_port           = 1521
+}
 
 resource "aws_vpc_security_group_ingress_rule" "db_lambda" {
   security_group_id            = aws_security_group.database.id
@@ -83,24 +83,24 @@ resource "aws_vpc_security_group_ingress_rule" "db_lambda" {
   to_port                      = 22
 }
 
-# resource "aws_vpc_security_group_ingress_rule" "db_workspace" {
-#   security_group_id = aws_security_group.database.id
-#   description       = "Database listener port access to Workspaces"
-#   cidr_ipv4         = local.application_data.accounts[local.environment].workspace_cidr
-#   from_port         = 1521
-#   ip_protocol       = "tcp"
-#   to_port           = 1521
-# }
+resource "aws_vpc_security_group_ingress_rule" "db_workspace" {
+  security_group_id = aws_security_group.database.id
+  description       = "Database listener port access to Workspaces"
+  cidr_ipv4         = local.application_data.accounts[local.environment].workspace_cidr
+  from_port         = 1521
+  ip_protocol       = "tcp"
+  to_port           = 1521
+}
 
 # This is a temp rule whilst OAS resides in LZ
-# resource "aws_vpc_security_group_ingress_rule" "oas_lz" {
-#   security_group_id = aws_security_group.database.id
-#   description       = "Allow OAS in LZ to access APEX"
-#   cidr_ipv4         = local.application_data.accounts[local.environment].oas_lz_cidr
-#   from_port         = 1521
-#   ip_protocol       = "tcp"
-#   to_port           = 1521
-# }
+resource "aws_vpc_security_group_ingress_rule" "oas_lz" {
+  security_group_id = aws_security_group.database.id
+  description       = "Allow OAS in LZ to access APEX"
+  cidr_ipv4         = local.application_data.accounts[local.environment].oas_lz_cidr
+  from_port         = 1521
+  ip_protocol       = "tcp"
+  to_port           = 1521
+}
 
 resource "aws_vpc_security_group_egress_rule" "db_outbound" {
   security_group_id = aws_security_group.database.id