From 08f79ae9f9c79eff7f39c3665bbc13764d4028d6 Mon Sep 17 00:00:00 2001 From: julialawrence Date: Mon, 15 Jul 2024 09:08:46 +0100 Subject: [PATCH 1/8] Adding role for sharing into Analytical Platform Data Production account. --- .../application_variables.json | 14 ++- .../digital-prison-reporting/locals.tf | 11 +- .../digital-prison-reporting/policy.tf | 104 +++++++++++++++++- 3 files changed, 123 insertions(+), 6 deletions(-) diff --git a/terraform/environments/digital-prison-reporting/application_variables.json b/terraform/environments/digital-prison-reporting/application_variables.json index 7e115ce4dd3..6e2cb066e51 100644 --- a/terraform/environments/digital-prison-reporting/application_variables.json +++ b/terraform/environments/digital-prison-reporting/application_variables.json @@ -573,7 +573,19 @@ "enable_cp_bodmis_k8s_secrets": false, "enable_dbt_k8s_secrets": true, "dpr_generic_athena_workgroup": true, - "analytics_generic_athena_workgroup": true + "analytics_generic_athena_workgroup": true, + "analytical_platform_share": { + "target_account_name": "analytcal-platform-data-production", + "target_account_id": "593291632749", + "data_locations": [ + "dpr-structured-historical-preproduction" + ], + "glue_database": "curated_prisons_history_preprod_dbt", + "glue_tables": [ + "nomis_offender_course_attendances", + "nomis_offender_program_profiles" + ] + } }, "production": { "project_short_id": "dpr", diff --git a/terraform/environments/digital-prison-reporting/locals.tf b/terraform/environments/digital-prison-reporting/locals.tf index 52c513d6cbd..401f2f3d6d0 100644 --- a/terraform/environments/digital-prison-reporting/locals.tf +++ b/terraform/environments/digital-prison-reporting/locals.tf @@ -301,14 +301,14 @@ locals { # CW Insights enable_cw_insights = local.application_data.accounts[local.environment].setup_cw_insights - # Setup Athena Workgroups + # Setup Athena Workgroups setup_dpr_generic_athena_workgroup = local.application_data.accounts[local.environment].dpr_generic_athena_workgroup setup_analytics_generic_athena_workgroup = local.application_data.accounts[local.environment].analytics_generic_athena_workgroup # Sonatype Secrets setup_sonatype_secrets = local.application_data.accounts[local.environment].setup_sonatype_secrets - # Nomis Secrets PlaceHolder + # Nomis Secrets PlaceHolder nomis_secrets_placeholder = { db_name = "nomis" password = "placeholder" @@ -319,7 +319,7 @@ locals { port = "1521" } - # Bodmis Secrets PlaceHolder + # Bodmis Secrets PlaceHolder bodmis_secrets_placeholder = { db_name = "bodmis" password = "placeholder" @@ -373,7 +373,7 @@ locals { cloud_platform_k8s_cluster_context = "placeholder" } - # Analytics Platform, DBT Secrets + # Analytics Platform, DBT Secrets enable_dbt_k8s_secrets = local.application_data.accounts[local.environment].enable_dbt_k8s_secrets dbt_k8s_secrets_placeholder = { oidc_cluster_identifier = "placeholder" @@ -394,6 +394,9 @@ locals { username = module.datamart.redshift_master_user } + analytical_platform_sharing = can(local.application_data.accounts[local.environment].analycal_platform_share.target_account_id) + + all_tags = merge( local.tags, { diff --git a/terraform/environments/digital-prison-reporting/policy.tf b/terraform/environments/digital-prison-reporting/policy.tf index fab623b83ab..fbb3308ef3d 100644 --- a/terraform/environments/digital-prison-reporting/policy.tf +++ b/terraform/environments/digital-prison-reporting/policy.tf @@ -41,6 +41,26 @@ data "aws_iam_policy_document" "glue-policy-data" { type = "AWS" } } + statement { + # Required for cross-account sharing via LakeFormation if producer has existing Glue policy + # ref: https://docs.aws.amazon.com/lake-formation/latest/dg/hybrid-cross-account.html + effect = "Allow" + + actions = [ + "glue:ShareResource" + ] + + principals { + type = "Service" + identifiers = ["ram.amazonaws.com"] + } + + resources = [ + "arn:aws:glue:${local.current_account_region}:${local.current_account_id}:table/*/*", + "arn:aws:glue:${local.current_account_region}:${local.current_account_id}:database/*", + "arn:aws:glue:${local.current_account_region}:${local.current_account_id}:catalog" + ] + } } # Resuse for all S3 read Only @@ -412,7 +432,7 @@ resource "aws_iam_role_policy" "dmsvpcpolicy" { EOF } -### Iam User Role for AWS Redshift Spectrum, +### Iam User Role for AWS Redshift Spectrum, resource "aws_iam_role" "redshift-spectrum-role" { name = "${local.project}-redshift-spectrum-role" @@ -714,3 +734,85 @@ resource "aws_iam_policy" "glue_catalog_readonly" { description = "Glue Catalog Readonly Policy" policy = data.aws_iam_policy_document.glue_catalog_readonly.json } + +# Analytical Platform Share Policy & Role + +data "aws_iam_policy_document" "analytical_platform_share_policy" { + for_each = local.analytical_platform_share + + statement { + effect = "Allow" + actions = [ + "lakeformation:GrantPermissions", + "lakeformation:RevokePermissions", + "lakeformation:BatchGrantPermissions", + "lakeformation:BatchRevokePermissions", + "lakeformation:RegisterResource", + "lakeformation:DeregisterResource", + "lakeformation:ListPermissions" + ] + resources = [ + "arn:aws:lakeformation:${local.current_account_region}:${local.current_account_id}:catalog:${local.current_account_id}" + ] + } + + statement { + effect = "Allow" + actions = ["iam:PutRolePolicy"] + resources = [ + "arn:aws:iam::${local.current_account_id}:role/*/AWSServiceRoleForLakeFormationDataAccess" + ] + } + + statement { + effect = "Allow" + actions = [ + "ram:CreateResourceShare", + "ram:DeleteResourceShare" + ] + resources = [ + "arn:aws:ram:${local.current_account_region}:${local.current_account_id}:resource-share/*" + ] + } + + statement { + effect = "Allow" + actions = [ + "glue:GetTable", + "glue:GetDatabase", + "glue:GetPartition" + ] + resources = concat( + "arn:aws:glue:${local.current_account_region}:${local.current_account_id}:database/${local.analytical_platform_share.glue_database}", + formatlist("arn:aws:glue:${local.current_account_region}:${local.current_account_id}:table/${local.analytical_platform_share.glue_database}/%s", [local.analytical_platform_share.glue_tables]), + "arn:aws:glue:${local.current_account_region}:${local.current_account_id}:catalog" + ) + } +} + +resource "aws_iam_role" "analytical_platform_share_role" { + for_each = local.analytical_platform_share + + name = "analytical-platform-share-role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + AWS = "arn:aws:iam::${each.value.target_account_id}:root" + } + Action = "sts:AssumeRole" + } + ] + }) +} + +resource "aws_iam_role_policy" "analytical_platform_share_policy_attachment" { + for_each = local.analytical_platform_share + + name = "analytical-platform-share-policy" + role = aws_iam_role.analytical_platform_share_role[each.key].name + policy = data.aws_iam_policy_document.analytical_platform_share_policy[each.key].json +} From eae3f8bf507dbc553fb5a540ca7f03aa1843c949 Mon Sep 17 00:00:00 2001 From: julialawrence Date: Tue, 16 Jul 2024 08:47:11 +0100 Subject: [PATCH 2/8] Fixing the local name --- .../environments/digital-prison-reporting/lake_formation.tf | 0 terraform/environments/digital-prison-reporting/locals.tf | 4 +++- terraform/environments/digital-prison-reporting/policy.tf | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 terraform/environments/digital-prison-reporting/lake_formation.tf diff --git a/terraform/environments/digital-prison-reporting/lake_formation.tf b/terraform/environments/digital-prison-reporting/lake_formation.tf new file mode 100644 index 00000000000..e69de29bb2d diff --git a/terraform/environments/digital-prison-reporting/locals.tf b/terraform/environments/digital-prison-reporting/locals.tf index 401f2f3d6d0..b3a9e6e84a7 100644 --- a/terraform/environments/digital-prison-reporting/locals.tf +++ b/terraform/environments/digital-prison-reporting/locals.tf @@ -394,7 +394,9 @@ locals { username = module.datamart.redshift_master_user } - analytical_platform_sharing = can(local.application_data.accounts[local.environment].analycal_platform_share.target_account_id) + analytical_platform_share = can(local.application_data.accounts[local.environment].analytical_platform_share) ? { + "analytical_platform_share" = local.application_data.accounts[local.environment].analytical_platform_share + } : {} all_tags = merge( diff --git a/terraform/environments/digital-prison-reporting/policy.tf b/terraform/environments/digital-prison-reporting/policy.tf index fbb3308ef3d..35e982dfe69 100644 --- a/terraform/environments/digital-prison-reporting/policy.tf +++ b/terraform/environments/digital-prison-reporting/policy.tf @@ -783,9 +783,9 @@ data "aws_iam_policy_document" "analytical_platform_share_policy" { "glue:GetPartition" ] resources = concat( - "arn:aws:glue:${local.current_account_region}:${local.current_account_id}:database/${local.analytical_platform_share.glue_database}", - formatlist("arn:aws:glue:${local.current_account_region}:${local.current_account_id}:table/${local.analytical_platform_share.glue_database}/%s", [local.analytical_platform_share.glue_tables]), - "arn:aws:glue:${local.current_account_region}:${local.current_account_id}:catalog" + ["arn:aws:glue:${local.current_account_region}:${local.current_account_id}:database/${local.analytical_platform_share.glue_database}"], + formatlist("arn:aws:glue:${local.current_account_region}:${local.current_account_id}:table/${local.analytical_platform_share.glue_database}/%s", local.analytical_platform_share.glue_tables), + ["arn:aws:glue:${local.current_account_region}:${local.current_account_id}:catalog"] ) } } From 0fcbf65c752da1792efc02d349ba9e2f4cee82c5 Mon Sep 17 00:00:00 2001 From: julialawrence Date: Wed, 17 Jul 2024 11:50:49 +0100 Subject: [PATCH 3/8] Refactoring the share local --- .../application_variables.json | 8 +++++--- .../environments/digital-prison-reporting/locals.tf | 4 +--- .../environments/digital-prison-reporting/policy.tf | 12 +++++++----- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/terraform/environments/digital-prison-reporting/application_variables.json b/terraform/environments/digital-prison-reporting/application_variables.json index 6e2cb066e51..c7099ddfc0b 100644 --- a/terraform/environments/digital-prison-reporting/application_variables.json +++ b/terraform/environments/digital-prison-reporting/application_variables.json @@ -574,18 +574,20 @@ "enable_dbt_k8s_secrets": true, "dpr_generic_athena_workgroup": true, "analytics_generic_athena_workgroup": true, - "analytical_platform_share": { - "target_account_name": "analytcal-platform-data-production", + "analytical_platform_share": [{ + "target_account_name": "analytical-platform-data-production", "target_account_id": "593291632749", "data_locations": [ "dpr-structured-historical-preproduction" ], + "resource_shares": [{ "glue_database": "curated_prisons_history_preprod_dbt", "glue_tables": [ "nomis_offender_course_attendances", "nomis_offender_program_profiles" ] - } + }] + }] }, "production": { "project_short_id": "dpr", diff --git a/terraform/environments/digital-prison-reporting/locals.tf b/terraform/environments/digital-prison-reporting/locals.tf index b3a9e6e84a7..514d5eb3778 100644 --- a/terraform/environments/digital-prison-reporting/locals.tf +++ b/terraform/environments/digital-prison-reporting/locals.tf @@ -394,9 +394,7 @@ locals { username = module.datamart.redshift_master_user } - analytical_platform_share = can(local.application_data.accounts[local.environment].analytical_platform_share) ? { - "analytical_platform_share" = local.application_data.accounts[local.environment].analytical_platform_share - } : {} + analytical_platform_share = can(local.application_data.accounts[local.environment].analytical_platform_share) ? { for share in local.application_data.accounts[local.environment].analytical_platform_share : share.target_account_name => share } : {} all_tags = merge( diff --git a/terraform/environments/digital-prison-reporting/policy.tf b/terraform/environments/digital-prison-reporting/policy.tf index 35e982dfe69..54478199874 100644 --- a/terraform/environments/digital-prison-reporting/policy.tf +++ b/terraform/environments/digital-prison-reporting/policy.tf @@ -782,11 +782,13 @@ data "aws_iam_policy_document" "analytical_platform_share_policy" { "glue:GetDatabase", "glue:GetPartition" ] - resources = concat( - ["arn:aws:glue:${local.current_account_region}:${local.current_account_id}:database/${local.analytical_platform_share.glue_database}"], - formatlist("arn:aws:glue:${local.current_account_region}:${local.current_account_id}:table/${local.analytical_platform_share.glue_database}/%s", local.analytical_platform_share.glue_tables), - ["arn:aws:glue:${local.current_account_region}:${local.current_account_id}:catalog"] - ) + resources = flatten([ + for resource in each.value.resource_shares : [ + "arn:aws:glue:${local.current_account_region}:${local.current_account_id}:database/${resource.glue_database}", + formatlist("arn:aws:glue:${local.current_account_region}:${local.current_account_id}:table/${resource.glue_database}/%s", resource.glue_tables), + "arn:aws:glue:${local.current_account_region}:${local.current_account_id}:catalog" + ] + ]) } } From db93389348502158bc0859c7b8e25197e8947f66 Mon Sep 17 00:00:00 2001 From: julialawrence Date: Wed, 17 Jul 2024 16:01:08 +0100 Subject: [PATCH 4/8] Adding LakeFormation settings. --- .../environments/digital-prison-reporting/data.tf | 9 ++++++++- .../digital-prison-reporting/lake_formation.tf | 13 +++++++++++++ .../environments/digital-prison-reporting/policy.tf | 6 +++--- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/terraform/environments/digital-prison-reporting/data.tf b/terraform/environments/digital-prison-reporting/data.tf index 6290835e598..02c34ab80d4 100644 --- a/terraform/environments/digital-prison-reporting/data.tf +++ b/terraform/environments/digital-prison-reporting/data.tf @@ -95,7 +95,7 @@ data "aws_secretsmanager_secret_version" "dbt_secrets" { } -# TLS Certificate for OIDC URL, DBT K8s Platform +# TLS Certificate for OIDC URL, DBT K8s Platform data "tls_certificate" "dbt_analytics" { url = "https://oidc.eks.eu-west-2.amazonaws.com/id/${jsondecode(data.aws_secretsmanager_secret_version.dbt_secrets.secret_string)["oidc_cluster_identifier"]}" } @@ -116,4 +116,11 @@ data "aws_secretsmanager_secret" "transfer_component_role_secret" { data "aws_secretsmanager_secret_version" "transfer_component_role_secret_version" { secret_id = data.aws_secretsmanager_secret.transfer_component_role_secret.id +} + +# For Lakeformation Management + +# Retrieves the source role of terraform's current caller identity +data "aws_iam_session_context" "current" { + arn = data.aws_caller_identity.current.arn } \ No newline at end of file diff --git a/terraform/environments/digital-prison-reporting/lake_formation.tf b/terraform/environments/digital-prison-reporting/lake_formation.tf index e69de29bb2d..df5a600d77c 100644 --- a/terraform/environments/digital-prison-reporting/lake_formation.tf +++ b/terraform/environments/digital-prison-reporting/lake_formation.tf @@ -0,0 +1,13 @@ +resource "aws_lakeformation_data_lake_settings" "lake_formation" { + admins = flatten([[for role in aws_iam_role.analytical_platform_share_role : role.arn], data.aws_iam_session_context.current.issuer_arn]) + + create_database_default_permissions { + permissions = ["ALL"] + principal = "IAM_ALLOWED_PRINCIPALS" + } + + create_table_default_permissions { + permissions = ["ALL"] + principal = "IAM_ALLOWED_PRINCIPALS" + } +} \ No newline at end of file diff --git a/terraform/environments/digital-prison-reporting/policy.tf b/terraform/environments/digital-prison-reporting/policy.tf index 54478199874..ca1453a2dc1 100644 --- a/terraform/environments/digital-prison-reporting/policy.tf +++ b/terraform/environments/digital-prison-reporting/policy.tf @@ -41,6 +41,7 @@ data "aws_iam_policy_document" "glue-policy-data" { type = "AWS" } } + statement { # Required for cross-account sharing via LakeFormation if producer has existing Glue policy # ref: https://docs.aws.amazon.com/lake-formation/latest/dg/hybrid-cross-account.html @@ -54,7 +55,6 @@ data "aws_iam_policy_document" "glue-policy-data" { type = "Service" identifiers = ["ram.amazonaws.com"] } - resources = [ "arn:aws:glue:${local.current_account_region}:${local.current_account_id}:table/*/*", "arn:aws:glue:${local.current_account_region}:${local.current_account_id}:database/*", @@ -795,7 +795,7 @@ data "aws_iam_policy_document" "analytical_platform_share_policy" { resource "aws_iam_role" "analytical_platform_share_role" { for_each = local.analytical_platform_share - name = "analytical-platform-share-role" + name = "${each.value.target_account_name}-share-role" assume_role_policy = jsonencode({ Version = "2012-10-17" @@ -814,7 +814,7 @@ resource "aws_iam_role" "analytical_platform_share_role" { resource "aws_iam_role_policy" "analytical_platform_share_policy_attachment" { for_each = local.analytical_platform_share - name = "analytical-platform-share-policy" + name = "${each.value.target_account_name}-share-policy" role = aws_iam_role.analytical_platform_share_role[each.key].name policy = data.aws_iam_policy_document.analytical_platform_share_policy[each.key].json } From 493bf0ad0230cfdba2c4ca8360a9889e664927d2 Mon Sep 17 00:00:00 2001 From: julialawrence Date: Wed, 17 Jul 2024 17:15:02 +0100 Subject: [PATCH 5/8] Explanatory comments and more robust definition of lake formation admins --- .../environments/digital-prison-reporting/lake_formation.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/environments/digital-prison-reporting/lake_formation.tf b/terraform/environments/digital-prison-reporting/lake_formation.tf index df5a600d77c..6d1ffcc44a6 100644 --- a/terraform/environments/digital-prison-reporting/lake_formation.tf +++ b/terraform/environments/digital-prison-reporting/lake_formation.tf @@ -1,12 +1,15 @@ resource "aws_lakeformation_data_lake_settings" "lake_formation" { admins = flatten([[for role in aws_iam_role.analytical_platform_share_role : role.arn], data.aws_iam_session_context.current.issuer_arn]) + # ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lakeformation_data_lake_settings#principal create_database_default_permissions { + # These settings should replicate current behaviour: LakeFormation is Ignored permissions = ["ALL"] principal = "IAM_ALLOWED_PRINCIPALS" } create_table_default_permissions { + # These settings should replicate current behaviour: LakeFormation is Ignored permissions = ["ALL"] principal = "IAM_ALLOWED_PRINCIPALS" } From 20f1d246064bc33f1f0436835eae76e8a625a610 Mon Sep 17 00:00:00 2001 From: julialawrence Date: Thu, 18 Jul 2024 08:43:24 +0100 Subject: [PATCH 6/8] Slight adjustment to admins declaration to handle non-sharing environments --- .../environments/digital-prison-reporting/lake_formation.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/digital-prison-reporting/lake_formation.tf b/terraform/environments/digital-prison-reporting/lake_formation.tf index 6d1ffcc44a6..83001ac204c 100644 --- a/terraform/environments/digital-prison-reporting/lake_formation.tf +++ b/terraform/environments/digital-prison-reporting/lake_formation.tf @@ -1,5 +1,5 @@ resource "aws_lakeformation_data_lake_settings" "lake_formation" { - admins = flatten([[for role in aws_iam_role.analytical_platform_share_role : role.arn], data.aws_iam_session_context.current.issuer_arn]) + admins = flatten([[for share in local.analytical_platform_share : aws_iam_role.analytical_platform_share_role[share].arn], data.aws_iam_session_context.current.issuer_arn]) # ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lakeformation_data_lake_settings#principal create_database_default_permissions { From 9e48bb7c3c7ccee1186471dcbf09f3729601673d Mon Sep 17 00:00:00 2001 From: julialawrence Date: Thu, 18 Jul 2024 08:48:49 +0100 Subject: [PATCH 7/8] Keep trying to link local to admin creation --- .../environments/digital-prison-reporting/lake_formation.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/digital-prison-reporting/lake_formation.tf b/terraform/environments/digital-prison-reporting/lake_formation.tf index 83001ac204c..14cfa0dd0f3 100644 --- a/terraform/environments/digital-prison-reporting/lake_formation.tf +++ b/terraform/environments/digital-prison-reporting/lake_formation.tf @@ -1,5 +1,5 @@ resource "aws_lakeformation_data_lake_settings" "lake_formation" { - admins = flatten([[for share in local.analytical_platform_share : aws_iam_role.analytical_platform_share_role[share].arn], data.aws_iam_session_context.current.issuer_arn]) + admins = flatten([[for share in local.analytical_platform_share : aws_iam_role.analytical_platform_share_role[share.target_account_name].arn], data.aws_iam_session_context.current.issuer_arn]) # ref: https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lakeformation_data_lake_settings#principal create_database_default_permissions { From f96ce678004172e4e61fc37f0e0bfac151ac714a Mon Sep 17 00:00:00 2001 From: julialawrence Date: Thu, 18 Jul 2024 09:03:41 +0100 Subject: [PATCH 8/8] EOL added where missing --- terraform/environments/digital-prison-reporting/data.tf | 2 +- .../environments/digital-prison-reporting/lake_formation.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/digital-prison-reporting/data.tf b/terraform/environments/digital-prison-reporting/data.tf index 02c34ab80d4..9d6d43ddf60 100644 --- a/terraform/environments/digital-prison-reporting/data.tf +++ b/terraform/environments/digital-prison-reporting/data.tf @@ -123,4 +123,4 @@ data "aws_secretsmanager_secret_version" "transfer_component_role_secret_version # Retrieves the source role of terraform's current caller identity data "aws_iam_session_context" "current" { arn = data.aws_caller_identity.current.arn -} \ No newline at end of file +} diff --git a/terraform/environments/digital-prison-reporting/lake_formation.tf b/terraform/environments/digital-prison-reporting/lake_formation.tf index 14cfa0dd0f3..766ee13e521 100644 --- a/terraform/environments/digital-prison-reporting/lake_formation.tf +++ b/terraform/environments/digital-prison-reporting/lake_formation.tf @@ -13,4 +13,4 @@ resource "aws_lakeformation_data_lake_settings" "lake_formation" { permissions = ["ALL"] principal = "IAM_ALLOWED_PRINCIPALS" } -} \ No newline at end of file +}