diff --git a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf index 177f940a86..46e9a1309a 100644 --- a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf +++ b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf @@ -15,36 +15,34 @@ # tfdoc:file:description drop off project and resources. locals { - drop_orch_service_accounts = [ - module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email - ] -} - -module "drop-project" { - source = "../../../modules/project" - parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "drp${local.project_suffix}" - group_iam = { - (local.groups.data-engineers) = [ - "roles/bigquery.dataEditor", - "roles/pubsub.editor", - "roles/storage.admin", + iam_drp = { + "roles/bigquery.dataEditor" = [ + module.drop-sa-bq-0.iam_email, local.groups_iam.data-engineers + ] + "roles/bigquery.user" = [ + module.load-sa-df-0.iam_email, local.groups_iam.data-engineers + ] + "roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email] + "roles/pubsub.subscriber" = [ + module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email ] - } - iam = { - "roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email] - "roles/bigquery.user" = [module.load-sa-df-0.iam_email] - "roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email] - "roles/pubsub.subscriber" = concat( - local.drop_orch_service_accounts, [module.load-sa-df-0.iam_email] - ) - "roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email] "roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email] "roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email] - "roles/storage.admin" = [module.load-sa-df-0.iam_email] + "roles/storage.objectAdmin" = [ + module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email + ] } +} + +module "drop-project" { + source = "../../../modules/project" + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}" + iam = var.project_config.billing_account_id != null ? local.iam_drp : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/02-load.tf b/blueprints/data-solutions/data-platform-foundations/02-load.tf index 74cb9f8b0c..9702fce1ec 100644 --- a/blueprints/data-solutions/data-platform-foundations/02-load.tf +++ b/blueprints/data-solutions/data-platform-foundations/02-load.tf @@ -15,6 +15,19 @@ # tfdoc:file:description Load project and VPC. locals { + iam_load = { + "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email] + "roles/dataflow.admin" = [ + module.orch-sa-cmp-0.iam_email, + module.load-sa-df-0.iam_email, + local.groups_iam.data-engineers + ] + "roles/dataflow.developer" = [ + local.groups_iam.data-engineers + ] + "roles/dataflow.worker" = [module.load-sa-df-0.iam_email] + "roles/storage.objectAdmin" = local.load_service_accounts + } load_service_accounts = [ "serviceAccount:${module.load-project.service_accounts.robots.dataflow}", module.load-sa-df-0.iam_email @@ -35,26 +48,13 @@ locals { module "load-project" { source = "../../../modules/project" - parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "lod${local.project_suffix}" - group_iam = { - (local.groups.data-engineers) = [ - "roles/compute.viewer", - "roles/dataflow.admin", - "roles/dataflow.developer", - "roles/viewer", - ] - } - iam = { - "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email] - "roles/dataflow.admin" = [ - module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email - ] - "roles/dataflow.worker" = [module.load-sa-df-0.iam_email] - "roles/storage.objectAdmin" = local.load_service_accounts - } + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}" + iam = var.project_config.billing_account_id != null ? local.iam_load : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -86,8 +86,13 @@ module "load-sa-df-0" { name = "load-df-0" display_name = "Data platform Dataflow load service account" iam = { - "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers] - "roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email] + "roles/iam.serviceAccountTokenCreator" = [ + local.groups_iam.data-engineers, + module.orch-sa-cmp-0.iam_email + ], + "roles/iam.serviceAccountUser" = [ + module.orch-sa-cmp-0.iam_email + ] } } @@ -107,11 +112,11 @@ module "load-vpc" { source = "../../../modules/net-vpc" count = local.use_shared_vpc ? 0 : 1 project_id = module.load-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-lod" subnets = [ { ip_cidr_range = "10.10.0.0/24" - name = "default" + name = "${var.prefix}-lod" region = var.region } ] @@ -131,7 +136,7 @@ module "load-nat" { source = "../../../modules/net-cloudnat" count = local.use_shared_vpc ? 0 : 1 project_id = module.load-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-lod" region = var.region router_network = module.load-vpc.0.name } diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf index a202afdd05..fc0eda12e6 100644 --- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf @@ -15,52 +15,22 @@ # tfdoc:file:description Orchestration project and VPC. locals { - orch_subnet = ( - local.use_shared_vpc - ? var.network_config.subnet_self_links.orchestration - : values(module.orch-vpc.0.subnet_self_links)[0] - ) - orch_vpc = ( - local.use_shared_vpc - ? var.network_config.network_self_link - : module.orch-vpc.0.self_link - ) - - # Note: This formatting is needed for output purposes since the fabric artifact registry - # module doesn't yet expose the docker usage path of a registry folder in the needed format. - orch_docker_path = format("%s-docker.pkg.dev/%s/%s", - var.region, module.orch-project.project_id, module.orch-artifact-reg.name) -} - -module "orch-project" { - source = "../../../modules/project" - parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "orc${local.project_suffix}" - group_iam = { - (local.groups.data-engineers) = [ - "roles/bigquery.dataEditor", - "roles/bigquery.jobUser", - "roles/cloudbuild.builds.editor", - "roles/composer.admin", - "roles/composer.environmentAndStorageObjectAdmin", - "roles/iap.httpsResourceAccessor", - "roles/iam.serviceAccountUser", - "roles/storage.objectAdmin", - "roles/storage.admin", - "roles/artifactregistry.admin", - "roles/serviceusage.serviceUsageConsumer", - ] - } - iam = { + iam_orch = { + "roles/artifactregistry.admin" = [local.groups_iam.data-engineers] + "roles/artifactregistry.reader" = [module.load-sa-df-0.iam_email] "roles/bigquery.dataEditor" = [ module.load-sa-df-0.iam_email, module.transf-sa-df-0.iam_email, + local.groups_iam.data-engineers ] "roles/bigquery.jobUser" = [ module.orch-sa-cmp-0.iam_email, + local.groups_iam.data-engineers ] + "roles/cloudbuild.builds.editor" = [local.groups_iam.data-engineers] + "roles/cloudbuild.serviceAgent" = [module.orch-sa-df-build.iam_email] + "roles/composer.admin" = [local.groups_iam.data-engineers] + "roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers] "roles/composer.ServiceAgentV2Ext" = [ "serviceAccount:${module.orch-project.service_accounts.robots.composer}" ] @@ -68,23 +38,46 @@ module "orch-project" { module.orch-sa-cmp-0.iam_email ] "roles/iam.serviceAccountUser" = [ - module.orch-sa-cmp-0.iam_email + module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers ] + "roles/iap.httpsResourceAccessor" = [local.groups_iam.data-engineers] + "roles/serviceusage.serviceUsageConsumer" = [local.groups_iam.data-engineers] "roles/storage.objectAdmin" = [ module.orch-sa-cmp-0.iam_email, module.orch-sa-df-build.iam_email, "serviceAccount:${module.orch-project.service_accounts.robots.composer}", "serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}", - ] - "roles/artifactregistry.reader" = [ - module.load-sa-df-0.iam_email, - ] - "roles/cloudbuild.serviceAgent" = [ - module.orch-sa-df-build.iam_email, + local.groups_iam.data-engineers ] "roles/storage.objectViewer" = [module.load-sa-df-0.iam_email] } - oslogin = false + orch_subnet = ( + local.use_shared_vpc + ? var.network_config.subnet_self_links.orchestration + : values(module.orch-vpc.0.subnet_self_links)[0] + ) + orch_vpc = ( + local.use_shared_vpc + ? var.network_config.network_self_link + : module.orch-vpc.0.self_link + ) + + # Note: This formatting is needed for output purposes since the fabric artifact registry + # module doesn't yet expose the docker usage path of a registry folder in the needed format. + orch_docker_path = format("%s-docker.pkg.dev/%s/%s", + var.region, module.orch-project.project_id, module.orch-artifact-reg.name) +} + +module "orch-project" { + source = "../../../modules/project" + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}" + iam = var.project_config.billing_account_id != null ? local.iam_orch : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null + oslogin = false services = concat(var.project_services, [ "artifactregistry.googleapis.com", "bigquery.googleapis.com", @@ -132,11 +125,11 @@ module "orch-vpc" { source = "../../../modules/net-vpc" count = local.use_shared_vpc ? 0 : 1 project_id = module.orch-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-orch" subnets = [ { ip_cidr_range = "10.10.0.0/24" - name = "default" + name = "${var.prefix}-orch" region = var.region secondary_ip_ranges = { pods = "10.10.8.0/22" @@ -160,7 +153,7 @@ module "orch-nat" { count = local.use_shared_vpc ? 0 : 1 source = "../../../modules/net-cloudnat" project_id = module.orch-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-orch" region = var.region router_network = module.orch-vpc.0.name } diff --git a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf index 3d3a818c57..394adedf8a 100644 --- a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf +++ b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf @@ -15,6 +15,19 @@ # tfdoc:file:description Trasformation project and VPC. locals { + iam_trf = { + "roles/bigquery.jobUser" = [ + module.transf-sa-bq-0.iam_email, local.groups_iam.data-engineers + ] + "roles/dataflow.admin" = [ + module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers + ] + "roles/dataflow.worker" = [module.transf-sa-df-0.iam_email] + "roles/storage.objectAdmin" = [ + module.transf-sa-df-0.iam_email, + "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}" + ] + } transf_subnet = ( local.use_shared_vpc ? var.network_config.subnet_self_links.orchestration @@ -29,31 +42,13 @@ locals { module "transf-project" { source = "../../../modules/project" - parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "trf${local.project_suffix}" - group_iam = { - (local.groups.data-engineers) = [ - "roles/bigquery.jobUser", - "roles/dataflow.admin", - ] - } - iam = { - "roles/bigquery.jobUser" = [ - module.transf-sa-bq-0.iam_email, - ] - "roles/dataflow.admin" = [ - module.orch-sa-cmp-0.iam_email, - ] - "roles/dataflow.worker" = [ - module.transf-sa-df-0.iam_email - ] - "roles/storage.objectAdmin" = [ - module.transf-sa-df-0.iam_email, - "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}" - ] - } + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}" + iam = var.project_config.billing_account_id != null ? local.iam_trf : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_trf : null services = concat(var.project_services, [ "bigquery.googleapis.com", "bigqueryreservation.googleapis.com", @@ -131,11 +126,11 @@ module "transf-vpc" { source = "../../../modules/net-vpc" count = local.use_shared_vpc ? 0 : 1 project_id = module.transf-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-trf" subnets = [ { ip_cidr_range = "10.10.0.0/24" - name = "default" + name = "${var.prefix}-trf" region = var.region } ] @@ -155,7 +150,7 @@ module "transf-nat" { source = "../../../modules/net-cloudnat" count = local.use_shared_vpc ? 0 : 1 project_id = module.transf-project.project_id - name = "${var.prefix}-default" + name = "${var.prefix}-trf" region = var.region router_network = module.transf-vpc.0.name } diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf index 0db5ce4404..67c43daefe 100644 --- a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf +++ b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf @@ -15,54 +15,48 @@ # tfdoc:file:description Data Warehouse projects. locals { - dwh_group_iam = { - (local.groups.data-engineers) = [ - "roles/bigquery.dataEditor", - "roles/storage.admin", - ], - (local.groups.data-analysts) = [ - "roles/bigquery.dataViewer", - "roles/bigquery.jobUser", - "roles/bigquery.metadataViewer", - "roles/bigquery.user", - "roles/datacatalog.viewer", - "roles/datacatalog.tagTemplateViewer", - "roles/storage.objectViewer", - ] - } dwh_lnd_iam = { "roles/bigquery.dataOwner" = [ module.load-sa-df-0.iam_email, + ] + "roles/bigquery.dataViewer" = [ module.transf-sa-df-0.iam_email, module.transf-sa-bq-0.iam_email, + local.groups_iam.data-engineers ] "roles/bigquery.jobUser" = [ - module.load-sa-df-0.iam_email, - ] - "roles/datacatalog.categoryAdmin" = [ - module.transf-sa-bq-0.iam_email - ] - "roles/storage.objectCreator" = [ - module.load-sa-df-0.iam_email, + module.load-sa-df-0.iam_email, local.groups_iam.data-engineers ] + "roles/datacatalog.categoryAdmin" = [module.transf-sa-bq-0.iam_email] + "roles/datacatalog.tagTemplateViewer" = [local.groups_iam.data-engineers] + "roles/datacatalog.viewer" = [local.groups_iam.data-engineers] + "roles/storage.objectCreator" = [module.load-sa-df-0.iam_email] + "roles/storage.objectViewer" = [local.groups_iam.data-engineers] } dwh_iam = { "roles/bigquery.dataOwner" = [ module.transf-sa-df-0.iam_email, module.transf-sa-bq-0.iam_email, ] + "roles/bigquery.dataViewer" = [ + local.groups_iam.data-analysts, + local.groups_iam.data-engineers + ] "roles/bigquery.jobUser" = [ module.transf-sa-bq-0.iam_email, + local.groups_iam.data-analysts, + local.groups_iam.data-engineers ] - "roles/datacatalog.categoryAdmin" = [ - module.load-sa-df-0.iam_email + "roles/datacatalog.tagTemplateViewer" = [ + local.groups_iam.data-analysts, local.groups_iam.data-engineers ] - "roles/storage.objectCreator" = [ - module.transf-sa-df-0.iam_email, + "roles/datacatalog.viewer" = [ + local.groups_iam.data-analysts, local.groups_iam.data-engineers ] "roles/storage.objectViewer" = [ - module.transf-sa-df-0.iam_email, + local.groups_iam.data-analysts, local.groups_iam.data-engineers ] + "roles/storage.objectAdmin" = [module.transf-sa-df-0.iam_email] } dwh_services = concat(var.project_services, [ "bigquery.googleapis.com", @@ -82,12 +76,13 @@ locals { module "dwh-lnd-project" { source = "../../../modules/project" - parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "dwh-lnd${local.project_suffix}" - group_iam = local.dwh_group_iam - iam = local.dwh_lnd_iam + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}" + iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {} + iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {} services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] @@ -97,12 +92,13 @@ module "dwh-lnd-project" { module "dwh-cur-project" { source = "../../../modules/project" - parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "dwh-cur${local.project_suffix}" - group_iam = local.dwh_group_iam - iam = local.dwh_iam + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}" + iam = var.project_config.billing_account_id != null ? local.dwh_iam : {} + iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {} services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] @@ -112,12 +108,13 @@ module "dwh-cur-project" { module "dwh-conf-project" { source = "../../../modules/project" - parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "dwh-conf${local.project_suffix}" - group_iam = local.dwh_group_iam - iam = local.dwh_iam + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}" + iam = var.project_config.billing_account_id != null ? local.dwh_iam : null + iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null services = local.dwh_services service_encryption_key_ids = { bq = [try(local.service_encryption_keys.bq, null)] @@ -138,7 +135,7 @@ module "dwh-lnd-bq-0" { module "dwh-cur-bq-0" { source = "../../../modules/bigquery-dataset" project_id = module.dwh-cur-project.project_id - id = "${replace(var.prefix, "-", "_")}_dwh_lnd_bq_0" + id = "${replace(var.prefix, "-", "_")}_dwh_cur_bq_0" location = var.location encryption_key = try(local.service_encryption_keys.bq, null) } diff --git a/blueprints/data-solutions/data-platform-foundations/06-common.tf b/blueprints/data-solutions/data-platform-foundations/06-common.tf index 80451500c2..5a84ee777e 100644 --- a/blueprints/data-solutions/data-platform-foundations/06-common.tf +++ b/blueprints/data-solutions/data-platform-foundations/06-common.tf @@ -14,35 +14,22 @@ # tfdoc:file:description common project. -module "common-project" { - source = "../../../modules/project" - parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "cmn${local.project_suffix}" - group_iam = { - (local.groups.data-analysts) = [ - "roles/datacatalog.viewer", - ] - (local.groups.data-engineers) = [ - "roles/dlp.reader", - "roles/dlp.user", - "roles/dlp.estimatesAdmin", - ] - (local.groups.data-security) = [ - "roles/dlp.admin", - "roles/datacatalog.admin" - ] - } - iam = { +locals { + iam_common = { + "roles/dlp.admin" = [local.groups_iam.data-security] + "roles/dlp.estimatesAdmin" = [local.groups_iam.data-engineers] + "roles/dlp.reader" = [local.groups_iam.data-engineers] "roles/dlp.user" = [ module.load-sa-df-0.iam_email, - module.transf-sa-df-0.iam_email + module.transf-sa-df-0.iam_email, + local.groups_iam.data-engineers ] + "roles/datacatalog.admin" = [local.groups_iam.data-security] "roles/datacatalog.viewer" = [ module.load-sa-df-0.iam_email, module.transf-sa-df-0.iam_email, - module.transf-sa-bq-0.iam_email + module.transf-sa-bq-0.iam_email, + local.groups_iam.data-analysts ] "roles/datacatalog.categoryFineGrainedReader" = [ module.transf-sa-df-0.iam_email, @@ -51,6 +38,16 @@ module "common-project" { # local.groups_iam.data-analysts ] } +} +module "common-project" { + source = "../../../modules/project" + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}" + iam = var.project_config.billing_account_id != null ? local.iam_common : null + iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null services = concat(var.project_services, [ "datacatalog.googleapis.com", "dlp.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf index 030be0b832..ea8fca0947 100644 --- a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf +++ b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf @@ -16,8 +16,9 @@ module "exp-project" { source = "../../../modules/project" - parent = var.folder_id - billing_account = var.billing_account_id - prefix = var.prefix - name = "exp${local.project_suffix}" + parent = var.project_config.parent + billing_account = var.project_config.billing_account_id + project_create = var.project_config.billing_account_id != null + prefix = var.project_config.billing_account_id == null ? null : var.prefix + name = var.project_config.billing_account_id == null ? var.project_config.project_ids.exp : "${var.project_config.project_ids.exp}${local.project_suffix}" } diff --git a/blueprints/data-solutions/data-platform-foundations/IAM.md b/blueprints/data-solutions/data-platform-foundations/IAM.md index dd898bd750..b982f8c4ee 100644 --- a/blueprints/data-solutions/data-platform-foundations/IAM.md +++ b/blueprints/data-solutions/data-platform-foundations/IAM.md @@ -17,51 +17,48 @@ Legend: + additive, • conditional. | members | roles | |---|---| -|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/pubsub.editor](https://cloud.google.com/iam/docs/understanding-roles#pubsub.editor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) | |drp-bq-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) | |drp-cs-0
serviceAccount|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | |drp-ps-0
serviceAccount|[roles/pubsub.publisher](https://cloud.google.com/iam/docs/understanding-roles#pubsub.publisher) | -|load-df-0
serviceAccount|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|load-df-0
serviceAccount|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |orc-cmp-0
serviceAccount|[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | ## Project dwh-conf | members | roles | |---|---| -|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | -|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| -|load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | |trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) | -|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | ## Project dwh-cur | members | roles | |---|---| -|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | -|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| -|load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | |trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) | -|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | +|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | ## Project dwh-lnd | members | roles | |---|---| -|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | -|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) | +|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| |load-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) | -|trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | -|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) | +|trf-bq-0
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) | +|trf-df-0
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) | ## Project lod | members | roles | |---|---| -|gcp-data-engineers
group|[roles/compute.viewer](https://cloud.google.com/iam/docs/understanding-roles#compute.viewer)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer)
[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) | +|gcp-data-engineers
group|[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer) | |SERVICE_IDENTITY_dataflow-service-producer-prod
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| |load-df-0
serviceAccount|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | @@ -71,7 +68,7 @@ Legend: + additive, • conditional. | members | roles | |---|---| -|gcp-data-engineers
group|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin)
[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor)
[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin)
[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor)
[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | +|gcp-data-engineers
group|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin)
[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor)
[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin)
[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor)
[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |SERVICE_IDENTITY_cloudcomposer-accounts
serviceAccount|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |SERVICE_IDENTITY_gcp-sa-cloudbuild
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) | |SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +| diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index 08b24b2116..ad08721692 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -213,13 +213,15 @@ While this blueprint can be used as a standalone deployment, it can also be call ```hcl module "data-platform" { source = "./fabric/blueprints/data-solutions/data-platform-foundations" - billing_account_id = var.billing_account_id - folder_id = var.folder_id organization_domain = "example.com" - prefix = "myprefix" + project_config = { + billing_account_id = "123456-123456-123456" + parent = "folders/12345678" + } + prefix = "myprefix" } -# tftest modules=43 resources=297 +# tftest modules=43 resources=278 ``` ## Customizations @@ -233,6 +235,14 @@ To create Cloud Key Management keys in the Data Platform you can uncomment the C To handle multiple groups of `data-analysts` accessing the same Data Warehouse layer projects but only to the dataset belonging to a specific group, you may want to assign roles at BigQuery dataset level instead of at project-level. To do this, you need to remove IAM binging at project-level for the `data-analysts` group and give roles at BigQuery dataset level using the `iam` variable on `bigquery-dataset` modules. +### Project Configuration + +The solution can be deployed by creating projects on a given parent (organization or folder) or on existing projects. Configure variable `project_config` accordingly. + +When you rely on existing projects, the blueprint is designed to rely on different projects configuring IAM binding with an additive approach. For discovery or experimentation purposes, you may also configure `project_config.project_ids` to point different projects to one project with the granularity you need. For example, deploy resources from the 'load' project with resources in the 'transformation' project. + +Once you have identified the required project granularity for your use case, we suggest adapting the terraform script accordingly and relying on authoritative IAM binding. + ## Demo pipeline The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features. @@ -244,20 +254,19 @@ You can find examples in the `[demo](./demo)` folder. | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [billing_account_id](variables.tf#L17) | Billing account id. | string | ✓ | | -| [folder_id](variables.tf#L122) | Folder to be used for the networking resources in folders/nnnn format. | string | ✓ | | -| [organization_domain](variables.tf#L166) | Organization domain. | string | ✓ | | -| [prefix](variables.tf#L171) | Prefix used for resource names. | string | ✓ | | -| [composer_config](variables.tf#L22) | Cloud Composer config. | object({…}) | | {…} | -| [data_catalog_tags](variables.tf#L105) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | -| [data_force_destroy](variables.tf#L116) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | -| [groups](variables.tf#L127) | User groups. | map(string) | | {…} | -| [location](variables.tf#L137) | Location used for multi-regional resources. | string | | "eu" | -| [network_config](variables.tf#L143) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | -| [project_services](variables.tf#L180) | List of core services enabled on all projects. | list(string) | | […] | -| [project_suffix](variables.tf#L191) | Suffix used only for project ids. | string | | null | -| [region](variables.tf#L197) | Region used for regional resources. | string | | "europe-west1" | -| [service_encryption_keys](variables.tf#L203) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | +| [organization_domain](variables.tf#L156) | Organization domain. | string | ✓ | | +| [prefix](variables.tf#L161) | Prefix used for resource names. | string | ✓ | | +| [project_config](variables.tf#L170) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…}) | ✓ | | +| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…}) | | {…} | +| [data_catalog_tags](variables.tf#L100) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {…} | +| [data_force_destroy](variables.tf#L111) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool | | false | +| [groups](variables.tf#L117) | User groups. | map(string) | | {…} | +| [location](variables.tf#L127) | Location used for multi-regional resources. | string | | "eu" | +| [network_config](variables.tf#L133) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…}) | | null | +| [project_services](variables.tf#L204) | List of core services enabled on all projects. | list(string) | | […] | +| [project_suffix](variables.tf#L215) | Suffix used only for project ids. | string | | null | +| [region](variables.tf#L221) | Region used for regional resources. | string | | "europe-west1" | +| [service_encryption_keys](variables.tf#L227) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…}) | | null | ## Outputs diff --git a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py index 4b15eaaba5..86b8e5bbe6 100644 --- a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py +++ b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py @@ -123,7 +123,7 @@ task_id="upsert_table_customers", project_id=DWH_LAND_PRJ, dataset_id=DWH_LAND_BQ_DATASET, - impersonation_chain=[TRF_SA_DF], + impersonation_chain=[LOD_SA_DF], table_resource={ "tableReference": {"tableId": "customers"}, }, @@ -133,7 +133,7 @@ task_id="upsert_table_purchases", project_id=DWH_LAND_PRJ, dataset_id=DWH_LAND_BQ_DATASET, - impersonation_chain=[TRF_SA_BQ], + impersonation_chain=[LOD_SA_DF], table_resource={ "tableReference": {"tableId": "purchases"} }, @@ -167,7 +167,7 @@ project_id=DWH_LAND_PRJ, dataset_id=DWH_LAND_BQ_DATASET, table_id="customers", - impersonation_chain=[TRF_SA_BQ], + impersonation_chain=[LOD_SA_DF], include_policy_tags=True, schema_fields_updates=[ { "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" }, @@ -182,7 +182,7 @@ project_id=DWH_LAND_PRJ, dataset_id=DWH_LAND_BQ_DATASET, table_id="purchases", - impersonation_chain=[TRF_SA_BQ], + impersonation_chain=[LOD_SA_DF], include_policy_tags=True, schema_fields_updates=[ { "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" }, diff --git a/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py b/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py index dc0c954b14..bade038871 100644 --- a/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py +++ b/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py @@ -122,13 +122,13 @@ delete_table_customers = BigQueryDeleteTableOperator( task_id="delete_table_customers", deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".customers", - impersonation_chain=[TRF_SA_DF] + impersonation_chain=[LOD_SA_DF] ) delete_table_purchases = BigQueryDeleteTableOperator( task_id="delete_table_purchases", deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".purchases", - impersonation_chain=[TRF_SA_DF] + impersonation_chain=[LOD_SA_DF] ) delete_table_customer_purchase_curated = BigQueryDeleteTableOperator( diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf index 6c25406a2f..4ec2fd7e1f 100644 --- a/blueprints/data-solutions/data-platform-foundations/variables.tf +++ b/blueprints/data-solutions/data-platform-foundations/variables.tf @@ -14,11 +14,6 @@ # tfdoc:file:description Terraform Variables. -variable "billing_account_id" { - description = "Billing account id." - type = string -} - variable "composer_config" { description = "Cloud Composer config." type = object({ @@ -119,11 +114,6 @@ variable "data_force_destroy" { default = false } -variable "folder_id" { - description = "Folder to be used for the networking resources in folders/nnnn format." - type = string -} - variable "groups" { description = "User groups." type = map(string) @@ -177,6 +167,40 @@ variable "prefix" { } } +variable "project_config" { + description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format." + type = object({ + billing_account_id = optional(string, null) + parent = string + project_ids = optional(object({ + drop = string + load = string + orc = string + trf = string + dwh-lnd = string + dwh-cur = string + dwh-conf = string + common = string + exp = string + }), { + drop = "drp" + load = "lod" + orc = "orc" + trf = "trf" + dwh-lnd = "dwh-lnd" + dwh-cur = "dwh-cur" + dwh-conf = "dwh-conf" + common = "cmn" + exp = "exp" + } + ) + }) + validation { + condition = var.project_config.billing_account_id != null || var.project_config.project_ids != null + error_message = "At least one attribute should be set." + } +} + variable "project_services" { description = "List of core services enabled on all projects." type = list(string) diff --git a/fast/stages/3-data-platform/dev/main.tf b/fast/stages/3-data-platform/dev/main.tf index 53d901d1be..c600a7586f 100644 --- a/fast/stages/3-data-platform/dev/main.tf +++ b/fast/stages/3-data-platform/dev/main.tf @@ -18,13 +18,15 @@ module "data-platform" { source = "../../../../blueprints/data-solutions/data-platform-foundations" - billing_account_id = var.billing_account.id composer_config = var.composer_config data_force_destroy = var.data_force_destroy data_catalog_tags = var.data_catalog_tags - folder_id = var.folder_ids.data-platform-dev - groups = var.groups - location = var.location + project_config = { + billing_account_id = var.billing_account.id + parent = var.folder_ids.data-platform-dev + } + groups = var.groups + location = var.location network_config = { host_project = var.host_project_ids.dev-spoke-0 network_self_link = var.vpc_self_links.dev-spoke-0 diff --git a/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf b/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf index 52317d6f5f..5acb29e835 100644 --- a/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf +++ b/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf @@ -17,7 +17,9 @@ module "test" { source = "../../../../../blueprints/data-solutions/data-platform-foundations/" organization_domain = "example.com" - billing_account_id = "123456-123456-123456" - folder_id = "folders/12345678" - prefix = "prefix" + project_config = { + billing_account_id = "123456-123456-123456" + parent = "folders/12345678" + } + prefix = "prefix" } diff --git a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py index 785f470537..630944f26d 100644 --- a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py +++ b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py @@ -23,4 +23,4 @@ def test_resources(e2e_plan_runner): modules, resources = e2e_plan_runner(FIXTURES_DIR) assert len(modules) == 42 - assert len(resources) == 296 + assert len(resources) == 277