Skip to content

Commit

Permalink
Merge pull request GoogleCloudPlatform#1165 from GoogleCloudPlatform/…
Browse files Browse the repository at this point in the history 
…lcaggio/dp-projectcreate

DataPlatform: Support project creation
  • Loading branch information
@lcaggio
lcaggio authored Feb 23, 2023
2 parents ad08406 + 47855cb commit 89da766
Show file tree
Hide file tree
Showing 15 changed files with 280 additions and 260 deletions.
50 changes: 24 additions & 26 deletions blueprints/data-solutions/data-platform-foundations/01-dropoff.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,36 +15,34 @@
# tfdoc:file:description drop off project and resources.

locals {
drop_orch_service_accounts = [
module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email
]
}

module "drop-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "drp${local.project_suffix}"
group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/pubsub.editor",
"roles/storage.admin",
iam_drp = {
"roles/bigquery.dataEditor" = [
module.drop-sa-bq-0.iam_email, local.groups_iam.data-engineers
]
"roles/bigquery.user" = [
module.load-sa-df-0.iam_email, local.groups_iam.data-engineers
]
"roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
"roles/pubsub.subscriber" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
}
iam = {
"roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email]
"roles/bigquery.user" = [module.load-sa-df-0.iam_email]
"roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
"roles/pubsub.subscriber" = concat(
local.drop_orch_service_accounts, [module.load-sa-df-0.iam_email]
)
"roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email]
"roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email]
"roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
"roles/storage.admin" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = [
module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email
]
}
}

module "drop-project" {
source = "../../../modules/project"
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}"
iam = var.project_config.billing_account_id != null ? local.iam_drp : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
Expand Down
55 changes: 30 additions & 25 deletions blueprints/data-solutions/data-platform-foundations/02-load.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,19 @@
# tfdoc:file:description Load project and VPC.

locals {
iam_load = {
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email,
module.load-sa-df-0.iam_email,
local.groups_iam.data-engineers
]
"roles/dataflow.developer" = [
local.groups_iam.data-engineers
]
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = local.load_service_accounts
}
load_service_accounts = [
"serviceAccount:${module.load-project.service_accounts.robots.dataflow}",
module.load-sa-df-0.iam_email
Expand All @@ -35,26 +48,13 @@ locals {

module "load-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "lod${local.project_suffix}"
group_iam = {
(local.groups.data-engineers) = [
"roles/compute.viewer",
"roles/dataflow.admin",
"roles/dataflow.developer",
"roles/viewer",
]
}
iam = {
"roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
"roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
"roles/storage.objectAdmin" = local.load_service_accounts
}
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}"
iam = var.project_config.billing_account_id != null ? local.iam_load : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
Expand Down Expand Up @@ -86,8 +86,13 @@ module "load-sa-df-0" {
name = "load-df-0"
display_name = "Data platform Dataflow load service account"
iam = {
"roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
"roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email]
"roles/iam.serviceAccountTokenCreator" = [
local.groups_iam.data-engineers,
module.orch-sa-cmp-0.iam_email
],
"roles/iam.serviceAccountUser" = [
module.orch-sa-cmp-0.iam_email
]
}
}

Expand All @@ -107,11 +112,11 @@ module "load-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.load-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-lod"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
name = "default"
name = "${var.prefix}-lod"
region = var.region
}
]
Expand All @@ -131,7 +136,7 @@ module "load-nat" {
source = "../../../modules/net-cloudnat"
count = local.use_shared_vpc ? 0 : 1
project_id = module.load-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-lod"
region = var.region
router_network = module.load-vpc.0.name
}
93 changes: 43 additions & 50 deletions blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,76 +15,69 @@
# tfdoc:file:description Orchestration project and VPC.

locals {
orch_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.orch-vpc.0.subnet_self_links)[0]
)
orch_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.orch-vpc.0.self_link
)

# Note: This formatting is needed for output purposes since the fabric artifact registry
# module doesn't yet expose the docker usage path of a registry folder in the needed format.
orch_docker_path = format("%s-docker.pkg.dev/%s/%s",
var.region, module.orch-project.project_id, module.orch-artifact-reg.name)
}

module "orch-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "orc${local.project_suffix}"
group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/bigquery.jobUser",
"roles/cloudbuild.builds.editor",
"roles/composer.admin",
"roles/composer.environmentAndStorageObjectAdmin",
"roles/iap.httpsResourceAccessor",
"roles/iam.serviceAccountUser",
"roles/storage.objectAdmin",
"roles/storage.admin",
"roles/artifactregistry.admin",
"roles/serviceusage.serviceUsageConsumer",
]
}
iam = {
iam_orch = {
"roles/artifactregistry.admin" = [local.groups_iam.data-engineers]
"roles/artifactregistry.reader" = [module.load-sa-df-0.iam_email]
"roles/bigquery.dataEditor" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email,
local.groups_iam.data-engineers
]
"roles/bigquery.jobUser" = [
module.orch-sa-cmp-0.iam_email,
local.groups_iam.data-engineers
]
"roles/cloudbuild.builds.editor" = [local.groups_iam.data-engineers]
"roles/cloudbuild.serviceAgent" = [module.orch-sa-df-build.iam_email]
"roles/composer.admin" = [local.groups_iam.data-engineers]
"roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers]
"roles/composer.ServiceAgentV2Ext" = [
"serviceAccount:${module.orch-project.service_accounts.robots.composer}"
]
"roles/composer.worker" = [
module.orch-sa-cmp-0.iam_email
]
"roles/iam.serviceAccountUser" = [
module.orch-sa-cmp-0.iam_email
module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers
]
"roles/iap.httpsResourceAccessor" = [local.groups_iam.data-engineers]
"roles/serviceusage.serviceUsageConsumer" = [local.groups_iam.data-engineers]
"roles/storage.objectAdmin" = [
module.orch-sa-cmp-0.iam_email,
module.orch-sa-df-build.iam_email,
"serviceAccount:${module.orch-project.service_accounts.robots.composer}",
"serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}",
]
"roles/artifactregistry.reader" = [
module.load-sa-df-0.iam_email,
]
"roles/cloudbuild.serviceAgent" = [
module.orch-sa-df-build.iam_email,
local.groups_iam.data-engineers
]
"roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
}
oslogin = false
orch_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
: values(module.orch-vpc.0.subnet_self_links)[0]
)
orch_vpc = (
local.use_shared_vpc
? var.network_config.network_self_link
: module.orch-vpc.0.self_link
)

# Note: This formatting is needed for output purposes since the fabric artifact registry
# module doesn't yet expose the docker usage path of a registry folder in the needed format.
orch_docker_path = format("%s-docker.pkg.dev/%s/%s",
var.region, module.orch-project.project_id, module.orch-artifact-reg.name)
}

module "orch-project" {
source = "../../../modules/project"
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}"
iam = var.project_config.billing_account_id != null ? local.iam_orch : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null
oslogin = false
services = concat(var.project_services, [
"artifactregistry.googleapis.com",
"bigquery.googleapis.com",
Expand Down Expand Up @@ -132,11 +125,11 @@ module "orch-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.orch-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-orch"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
name = "default"
name = "${var.prefix}-orch"
region = var.region
secondary_ip_ranges = {
pods = "10.10.8.0/22"
Expand All @@ -160,7 +153,7 @@ module "orch-nat" {
count = local.use_shared_vpc ? 0 : 1
source = "../../../modules/net-cloudnat"
project_id = module.orch-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-orch"
region = var.region
router_network = module.orch-vpc.0.name
}
Expand Down
51 changes: 23 additions & 28 deletions blueprints/data-solutions/data-platform-foundations/04-transformation.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,19 @@
# tfdoc:file:description Trasformation project and VPC.

locals {
iam_trf = {
"roles/bigquery.jobUser" = [
module.transf-sa-bq-0.iam_email, local.groups_iam.data-engineers
]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers
]
"roles/dataflow.worker" = [module.transf-sa-df-0.iam_email]
"roles/storage.objectAdmin" = [
module.transf-sa-df-0.iam_email,
"serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
]
}
transf_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
Expand All @@ -29,31 +42,13 @@ locals {

module "transf-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "trf${local.project_suffix}"
group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.jobUser",
"roles/dataflow.admin",
]
}
iam = {
"roles/bigquery.jobUser" = [
module.transf-sa-bq-0.iam_email,
]
"roles/dataflow.admin" = [
module.orch-sa-cmp-0.iam_email,
]
"roles/dataflow.worker" = [
module.transf-sa-df-0.iam_email
]
"roles/storage.objectAdmin" = [
module.transf-sa-df-0.iam_email,
"serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
]
}
parent = var.project_config.parent
billing_account = var.project_config.billing_account_id
project_create = var.project_config.billing_account_id != null
prefix = var.project_config.billing_account_id == null ? null : var.prefix
name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}"
iam = var.project_config.billing_account_id != null ? local.iam_trf : null
iam_additive = var.project_config.billing_account_id == null ? local.iam_trf : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
Expand Down Expand Up @@ -131,11 +126,11 @@ module "transf-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.transf-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-trf"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
name = "default"
name = "${var.prefix}-trf"
region = var.region
}
]
Expand All @@ -155,7 +150,7 @@ module "transf-nat" {
source = "../../../modules/net-cloudnat"
count = local.use_shared_vpc ? 0 : 1
project_id = module.transf-project.project_id
name = "${var.prefix}-default"
name = "${var.prefix}-trf"
region = var.region
router_network = module.transf-vpc.0.name
}
Loading

0 comments on commit 89da766

Please sign in to comment.