Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update Data Platform #1089

Merged
merged 8 commits into from
Jan 12, 2023
Merged

Update Data Platform #1089

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
488b6d4
Remove plg project, composer 2
lcaggio Jan 10, 2023
c172c7b
Remove plg project and update to composer 2
lcaggio Jan 10, 2023
72abd47
Fix documentation.
lcaggio Jan 11, 2023
a5f4e08
Fix FAST documentation.
lcaggio Jan 11, 2023
07a975f
Fix example test.
lcaggio Jan 11, 2023
50a52ae
Implement PR comments.
lcaggio Jan 12, 2023
1672cfe
Fix README.
lcaggio Jan 12, 2023
431185e
Merge branch 'master' into lcaggio/dp-20230111
lcaggio Jan 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 77 additions & 73 deletions blueprints/data-solutions/data-platform-foundations/03-composer.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,40 @@

# tfdoc:file:description Orchestration Cloud Composer definition.

locals {
env_variables = {
BQ_LOCATION = var.location
DATA_CAT_TAGS = try(jsonencode(module.common-datacatalog.tags), "{}")
DF_KMS_KEY = try(var.service_encryption_keys.dataflow, "")
DRP_PRJ = module.drop-project.project_id
DRP_BQ = module.drop-bq-0.dataset_id
DRP_GCS = module.drop-cs-0.url
DRP_PS = module.drop-ps-0.id
DWH_LAND_PRJ = module.dwh-lnd-project.project_id
DWH_LAND_BQ_DATASET = module.dwh-lnd-bq-0.dataset_id
DWH_LAND_GCS = module.dwh-lnd-cs-0.url
DWH_CURATED_PRJ = module.dwh-cur-project.project_id
DWH_CURATED_BQ_DATASET = module.dwh-cur-bq-0.dataset_id
DWH_CURATED_GCS = module.dwh-cur-cs-0.url
DWH_CONFIDENTIAL_PRJ = module.dwh-conf-project.project_id
DWH_CONFIDENTIAL_BQ_DATASET = module.dwh-conf-bq-0.dataset_id
DWH_CONFIDENTIAL_GCS = module.dwh-conf-cs-0.url
GCP_REGION = var.region
LOD_PRJ = module.load-project.project_id
LOD_GCS_STAGING = module.load-cs-df-0.url
LOD_NET_VPC = local.load_vpc
LOD_NET_SUBNET = local.load_subnet
LOD_SA_DF = module.load-sa-df-0.email
ORC_PRJ = module.orch-project.project_id
ORC_GCS = module.orch-cs-0.url
TRF_PRJ = module.transf-project.project_id
TRF_GCS_STAGING = module.transf-cs-df-0.url
TRF_NET_VPC = local.transf_vpc
TRF_NET_SUBNET = local.transf_subnet
TRF_SA_DF = module.transf-sa-df-0.email
TRF_SA_BQ = module.transf-sa-bq-0.email
}
}
module "orch-sa-cmp-0" {
source = "../../../modules/iam-service-account"
project_id = module.orch-project.project_id
Expand All @@ -27,21 +61,51 @@ module "orch-sa-cmp-0" {
}

resource "google_composer_environment" "orch-cmp-0" {
provider = google-beta
project = module.orch-project.project_id
name = "${var.prefix}-orc-cmp-0"
region = var.region
count = var.composer_config.disable_deployment == true ? 0 : 1
project = module.orch-project.project_id
name = "${var.prefix}-orc-cmp-0"
region = var.region
config {
node_count = var.composer_config.node_count
software_config {
airflow_config_overrides = try(var.composer_config.software_config.airflow_config_overrides, null)
pypi_packages = try(var.composer_config.software_config.pypi_packages, null)
env_variables = merge(try(var.composer_config.software_config.env_variables, null), local.env_variables)
image_version = try(var.composer_config.software_config.image_version, null)
}
dynamic "workloads_config" {
for_each = (try(var.composer_config.workloads_config, null) != null ? { 1 = 1 } : {})

content {
scheduler {
cpu = try(var.composer_config.workloads_config.scheduler.cpu, null)
memory_gb = try(var.composer_config.workloads_config.scheduler.memory_gb, null)
storage_gb = try(var.composer_config.workloads_config.scheduler.storage_gb, null)
count = try(var.composer_config.workloads_config.scheduler.count, null)
}
web_server {
cpu = try(var.composer_config.workloads_config.web_server.cpu, null)
memory_gb = try(var.composer_config.workloads_config.web_server.memory_gb, null)
storage_gb = try(var.composer_config.workloads_config.web_server.storage_gb, null)
}
worker {
cpu = try(var.composer_config.workloads_config.worker.cpu, null)
memory_gb = try(var.composer_config.workloads_config.worker.memory_gb, null)
storage_gb = try(var.composer_config.workloads_config.worker.storage_gb, null)
min_count = try(var.composer_config.workloads_config.worker.min_count, null)
max_count = try(var.composer_config.workloads_config.worker.max_count, null)
}
}
}

environment_size = var.composer_config.environment_size

node_config {
zone = "${var.region}-b"
service_account = module.orch-sa-cmp-0.email
network = local.orch_vpc
subnetwork = local.orch_subnet
tags = ["composer-worker", "http-server", "https-server"]
enable_ip_masq_agent = true
service_account = module.orch-sa-cmp-0.email
enable_ip_masq_agent = "true"
tags = ["composer-worker"]
ip_allocation_policy {
use_ip_aliases = "true"
cluster_secondary_range_name = try(
var.network_config.composer_secondary_ranges.pods, "pods"
)
Expand All @@ -58,80 +122,20 @@ resource "google_composer_environment" "orch-cmp-0" {
master_ipv4_cidr_block = try(
var.network_config.composer_ip_ranges.gke_master, "10.20.11.0/28"
)
web_server_ipv4_cidr_block = try(
var.network_config.composer_ip_ranges.web_server, "10.20.11.16/28"
)
}
software_config {
image_version = var.composer_config.airflow_version
env_variables = merge(
var.composer_config.env_variables, {
BQ_LOCATION = var.location
DATA_CAT_TAGS = try(jsonencode(module.common-datacatalog.tags), "{}")
DF_KMS_KEY = try(var.service_encryption_keys.dataflow, "")
DRP_PRJ = module.drop-project.project_id
DRP_BQ = module.drop-bq-0.dataset_id
DRP_GCS = module.drop-cs-0.url
DRP_PS = module.drop-ps-0.id
DWH_LAND_PRJ = module.dwh-lnd-project.project_id
DWH_LAND_BQ_DATASET = module.dwh-lnd-bq-0.dataset_id
DWH_LAND_GCS = module.dwh-lnd-cs-0.url
DWH_CURATED_PRJ = module.dwh-cur-project.project_id
DWH_CURATED_BQ_DATASET = module.dwh-cur-bq-0.dataset_id
DWH_CURATED_GCS = module.dwh-cur-cs-0.url
DWH_CONFIDENTIAL_PRJ = module.dwh-conf-project.project_id
DWH_CONFIDENTIAL_BQ_DATASET = module.dwh-conf-bq-0.dataset_id
DWH_CONFIDENTIAL_GCS = module.dwh-conf-cs-0.url
DWH_PLG_PRJ = module.dwh-plg-project.project_id
DWH_PLG_BQ_DATASET = module.dwh-plg-bq-0.dataset_id
DWH_PLG_GCS = module.dwh-plg-cs-0.url
GCP_REGION = var.region
LOD_PRJ = module.load-project.project_id
LOD_GCS_STAGING = module.load-cs-df-0.url
LOD_NET_VPC = local.load_vpc
LOD_NET_SUBNET = local.load_subnet
LOD_SA_DF = module.load-sa-df-0.email
ORC_PRJ = module.orch-project.project_id
ORC_GCS = module.orch-cs-0.url
TRF_PRJ = module.transf-project.project_id
TRF_GCS_STAGING = module.transf-cs-df-0.url
TRF_NET_VPC = local.transf_vpc
TRF_NET_SUBNET = local.transf_subnet
TRF_SA_DF = module.transf-sa-df-0.email
TRF_SA_BQ = module.transf-sa-bq-0.email
}
)
}

dynamic "encryption_config" {
for_each = (
try(local.service_encryption_keys.composer != null, false)
try(var.service_encryption_keys[var.region], null) != null
? { 1 = 1 }
: {}
)
content {
kms_key_name = try(local.service_encryption_keys.composer, null)
kms_key_name = try(var.service_encryption_keys[var.region], null)
}
}

# dynamic "web_server_network_access_control" {
# for_each = toset(
# var.network_config.web_server_network_access_control == null
# ? []
# : [var.network_config.web_server_network_access_control]
# )
# content {
# dynamic "allowed_ip_range" {
# for_each = toset(web_server_network_access_control.key)
# content {
# value = allowed_ip_range.key
# }
# }
# }
# }

}
depends_on = [
google_project_iam_member.shared_vpc,
module.orch-project
]
}
8 changes: 3 additions & 5 deletions blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ module "orch-project" {
"roles/bigquery.jobUser" = [
module.orch-sa-cmp-0.iam_email,
]
"roles/composer.ServiceAgentV2Ext" = [
"serviceAccount:${module.orch-project.service_accounts.robots.composer}"
]
"roles/composer.worker" = [
module.orch-sa-cmp-0.iam_email
]
Expand All @@ -67,11 +70,6 @@ module "orch-project" {
"roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
}
oslogin = false
org_policies = {
"constraints/compute.requireOsLogin" = {
enforce = false
}
}
services = concat(var.project_services, [
"artifactregistry.googleapis.com",
"bigquery.googleapis.com",
Expand Down
49 changes: 0 additions & 49 deletions blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,21 +30,6 @@ locals {
"roles/storage.objectViewer",
]
}
dwh_plg_group_iam = {
(local.groups.data-engineers) = [
"roles/bigquery.dataEditor",
"roles/storage.admin",
],
(local.groups.data-analysts) = [
"roles/bigquery.dataEditor",
"roles/bigquery.jobUser",
"roles/bigquery.metadataViewer",
"roles/bigquery.user",
"roles/datacatalog.viewer",
"roles/datacatalog.tagTemplateViewer",
"roles/storage.objectAdmin",
]
}
dwh_lnd_iam = {
"roles/bigquery.dataOwner" = [
module.load-sa-df-0.iam_email,
Expand Down Expand Up @@ -140,21 +125,6 @@ module "dwh-conf-project" {
}
}

module "dwh-plg-project" {
source = "../../../modules/project"
parent = var.folder_id
billing_account = var.billing_account_id
prefix = var.prefix
name = "dwh-plg${local.project_suffix}"
group_iam = local.dwh_plg_group_iam
iam = {}
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
storage = [try(local.service_encryption_keys.storage, null)]
}
}

# Bigquery

module "dwh-lnd-bq-0" {
Expand All @@ -181,14 +151,6 @@ module "dwh-conf-bq-0" {
encryption_key = try(local.service_encryption_keys.bq, null)
}

module "dwh-plg-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.dwh-plg-project.project_id
id = "${replace(var.prefix, "-", "_")}_dwh_plg_bq_0"
location = var.location
encryption_key = try(local.service_encryption_keys.bq, null)
}

# Cloud storage

module "dwh-lnd-cs-0" {
Expand Down Expand Up @@ -223,14 +185,3 @@ module "dwh-conf-cs-0" {
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}

module "dwh-plg-cs-0" {
source = "../../../modules/gcs"
project_id = module.dwh-plg-project.project_id
prefix = var.prefix
name = "dwh-plg-cs-0"
location = var.location
storage_class = "MULTI_REGIONAL"
encryption_key = try(local.service_encryption_keys.storage, null)
force_destroy = var.data_force_destroy
}
10 changes: 1 addition & 9 deletions blueprints/data-solutions/data-platform-foundations/IAM.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,6 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
|<b>trf-bq-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) <br>[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
|<b>trf-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) |

## Project <i>dwh-plg</i>

| members | roles |
|---|---|
|<b>gcp-data-analysts</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer) <br>[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) <br>[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer) <br>[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|

## Project <i>lod</i>

| members | roles |
Expand All @@ -80,7 +72,7 @@ Legend: <code>+</code> additive, <code>•</code> conditional.
| members | roles |
|---|---|
|<b>gcp-data-engineers</b><br><small><i>group</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor) <br>[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin) <br>[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor) <br>[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|<b>SERVICE_IDENTITY_cloudcomposer-accounts</b><br><small><i>serviceAccount</i></small>|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|<b>SERVICE_IDENTITY_cloudcomposer-accounts</b><br><small><i>serviceAccount</i></small>|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|<b>SERVICE_IDENTITY_service-networking</b><br><small><i>serviceAccount</i></small>|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) <code>+</code>|
|<b>load-df-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) <br>[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|<b>orc-cmp-0</b><br><small><i>serviceAccount</i></small>|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) <br>[roles/composer.worker](https://cloud.google.com/iam/docs/understanding-roles#composer.worker) <br>[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser) <br>[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
Expand Down
Loading