diff --git a/blueprints/factories/project-factory/README.md b/blueprints/factories/project-factory/README.md index 3988a20607..2e67158277 100644 --- a/blueprints/factories/project-factory/README.md +++ b/blueprints/factories/project-factory/README.md @@ -29,10 +29,6 @@ In addition to the yaml files describing projects, the project factory accepts t Some examples on where to use each of the three sets are provided below. -## Overriding project id - -By default, the project id is derived from the file path relative to the data root. Path slashes are replaced with dashes to allow for multilevel data folders (e.g. `app01/fe.yaml`, `app02/fe.yaml`, etc.). The project id can be overridden by specifying a custom `name` attribute in the yaml file. Refer to the example below for details. - ## Example ```hcl @@ -61,7 +57,7 @@ module "project-factory" { # location where the yaml files are read from factory_data_path = "data" } -# tftest modules=8 resources=32 files=app01,app02-be,app03-be,app03-fe inventory=example.yaml +# tftest modules=7 resources=26 files=prj-app-1,prj-app-2,prj-app-3 inventory=example.yaml ``` ```yaml @@ -84,12 +80,12 @@ service_accounts: app-1-fe: display_name: "Test app 1 frontend." -# tftest-file id=app01 path=data/app01.yaml +# tftest-file id=prj-app-1 path=data/prj-app-1.yaml ``` ```yaml labels: - app: app02 + app: app-2 team: foo parent: folders/12345678 service_accounts: @@ -101,29 +97,17 @@ services: shared_vpc_service_config: host_project: foo-host -# tftest-file id=app02-be path=data/app02/be.yaml +# tftest-file id=prj-app-2 path=data/prj-app-2.yaml ``` ```yaml -name: app03-be-0 parent: folders/12345678 services: - run.googleapis.com - storage.googleapis.com -# tftest-file id=app03-be path=data/app03/be.yaml +# tftest-file id=prj-app-3 path=data/prj-app-3.yaml ``` - -```yaml -name: app03-fe-0 -parent: folders/12345678 -services: -- run.googleapis.com -- storage.googleapis.com - -# tftest-file id=app03-fe path=data/app03/fe.yaml -``` - ## Variables diff --git a/blueprints/factories/project-factory/factory.tf b/blueprints/factories/project-factory/factory.tf index 3a4295b33f..eabb551ad1 100644 --- a/blueprints/factories/project-factory/factory.tf +++ b/blueprints/factories/project-factory/factory.tf @@ -44,7 +44,6 @@ locals { try(v.metric_scopes, null), var.data_defaults.metric_scopes ) - name = try(v.name, replace(k, "/", "-")) org_policies = try(v.org_policies, {}) parent = coalesce( var.data_overrides.parent, diff --git a/blueprints/factories/project-factory/main.tf b/blueprints/factories/project-factory/main.tf index a23ee5ba76..81f1d31652 100644 --- a/blueprints/factories/project-factory/main.tf +++ b/blueprints/factories/project-factory/main.tf @@ -18,7 +18,7 @@ module "projects" { source = "../../../modules/project" for_each = local.projects billing_account = each.value.billing_account - name = each.value.name + name = each.key parent = try(each.value.parent, null) prefix = each.value.prefix auto_create_network = try(each.value.auto_create_network, false) diff --git a/modules/dataproc/README.md b/modules/dataproc/README.md index 5cd220cbaa..f00101345c 100644 --- a/modules/dataproc/README.md +++ b/modules/dataproc/README.md @@ -149,7 +149,7 @@ module "processing-dp-cluster" { | [name](variables.tf#L235) | Cluster name. | string | ✓ | | | [project_id](variables.tf#L250) | Project ID. | string | ✓ | | | [region](variables.tf#L255) | Dataproc region. | string | ✓ | | -| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | +| [dataproc_config](variables.tf#L17) | Dataproc cluster config. | object({…}) | | {} | | [group_iam](variables.tf#L185) | Authoritative IAM binding for organization groups, in {GROUP_EMAIL => [ROLES]} format. Group emails need to be static. Can be used in combination with the `iam` variable. | map(list(string)) | | {} | | [iam](variables.tf#L192) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_bindings](variables.tf#L199) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | diff --git a/modules/dataproc/main.tf b/modules/dataproc/main.tf index 55bef5c70e..f229d7c7ae 100644 --- a/modules/dataproc/main.tf +++ b/modules/dataproc/main.tf @@ -108,8 +108,8 @@ resource "google_dataproc_cluster" "cluster" { dynamic "accelerators" { for_each = var.dataproc_config.cluster_config.worker_config.accelerators == null ? [] : [""] content { - accelerator_type = var.dataproc_config.cluster_config.accelerators.accelerator_type - accelerator_count = var.dataproc_config.cluster_config.accelerators.accelerator_count + accelerator_type = var.dataproc_config.cluster_config.worker_config.accelerators.accelerator_type + accelerator_count = var.dataproc_config.cluster_config.worker_config.accelerators.accelerator_count } } } @@ -185,10 +185,10 @@ resource "google_dataproc_cluster" "cluster" { for_each = var.dataproc_config.cluster_config.dataproc_metric_config == null ? [] : [""] content { dynamic "metrics" { - for_each = var.dataproc_config.cluster_config.dataproc_metric_config.metrics == null ? [] : [""] + for_each = coalesce(var.dataproc_config.cluster_config.dataproc_metric_config.metrics, []) content { - metric_source = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_source - metric_overrides = var.dataproc_config.cluster_config.dataproc_metric_config.metrics.metric_overrides + metric_source = metrics.value.metric_source + metric_overrides = metrics.value.metric_overrides } } } diff --git a/modules/dataproc/variables.tf b/modules/dataproc/variables.tf index 8b77c5b96b..f4170a94f0 100644 --- a/modules/dataproc/variables.tf +++ b/modules/dataproc/variables.tf @@ -128,7 +128,7 @@ variable "dataproc_config" { dataproc_metric_config = optional(object({ metrics = list(object({ metric_source = string - metric_overrides = optional(string) + metric_overrides = optional(list(string)) })) })) metastore_config = optional(object({ diff --git a/tests/blueprints/factories/project_factory/examples/example.yaml b/tests/blueprints/factories/project_factory/examples/example.yaml index 9e4481b41d..71391a526a 100644 --- a/tests/blueprints/factories/project_factory/examples/example.yaml +++ b/tests/blueprints/factories/project_factory/examples/example.yaml @@ -13,21 +13,21 @@ # limitations under the License. values: - module.project-factory.module.projects["app01"].data.google_storage_project_service_account.gcs_sa[0]: - project: test-pf-app01 + module.project-factory.module.projects["prj-app-1"].data.google_storage_project_service_account.gcs_sa[0]: + project: test-pf-prj-app-1 user_project: null - module.project-factory.module.projects["app01"].google_essential_contacts_contact.contact["admin@example.com"]: + module.project-factory.module.projects["prj-app-1"].google_essential_contacts_contact.contact["admin@example.com"]: email: admin@example.com language_tag: en notification_category_subscriptions: - ALL - parent: projects/test-pf-app01 + parent: projects/test-pf-prj-app-1 timeouts: null - ? module.project-factory.module.projects["app01"].google_kms_crypto_key_iam_member.service_identity_cmek["compute.projects/kms-central-prj/locations/europe-west3/keyRings/my-keyring/cryptoKeys/europe3-gce"] + ? module.project-factory.module.projects["prj-app-1"].google_kms_crypto_key_iam_member.service_identity_cmek["compute.projects/kms-central-prj/locations/europe-west3/keyRings/my-keyring/cryptoKeys/europe3-gce"] : condition: [] crypto_key_id: projects/kms-central-prj/locations/europe-west3/keyRings/my-keyring/cryptoKeys/europe3-gce role: roles/cloudkms.cryptoKeyEncrypterDecrypter - module.project-factory.module.projects["app01"].google_project.project[0]: + module.project-factory.module.projects["prj-app-1"].google_project.project[0]: auto_create_network: false billing_account: 012345-67890A-BCDEF0 effective_labels: @@ -39,104 +39,104 @@ values: app: app-1 environment: test team: foo - name: test-pf-app01 + name: test-pf-prj-app-1 org_id: null - project_id: test-pf-app01 + project_id: test-pf-prj-app-1 skip_delete: false terraform_labels: app: app-1 environment: test team: foo timeouts: null - module.project-factory.module.projects["app01"].google_project_service.project_services["container.googleapis.com"]: + module.project-factory.module.projects["prj-app-1"].google_project_service.project_services["container.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false - project: test-pf-app01 + project: test-pf-prj-app-1 service: container.googleapis.com timeouts: null - module.project-factory.module.projects["app01"].google_project_service.project_services["stackdriver.googleapis.com"]: + module.project-factory.module.projects["prj-app-1"].google_project_service.project_services["stackdriver.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false - project: test-pf-app01 + project: test-pf-prj-app-1 service: stackdriver.googleapis.com timeouts: null - module.project-factory.module.projects["app01"].google_project_service.project_services["storage.googleapis.com"]: + module.project-factory.module.projects["prj-app-1"].google_project_service.project_services["storage.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false - project: test-pf-app01 + project: test-pf-prj-app-1 service: storage.googleapis.com timeouts: null - module.project-factory.module.projects["app02/be"].data.google_storage_project_service_account.gcs_sa[0]: - project: test-pf-app02-be + module.project-factory.module.projects["prj-app-2"].data.google_storage_project_service_account.gcs_sa[0]: + project: test-pf-prj-app-2 user_project: null - module.project-factory.module.projects["app02/be"].google_compute_shared_vpc_service_project.shared_vpc_service[0]: + module.project-factory.module.projects["prj-app-2"].google_compute_shared_vpc_service_project.shared_vpc_service[0]: deletion_policy: null host_project: foo-host - service_project: test-pf-app02-be + service_project: test-pf-prj-app-2 timeouts: null - module.project-factory.module.projects["app02/be"].google_essential_contacts_contact.contact["admin@example.com"]: + module.project-factory.module.projects["prj-app-2"].google_essential_contacts_contact.contact["admin@example.com"]: email: admin@example.com language_tag: en notification_category_subscriptions: - ALL - parent: projects/test-pf-app02-be + parent: projects/test-pf-prj-app-2 timeouts: null - module.project-factory.module.projects["app02/be"].google_project.project[0]: + module.project-factory.module.projects["prj-app-2"].google_project.project[0]: auto_create_network: false billing_account: 012345-67890A-ABCDEF effective_labels: - app: app02 + app: app-2 environment: test team: foo folder_id: '12345678' labels: - app: app02 + app: app-2 environment: test team: foo - name: test-pf-app02-be + name: test-pf-prj-app-2 org_id: null - project_id: test-pf-app02-be + project_id: test-pf-prj-app-2 skip_delete: false terraform_labels: - app: app02 + app: app-2 environment: test team: foo timeouts: null - module.project-factory.module.projects["app02/be"].google_project_service.project_services["compute.googleapis.com"]: + module.project-factory.module.projects["prj-app-2"].google_project_service.project_services["compute.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false - project: test-pf-app02-be + project: test-pf-prj-app-2 service: compute.googleapis.com timeouts: null - module.project-factory.module.projects["app02/be"].google_project_service.project_services["run.googleapis.com"]: + module.project-factory.module.projects["prj-app-2"].google_project_service.project_services["run.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false - project: test-pf-app02-be + project: test-pf-prj-app-2 service: run.googleapis.com timeouts: null - module.project-factory.module.projects["app02/be"].google_project_service.project_services["stackdriver.googleapis.com"]: + module.project-factory.module.projects["prj-app-2"].google_project_service.project_services["stackdriver.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false - project: test-pf-app02-be + project: test-pf-prj-app-2 service: stackdriver.googleapis.com timeouts: null - module.project-factory.module.projects["app02/be"].google_project_service.project_services["storage.googleapis.com"]: + module.project-factory.module.projects["prj-app-2"].google_project_service.project_services["storage.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false - project: test-pf-app02-be + project: test-pf-prj-app-2 service: storage.googleapis.com timeouts: null - module.project-factory.module.projects["app03/be"].data.google_storage_project_service_account.gcs_sa[0]: - project: test-pf-app03-be-0 + module.project-factory.module.projects["prj-app-3"].data.google_storage_project_service_account.gcs_sa[0]: + project: test-pf-prj-app-3 user_project: null - module.project-factory.module.projects["app03/be"].google_essential_contacts_contact.contact["admin@example.com"]: + module.project-factory.module.projects["prj-app-3"].google_essential_contacts_contact.contact["admin@example.com"]: email: admin@example.com language_tag: en notification_category_subscriptions: - ALL - parent: projects/test-pf-app03-be-0 + parent: projects/test-pf-prj-app-3 timeouts: null - module.project-factory.module.projects["app03/be"].google_project.project[0]: + module.project-factory.module.projects["prj-app-3"].google_project.project[0]: auto_create_network: false billing_account: 012345-67890A-ABCDEF effective_labels: @@ -144,114 +144,71 @@ values: folder_id: '12345678' labels: environment: test - name: test-pf-app03-be-0 + name: test-pf-prj-app-3 org_id: null - project_id: test-pf-app03-be-0 + project_id: test-pf-prj-app-3 skip_delete: false terraform_labels: environment: test timeouts: null - module.project-factory.module.projects["app03/be"].google_project_service.project_services["run.googleapis.com"]: + module.project-factory.module.projects["prj-app-3"].google_project_service.project_services["run.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false - project: test-pf-app03-be-0 + project: test-pf-prj-app-3 service: run.googleapis.com timeouts: null - module.project-factory.module.projects["app03/be"].google_project_service.project_services["stackdriver.googleapis.com"]: + module.project-factory.module.projects["prj-app-3"].google_project_service.project_services["stackdriver.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false - project: test-pf-app03-be-0 + project: test-pf-prj-app-3 service: stackdriver.googleapis.com timeouts: null - module.project-factory.module.projects["app03/be"].google_project_service.project_services["storage.googleapis.com"]: + module.project-factory.module.projects["prj-app-3"].google_project_service.project_services["storage.googleapis.com"]: disable_dependent_services: false disable_on_destroy: false - project: test-pf-app03-be-0 + project: test-pf-prj-app-3 service: storage.googleapis.com timeouts: null - module.project-factory.module.projects["app03/fe"].data.google_storage_project_service_account.gcs_sa[0]: - project: test-pf-app03-fe-0 - user_project: null - module.project-factory.module.projects["app03/fe"].google_essential_contacts_contact.contact["admin@example.com"]: - email: admin@example.com - language_tag: en - notification_category_subscriptions: - - ALL - parent: projects/test-pf-app03-fe-0 - timeouts: null - module.project-factory.module.projects["app03/fe"].google_project.project[0]: - auto_create_network: false - billing_account: 012345-67890A-ABCDEF - effective_labels: - environment: test - folder_id: '12345678' - labels: - environment: test - name: test-pf-app03-fe-0 - org_id: null - project_id: test-pf-app03-fe-0 - skip_delete: false - terraform_labels: - environment: test - timeouts: null - module.project-factory.module.projects["app03/fe"].google_project_service.project_services["run.googleapis.com"]: - disable_dependent_services: false - disable_on_destroy: false - project: test-pf-app03-fe-0 - service: run.googleapis.com - timeouts: null - module.project-factory.module.projects["app03/fe"].google_project_service.project_services["stackdriver.googleapis.com"]: - disable_dependent_services: false - disable_on_destroy: false - project: test-pf-app03-fe-0 - service: stackdriver.googleapis.com - timeouts: null - module.project-factory.module.projects["app03/fe"].google_project_service.project_services["storage.googleapis.com"]: - disable_dependent_services: false - disable_on_destroy: false - project: test-pf-app03-fe-0 - service: storage.googleapis.com - timeouts: null - ? module.project-factory.module.service-accounts["app01-app-1-be"].google_project_iam_member.project-roles["test-pf-app01-roles/logging.logWriter"] + ? module.project-factory.module.service-accounts["prj-app-1-app-1-be"].google_project_iam_member.project-roles["test-pf-prj-app-1-roles/logging.logWriter"] : condition: [] - project: test-pf-app01 + project: test-pf-prj-app-1 role: roles/logging.logWriter - ? module.project-factory.module.service-accounts["app01-app-1-be"].google_project_iam_member.project-roles["test-pf-app01-roles/monitoring.metricWriter"] + ? module.project-factory.module.service-accounts["prj-app-1-app-1-be"].google_project_iam_member.project-roles["test-pf-prj-app-1-roles/monitoring.metricWriter"] : condition: [] - project: test-pf-app01 + project: test-pf-prj-app-1 role: roles/monitoring.metricWriter - module.project-factory.module.service-accounts["app01-app-1-be"].google_service_account.service_account[0]: + module.project-factory.module.service-accounts["prj-app-1-app-1-be"].google_service_account.service_account[0]: account_id: app-1-be description: null disabled: false display_name: null - project: test-pf-app01 + project: test-pf-prj-app-1 timeouts: null - module.project-factory.module.service-accounts["app01-app-1-fe"].google_service_account.service_account[0]: + module.project-factory.module.service-accounts["prj-app-1-app-1-fe"].google_service_account.service_account[0]: account_id: app-1-fe description: null disabled: false display_name: Test app 1 frontend. - project: test-pf-app01 + project: test-pf-prj-app-1 timeouts: null - module.project-factory.module.service-accounts["app02/be-app-2-be"].google_service_account.service_account[0]: + module.project-factory.module.service-accounts["prj-app-2-app-2-be"].google_service_account.service_account[0]: account_id: app-2-be description: null disabled: false display_name: null - project: test-pf-app02-be + project: test-pf-prj-app-2 timeouts: null counts: google_compute_shared_vpc_service_project: 1 - google_essential_contacts_contact: 4 + google_essential_contacts_contact: 3 google_kms_crypto_key_iam_member: 1 - google_project: 4 + google_project: 3 google_project_iam_member: 2 - google_project_service: 13 + google_project_service: 10 google_service_account: 3 - google_storage_project_service_account: 4 - modules: 8 - resources: 32 + google_storage_project_service_account: 3 + modules: 7 + resources: 26 outputs: {}