diff --git a/blueprints/README.md b/blueprints/README.md
index 60a84912dc..b19b02d30e 100644
--- a/blueprints/README.md
+++ b/blueprints/README.md
@@ -5,7 +5,7 @@ This section provides **[networking blueprints](./networking/)** that implement
Currently available blueprints:
- **apigee** - [Apigee Hybrid on GKE](./apigee/hybrid-gke/), [Apigee X analytics in BigQuery](./apigee/bigquery-analytics), [Apigee network patterns](./apigee/network-patterns/)
-- **cloud operations** - [Active Directory Federation Services](./cloud-operations/adfs), [Cloud Asset Inventory feeds for resource change tracking and remediation](./cloud-operations/asset-inventory-feed-remediation), [Fine-grained Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Cloud DNS & Shared VPC design](./cloud-operations/dns-shared-vpc), [Delegated Role Grants](./cloud-operations/iam-delegated-role-grants), [Networking Dashboard](./cloud-operations/network-dashboard), [Managing on-prem service account keys by uploading public keys](./cloud-operations/onprem-sa-key-management), [Compute Image builder with Hashicorp Packer](./cloud-operations/packer-image-builder), [Packer example](./cloud-operations/packer-image-builder/packer), [Compute Engine quota monitoring](./cloud-operations/quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Configuring workload identity federation for Terraform Cloud/Enterprise workflow](./cloud-operations/terraform-enterprise-wif), [TCP healthcheck and restart for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [Migrate for Compute Engine (v5) blueprints](./cloud-operations/vm-migration), [Configuring workload identity federation to access Google Cloud resources from apps running on Azure](./cloud-operations/workload-identity-federation)
+- **cloud operations** - [Active Directory Federation Services](./cloud-operations/adfs), [Cloud Asset Inventory feeds for resource change tracking and remediation](./cloud-operations/asset-inventory-feed-remediation), [Fine-grained Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Cloud DNS & Shared VPC design](./cloud-operations/dns-shared-vpc), [Delegated Role Grants](./cloud-operations/iam-delegated-role-grants), [Networking Dashboard](./cloud-operations/network-dashboard), [Managing on-prem service account keys by uploading public keys](./cloud-operations/onprem-sa-key-management), [Compute Image builder with Hashicorp Packer](./cloud-operations/packer-image-builder), [Packer example](./cloud-operations/packer-image-builder/packer), [Compute Engine quota monitoring](./cloud-operations/quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Configuring workload identity federation with Terraform Cloud/Enterprise workflows](./cloud-operations/terraform-cloud-dynamic-credentials), [TCP healthcheck and restart for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [Migrate for Compute Engine (v5) blueprints](./cloud-operations/vm-migration), [Configuring workload identity federation to access Google Cloud resources from apps running on Azure](./cloud-operations/workload-identity-federation)
- **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder)
- **factories** - [The why and the how of Resource Factories](./factories), [Google Cloud Identity Group Factory](./factories/cloud-identity-group-factory), [Google Cloud BQ Factory](./factories/bigquery-factory), [Google Cloud VPC Firewall Factory](./factories/net-vpc-firewall-yaml), [Minimal Project Factory](./factories/project-factory)
- **GKE** - [Binary Authorization Pipeline Blueprint](./gke/binauthz), [Storage API](./gke/binauthz/image), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api), [GKE Multitenant Blueprint](./gke/multitenant-fleet), [Shared VPC with GKE support](./networking/shared-vpc-gke/)
diff --git a/blueprints/cloud-operations/README.md b/blueprints/cloud-operations/README.md
index 863aee5812..99013624e5 100644
--- a/blueprints/cloud-operations/README.md
+++ b/blueprints/cloud-operations/README.md
@@ -64,9 +64,9 @@ This [blueprint](./onprem-sa-key-management) shows how to manage IAM Service Acc
-## Workload identity federation for Terraform Enterprise workflow
+## Workload identity federation with Terraform Cloud workflows
- This [blueprint](./terraform-enterprise-wif) shows how to configure [Wokload Identity Federation](https://cloud.google.com/iam/docs/workload-identity-federation) between [Terraform Cloud/Enterprise](https://developer.hashicorp.com/terraform/enterprise) instance and Google Cloud.
+ This [blueprint](./terraform-cloud-dynamic-credentials) shows how to configure [Wokload Identity Federation](https://cloud.google.com/iam/docs/workload-identity-federation) between [Terraform Cloud/Enterprise](https://developer.hashicorp.com/terraform/enterprise) instance and Google Cloud.
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/README.md b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/README.md
similarity index 50%
rename from blueprints/cloud-operations/terraform-enterprise-wif/README.md
rename to blueprints/cloud-operations/terraform-cloud-dynamic-credentials/README.md
index 4bb282c560..3cec722e0f 100644
--- a/blueprints/cloud-operations/terraform-enterprise-wif/README.md
+++ b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/README.md
@@ -1,10 +1,10 @@
-# Configuring workload identity federation for Terraform Cloud/Enterprise workflow
+# Configuration of workload identity federation for Terraform Cloud/Enterprise workflows
-The most common way to use Terraform Cloud for GCP deployments is to store a GCP Service Account Key as a part of TFE Workflow configuration, as we all know there are security risks due to the fact that keys are long term credentials that could be compromised.
+The most common way to use Terraform Cloud for GCP deployments is to store a GCP Service Account Key as a part of TFC Workflow configuration, as we all know there are security risks due to the fact that keys are long term credentials that could be compromised.
Workload identity federation enables applications running outside of Google Cloud to replace long-lived service account keys with short-lived access tokens. This is achieved by configuring Google Cloud to trust an external identity provider, so applications can use the credentials issued by the external identity provider to impersonate a service account.
-This blueprint shows how to set up [Workload Identity Federation](https://cloud.google.com/iam/docs/workload-identity-federation) between [Terraform Cloud/Enterprise](https://developer.hashicorp.com/terraform/enterprise) instance and Google Cloud. This will be possible by configuring workload identity federation to trust oidc tokens generated for a specific workflow in a Terraform Enterprise organization.
+This blueprint shows how to set up [Workload Identity Federation](https://cloud.google.com/iam/docs/workload-identity-federation) between [Terraform Cloud/Enterprise](https://developer.hashicorp.com/terraform/enterprise) instance and Google Cloud. This will be possible by configuring workload identity federation and [Terraform Cloud Dynamic Provider Credentials](https://www.hashicorp.com/blog/terraform-cloud-adds-dynamic-provider-credentials-vault-official-cloud-providers).
The following diagram illustrates how the VM will get a short-lived access token and use it to access a resource:
@@ -12,8 +12,8 @@ The following diagram illustrates how the VM will get a short-lived access token
## Running the blueprint
-### Create Terraform Enterprise Workflow
-If you don't have an existing Terraform Enterprise organization you can sign up for a [free trial](https://app.terraform.io/public/signup/account) account.
+### Create Terraform Cloud Workflow
+If you don't have an existing Terraform Cloud organization you can sign up for a [free trial](https://app.terraform.io/public/signup/account) account.
Create a new Workspace for a `CLI-driven workflow` (Identity Federation will work for any workflow type, but for simplicity of the blueprint we use CLI driven workflow).
@@ -21,7 +21,7 @@ Note workspace name and id (id starts with `ws-`), we will use them on a later s
Go to the organization settings and note the org name and id (id starts with `org-`).
-### Deploy GCP Workload Identity Pool Provider for Terraform Enterprise
+### Deploy GCP Workload Identity Pool Provider for Terraform Cloud integration
> **_NOTE:_** This is a preparation part and should be executed on behalf of a user with enough permissions.
@@ -32,7 +32,7 @@ Required permissions when new project is created:
- Workload Identity Admin on the project level
- Project IAM Admin on the project level
-Fill out required variables, use TFE Org and Workspace IDs from the previous steps (IDs are not the names).
+Fill out required variables, use TFC Org and Workspace IDs from the previous steps (IDs are not the names).
```bash
cd gcp-workload-identity-provider
@@ -50,34 +50,41 @@ terraform init
terraform apply
```
-As a result a set of outputs will be provided (your values will be different), note the output since we will use it on the next steps.
+You will receive a set of outputs (your values may be different), note them because we will need them in the next steps.
```
-impersonate_service_account_email = "sa-tfe@fe-test-oidc.iam.gserviceaccount.com"
-project_id = "tfe-test-oidc"
-workload_identity_audience = "//iam.googleapis.com/projects/476538149566/locations/global/workloadIdentityPools/tfe-pool/providers/tfe-provider"
-workload_identity_pool_provider_id = "projects/476538149566/locations/global/workloadIdentityPools/tfe-pool/providers/tfe-provider"
+project_id = "tfc-dynamic-creds-gcp"
+tfc_workspace_wariables = {
+ "TFC_GCP_PROJECT_NUMBER" = "200635100209"
+ "TFC_GCP_PROVIDER_AUTH" = "true"
+ "TFC_GCP_RUN_SERVICE_ACCOUNT_EMAIL" = "sa-tfc@tfc-dynamic-creds-gcp.iam.gserviceaccount.com"
+ "TFC_GCP_WORKLOAD_POOL_ID" = "tfc-pool"
+ "TFC_GCP_WORKLOAD_PROVIDER_ID" = "tfc-provider"
+}
```
-### Configure OIDC provider for your TFE Workflow
+### Configure Dynamic Provider Credentials for your TFC Workflow
-To enable OIDC for a TFE workflow it's enough to setup an environment variable `TFC_WORKLOAD_IDENTITY_AUDIENCE`.
+To configure [GCP Dynamic Provider Credentials](https://developer.hashicorp.com/terraform/cloud-docs/workspaces/dynamic-provider-credentials/gcp-configuration) for a TFC workflow, you need to set a set of environment variables:
+- `TFC_GCP_PROVIDER_AUTH`
+- `TFC_GCP_PROJECT_NUMBER`
+- `TFC_GCP_RUN_SERVICE_ACCOUNT_EMAIL`
+- `TFC_GCP_WORKLOAD_POOL_ID`
+- `TFC_GCP_WORKLOAD_PROVIDER_ID`
-Go the the Workflow -> Variables and add a new variable `TFC_WORKLOAD_IDENTITY_AUDIENCE` equal to the value of `workload_identity_audience` output, in our example it's:
+Go to the Workflow -> Variables page and click the + Add variable button. For variable type select ` Environment variable`. The variable names listed above are the names of the variables that you need to set. The values provided in the terraform output in the previous step are the values that you need to provide for each variable.
-```
-TFC_WORKLOAD_IDENTITY_AUDIENCE = "//iam.googleapis.com/projects/476538149566/locations/global/workloadIdentityPools/tfe-pool/providers/tfe-provider"
-```
-
-At that point we setup GCP Identity Federation to trust TFE generated OIDC tokens, so the TFE workflow can use the token to impersonate a GCP Service Account.
+At that point we set up GCP Identity Federation to trust TFC generated OIDC tokens, workflow should be able to use Dynamic Provider Credentials to impersonate a GCP Service Account.
## Testing the blueprint
-In order to test the setup we will deploy a GCS bucket from TFE Workflow using OIDC token for Service Account Impersonation.
+To test the setup, we will deploy a GCS bucket from the TFC Workflow created in the previous step.
+
+This will allow us to verify that the workflow can successfully interact with GCP services using the TFC Dynamic Provider Credentials.
### Configure backend and variables
-First, we need to configure TFE Remote backend for our testing terraform code, use TFE Organization name and workspace name (names are not the same as ids)
+First, we need to configure the TFC Remote backend for our testing Terraform code. Use the TFC Organization name and workspace name (names are not the same as ids).
```
cd ../tfc-workflow-using-wif
@@ -89,7 +96,7 @@ vi backend.tf
```
-Fill out variables based on the output from the preparation steps:
+Fill out `project_id` variable based on the output from the preparation steps:
```
mv terraform.auto.tfvars.template terraform.auto.tfvars
@@ -100,7 +107,7 @@ vi terraform.auto.tfvars
### Authenticate terraform for triggering CLI-driven workflow
-Follow this [documentation](https://learn.hashicorp.com/tutorials/terraform/cloud-login) to login ti terraform cloud from the CLI.
+Follow this [documentation](https://learn.hashicorp.com/tutorials/terraform/cloud-login) to login to terraform cloud from the CLI.
### Trigger the workflow
@@ -110,6 +117,6 @@ terraform init
terraform apply
```
-As a result we have a successfully deployed GCS bucket from Terraform Enterprise workflow using Workload Identity Federation.
+As a result we have a successfully deployed GCS bucket from Terraform Cloud workflow using Workload Identity Federation.
Once done testing, you can clean up resources by running `terraform destroy` first in the `tfc-workflow-using-wif` and then `gcp-workload-identity-provider` folders.
diff --git a/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/diagram.png b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/diagram.png
new file mode 100644
index 0000000000..9216226ad1
Binary files /dev/null and b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/diagram.png differ
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider/README.md b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/README.md
similarity index 54%
rename from blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider/README.md
rename to blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/README.md
index 35198e8d1c..a7dfac5734 100644
--- a/blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider/README.md
+++ b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/README.md
@@ -1,10 +1,14 @@
-# GCP Workload Identity Provider for Terraform Enterprise
+# GCP Workload Identity Provider for Terraform Cloud Dynamic Credentials
-This terraform code is a part of [GCP Workload Identity Federation for Terraform Enterprise](../) blueprint.
+This terraform code is a part of [GCP Workload Identity Federation for Terraform Cloud](../) blueprint.
The codebase provisions the following list of resources:
-- GCS Bucket
+- (optional) GCP Project
+- IAM Service Account
+- Workload Identity Pool
+- Workload Identity Provider
+- IAM Permissins
## Variables
@@ -13,21 +17,19 @@ The codebase provisions the following list of resources:
|---|---|:---:|:---:|:---:|
| [billing_account](variables.tf#L16) | Billing account id used as default for new projects. | string
| ✓ | |
| [project_id](variables.tf#L43) | Existing project id. | string
| ✓ | |
-| [tfe_organization_id](variables.tf#L48) | TFE organization id. | string
| ✓ | |
-| [tfe_workspace_id](variables.tf#L53) | TFE workspace id. | string
| ✓ | |
-| [issuer_uri](variables.tf#L21) | Terraform Enterprise uri. Replace the uri if a self hosted instance is used. | string
| | "https://app.terraform.io/"
|
+| [tfc_organization_id](variables.tf#L48) | TFC organization id. | string
| ✓ | |
+| [tfc_workspace_id](variables.tf#L53) | TFC workspace id. | string
| ✓ | |
+| [issuer_uri](variables.tf#L21) | Terraform Cloud/Enterprise uri. Replace the uri if a self hosted instance is used. | string
| | "https://app.terraform.io/"
|
| [parent](variables.tf#L27) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string
| | null
|
| [project_create](variables.tf#L37) | Create project instead of using an existing one. | bool
| | true
|
-| [workload_identity_pool_id](variables.tf#L58) | Workload identity pool id. | string
| | "tfe-pool"
|
-| [workload_identity_pool_provider_id](variables.tf#L64) | Workload identity pool provider id. | string
| | "tfe-provider"
|
+| [workload_identity_pool_id](variables.tf#L58) | Workload identity pool id. | string
| | "tfc-pool"
|
+| [workload_identity_pool_provider_id](variables.tf#L64) | Workload identity pool provider id. | string
| | "tfc-provider"
|
## Outputs
| name | description | sensitive |
|---|---|:---:|
-| [impersonate_service_account_email](outputs.tf#L16) | Service account to be impersonated by workload identity. | |
-| [project_id](outputs.tf#L21) | GCP Project ID. | |
-| [workload_identity_audience](outputs.tf#L26) | TFC Workload Identity Audience. | |
-| [workload_identity_pool_provider_id](outputs.tf#L31) | GCP workload identity pool provider ID. | |
+| [project_id](outputs.tf#L15) | GCP Project ID. | |
+| [tfc_workspace_wariables](outputs.tf#L20) | Variables to be set on the TFC workspace. | |
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider/main.tf b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/main.tf
similarity index 77%
rename from blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider/main.tf
rename to blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/main.tf
index 5ced2e3c57..e4275350d5 100644
--- a/blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider/main.tf
+++ b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/main.tf
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -36,25 +36,27 @@ module "project" {
# Workload Identity Pool and Provider #
###############################################################################
-resource "google_iam_workload_identity_pool" "tfe-pool" {
+resource "google_iam_workload_identity_pool" "tfc-pool" {
project = module.project.project_id
workload_identity_pool_id = var.workload_identity_pool_id
- display_name = "TFE Pool"
- description = "Identity pool for Terraform Enterprise OIDC integration"
+ display_name = "TFC Pool"
+ description = "Identity pool for Terraform Cloud Dynamic Credentials integration"
}
-resource "google_iam_workload_identity_pool_provider" "tfe-pool-provider" {
+resource "google_iam_workload_identity_pool_provider" "tfc-pool-provider" {
project = module.project.project_id
- workload_identity_pool_id = google_iam_workload_identity_pool.tfe-pool.workload_identity_pool_id
+ workload_identity_pool_id = google_iam_workload_identity_pool.tfc-pool.workload_identity_pool_id
workload_identity_pool_provider_id = var.workload_identity_pool_provider_id
- display_name = "TFE Pool Provider"
- description = "OIDC identity pool provider for TFE Integration"
- # Use condition to make sure only token generated for a specific TFE Org can be used across org workspaces
- attribute_condition = "attribute.terraform_organization_id == \"${var.tfe_organization_id}\""
+ display_name = "TFC Pool Provider"
+ description = "OIDC identity pool provider for Terraform Cloud Dynamic Credentials integration"
+ # Use condition to make sure only token generated for a specific TFC Org can be used across org workspaces
+ attribute_condition = "attribute.terraform_organization_id == \"${var.tfc_organization_id}\""
attribute_mapping = {
"google.subject" = "assertion.sub"
"attribute.aud" = "assertion.aud"
"attribute.terraform_run_phase" = "assertion.terraform_run_phase"
+ "attribute.terraform_project_id" = "assertion.terraform_project_id",
+ "attribute.terraform_project_name" = "assertion.terraform_project_name",
"attribute.terraform_workspace_id" = "assertion.terraform_workspace_id"
"attribute.terraform_workspace_name" = "assertion.terraform_workspace_name"
"attribute.terraform_organization_id" = "assertion.terraform_organization_id"
@@ -72,15 +74,15 @@ resource "google_iam_workload_identity_pool_provider" "tfe-pool-provider" {
# Service Account and IAM bindings #
###############################################################################
-module "sa-tfe" {
+module "sa-tfc" {
source = "../../../../modules/iam-service-account"
project_id = module.project.project_id
- name = "sa-tfe"
+ name = "sa-tfc"
iam = {
- # We allow only tokens generated by a specific TFE workspace impersonation of the service account,
- # that way one identity pool can be used for a TFE Organization, but every workspace will be able to impersonate only a specifc SA
- "roles/iam.workloadIdentityUser" = ["principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.tfe-pool.name}/attribute.terraform_workspace_id/${var.tfe_workspace_id}"]
+ # We allow only tokens generated by a specific TFC workspace impersonation of the service account,
+ # that way one identity pool can be used for a TFC Organization, but every workspace will be able to impersonate only a specifc SA
+ "roles/iam.workloadIdentityUser" = ["principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.tfc-pool.name}/attribute.terraform_workspace_id/${var.tfc_workspace_id}"]
}
iam_project_roles = {
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider/outputs.tf b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/outputs.tf
similarity index 53%
rename from blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider/outputs.tf
rename to blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/outputs.tf
index 46d7f6b0f2..e38a4da668 100644
--- a/blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider/outputs.tf
+++ b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/outputs.tf
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -12,23 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
-output "impersonate_service_account_email" {
- description = "Service account to be impersonated by workload identity."
- value = module.sa-tfe.email
-}
-
output "project_id" {
description = "GCP Project ID."
value = module.project.project_id
}
-output "workload_identity_audience" {
- description = "TFC Workload Identity Audience."
- value = "//iam.googleapis.com/${google_iam_workload_identity_pool_provider.tfe-pool-provider.name}"
-}
-
-output "workload_identity_pool_provider_id" {
- description = "GCP workload identity pool provider ID."
- value = google_iam_workload_identity_pool_provider.tfe-pool-provider.name
+output "tfc_workspace_wariables" {
+ description = "Variables to be set on the TFC workspace."
+ value = {
+ TFC_GCP_PROVIDER_AUTH = "true",
+ TFC_GCP_PROJECT_NUMBER = module.project.number,
+ TFC_GCP_WORKLOAD_POOL_ID = google_iam_workload_identity_pool.tfc-pool.workload_identity_pool_id,
+ TFC_GCP_WORKLOAD_PROVIDER_ID = google_iam_workload_identity_pool_provider.tfc-pool-provider.workload_identity_pool_provider_id,
+ TFC_GCP_RUN_SERVICE_ACCOUNT_EMAIL = module.sa-tfc.email
+ }
}
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider/variables.tf b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/variables.tf
similarity index 83%
rename from blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider/variables.tf
rename to blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/variables.tf
index 3719b1839e..3d4da65892 100644
--- a/blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider/variables.tf
+++ b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -19,7 +19,7 @@ variable "billing_account" {
}
variable "issuer_uri" {
- description = "Terraform Enterprise uri. Replace the uri if a self hosted instance is used."
+ description = "Terraform Cloud/Enterprise uri. Replace the uri if a self hosted instance is used."
type = string
default = "https://app.terraform.io/"
}
@@ -45,24 +45,24 @@ variable "project_id" {
type = string
}
-variable "tfe_organization_id" {
- description = "TFE organization id."
+variable "tfc_organization_id" {
+ description = "TFC organization id."
type = string
}
-variable "tfe_workspace_id" {
- description = "TFE workspace id."
+variable "tfc_workspace_id" {
+ description = "TFC workspace id."
type = string
}
variable "workload_identity_pool_id" {
description = "Workload identity pool id."
type = string
- default = "tfe-pool"
+ default = "tfc-pool"
}
variable "workload_identity_pool_provider_id" {
description = "Workload identity pool provider id."
type = string
- default = "tfe-provider"
+ default = "tfc-provider"
}
diff --git a/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/tfc-workflow-using-wif/README.md b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/tfc-workflow-using-wif/README.md
new file mode 100644
index 0000000000..262472d03a
--- /dev/null
+++ b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/tfc-workflow-using-wif/README.md
@@ -0,0 +1,16 @@
+# Test GCP Workload Identity Provider for Terraform Dynamic Credentials
+
+This terraform code is a part of [GCP Workload Identity Federation for Terraform Cloud](../) blueprint. For instructions please refer to the blueprint [readme](../README.md).
+
+The codebase provisions the following list of resources:
+
+- GCS Bucket
+
+
+## Variables
+
+| name | description | type | required | default |
+|---|---|:---:|:---:|:---:|
+| [project_id](variables.tf#L15) | GCP project ID. | string
| ✓ | |
+
+
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/backend.tf.template b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/tfc-workflow-using-wif/backend.tf.template
similarity index 89%
rename from blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/backend.tf.template
rename to blueprints/cloud-operations/terraform-cloud-dynamic-credentials/tfc-workflow-using-wif/backend.tf.template
index 87d4737dfb..01781fe97e 100644
--- a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/backend.tf.template
+++ b/blueprints/cloud-operations/terraform-cloud-dynamic-credentials/tfc-workflow-using-wif/backend.tf.template
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -18,10 +18,10 @@
terraform {
backend "remote" {
- organization = "string
| ✓ | |
-| [project_id](variables.tf#L21) | GCP project ID. | string
| ✓ | |
-
-
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/provider.tf b/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/provider.tf
deleted file mode 100644
index ae132fd40b..0000000000
--- a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/provider.tf
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-module "tfe_oidc" {
- source = "./tfc-oidc"
-
- impersonate_service_account_email = var.impersonate_service_account_email
-}
-
-provider "google" {
- credentials = module.tfe_oidc.credentials
-}
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/README.md b/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/README.md
deleted file mode 100644
index fd869ae1ae..0000000000
--- a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-# Terraform Enterprise OIDC Credential for GCP Workload Identity Federation
-
-This is a helper module to prepare GCP Credentials from Terraform Enterprise workload identity token. For more information see [Terraform Enterprise Workload Identity Federation](../) blueprint.
-
-## Example
-```hcl
-module "tfe_oidc" {
- source = "./tfc-oidc"
-
- impersonate_service_account_email = "tfe-test@tfe-test-wif.iam.gserviceaccount.com"
-}
-
-provider "google" {
- credentials = module.tfe_oidc.credentials
-}
-
-provider "google-beta" {
- credentials = module.tfe_oidc.credentials
-}
-
-# tftest skip
-```
-
-
-## Variables
-
-| name | description | type | required | default |
-|---|---|:---:|:---:|:---:|
-| [impersonate_service_account_email](variables.tf#L17) | Service account to be impersonated by workload identity federation. | string
| ✓ | |
-| [tmp_oidc_token_path](variables.tf#L22) | Name of the temporary file where TFC OIDC token will be stored to authentificate terraform provider google. | string
| | ".oidc_token"
|
-
-## Outputs
-
-| name | description | sensitive |
-|---|---|:---:|
-| [credentials](outputs.tf#L17) | Credentials in format to pass the to gcp provider. | |
-
-
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/main.tf b/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/main.tf
deleted file mode 100644
index f40b84593e..0000000000
--- a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/main.tf
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Copyright 2022 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-data "external" "oidc_token_file" {
- program = ["bash", "${path.module}/write_token.sh", "${var.tmp_oidc_token_path}"]
-}
-
-data "external" "workload_identity_pool" {
- program = ["bash", "${path.module}/get_audience.sh"]
-}
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/outputs.tf b/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/outputs.tf
deleted file mode 100644
index a642b850ae..0000000000
--- a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/outputs.tf
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Copyright 2022 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-output "credentials" {
- description = "Credentials in format to pass the to gcp provider."
- value = jsonencode({
- "type" : "external_account",
- "audience" : data.external.workload_identity_pool.result.audience,
- "subject_token_type" : "urn:ietf:params:oauth:token-type:jwt",
- "token_url" : "https://sts.googleapis.com/v1/token",
- "credential_source" : data.external.oidc_token_file.result
- "service_account_impersonation_url" : "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/${var.impersonate_service_account_email}:generateAccessToken"
- })
-}
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/variables.tf b/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/variables.tf
deleted file mode 100644
index 056453140e..0000000000
--- a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/variables.tf
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Copyright 2022 Google LLC
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-variable "impersonate_service_account_email" {
- description = "Service account to be impersonated by workload identity federation."
- type = string
-}
-
-variable "tmp_oidc_token_path" {
- description = "Name of the temporary file where TFC OIDC token will be stored to authentificate terraform provider google."
- type = string
- default = ".oidc_token"
-}
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/versions.tf b/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/versions.tf
deleted file mode 100644
index 08492c6f95..0000000000
--- a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/tfc-oidc/versions.tf
+++ /dev/null
@@ -1,29 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-terraform {
- required_version = ">= 1.3.1"
- required_providers {
- google = {
- source = "hashicorp/google"
- version = ">= 4.50.0" # tftest
- }
- google-beta = {
- source = "hashicorp/google-beta"
- version = ">= 4.50.0" # tftest
- }
- }
-}
-
-
diff --git a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/variables.tf b/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/variables.tf
deleted file mode 100644
index bc9ca9f8cd..0000000000
--- a/blueprints/cloud-operations/terraform-enterprise-wif/tfc-workflow-using-wif/variables.tf
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright 2022 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-variable "impersonate_service_account_email" {
- description = "Service account to be impersonated by workload identity."
- type = string
-}
-
-variable "project_id" {
- description = "GCP project ID."
- type = string
-}
diff --git a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf
index 177f940a86..46e9a1309a 100644
--- a/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf
+++ b/blueprints/data-solutions/data-platform-foundations/01-dropoff.tf
@@ -15,36 +15,34 @@
# tfdoc:file:description drop off project and resources.
locals {
- drop_orch_service_accounts = [
- module.load-sa-df-0.iam_email, module.orch-sa-cmp-0.iam_email
- ]
-}
-
-module "drop-project" {
- source = "../../../modules/project"
- parent = var.folder_id
- billing_account = var.billing_account_id
- prefix = var.prefix
- name = "drp${local.project_suffix}"
- group_iam = {
- (local.groups.data-engineers) = [
- "roles/bigquery.dataEditor",
- "roles/pubsub.editor",
- "roles/storage.admin",
+ iam_drp = {
+ "roles/bigquery.dataEditor" = [
+ module.drop-sa-bq-0.iam_email, local.groups_iam.data-engineers
+ ]
+ "roles/bigquery.user" = [
+ module.load-sa-df-0.iam_email, local.groups_iam.data-engineers
+ ]
+ "roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
+ "roles/pubsub.subscriber" = [
+ module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
]
- }
- iam = {
- "roles/bigquery.dataEditor" = [module.drop-sa-bq-0.iam_email]
- "roles/bigquery.user" = [module.load-sa-df-0.iam_email]
- "roles/pubsub.publisher" = [module.drop-sa-ps-0.iam_email]
- "roles/pubsub.subscriber" = concat(
- local.drop_orch_service_accounts, [module.load-sa-df-0.iam_email]
- )
- "roles/storage.objectAdmin" = [module.load-sa-df-0.iam_email]
"roles/storage.objectCreator" = [module.drop-sa-cs-0.iam_email]
"roles/storage.objectViewer" = [module.orch-sa-cmp-0.iam_email]
- "roles/storage.admin" = [module.load-sa-df-0.iam_email]
+ "roles/storage.objectAdmin" = [
+ module.load-sa-df-0.iam_email, module.load-sa-df-0.iam_email
+ ]
}
+}
+
+module "drop-project" {
+ source = "../../../modules/project"
+ parent = var.project_config.parent
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
+ name = var.project_config.billing_account_id == null ? var.project_config.project_ids.drop : "${var.project_config.project_ids.drop}${local.project_suffix}"
+ iam = var.project_config.billing_account_id != null ? local.iam_drp : null
+ iam_additive = var.project_config.billing_account_id == null ? local.iam_drp : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
diff --git a/blueprints/data-solutions/data-platform-foundations/02-load.tf b/blueprints/data-solutions/data-platform-foundations/02-load.tf
index 74cb9f8b0c..9702fce1ec 100644
--- a/blueprints/data-solutions/data-platform-foundations/02-load.tf
+++ b/blueprints/data-solutions/data-platform-foundations/02-load.tf
@@ -15,6 +15,19 @@
# tfdoc:file:description Load project and VPC.
locals {
+ iam_load = {
+ "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
+ "roles/dataflow.admin" = [
+ module.orch-sa-cmp-0.iam_email,
+ module.load-sa-df-0.iam_email,
+ local.groups_iam.data-engineers
+ ]
+ "roles/dataflow.developer" = [
+ local.groups_iam.data-engineers
+ ]
+ "roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
+ "roles/storage.objectAdmin" = local.load_service_accounts
+ }
load_service_accounts = [
"serviceAccount:${module.load-project.service_accounts.robots.dataflow}",
module.load-sa-df-0.iam_email
@@ -35,26 +48,13 @@ locals {
module "load-project" {
source = "../../../modules/project"
- parent = var.folder_id
- billing_account = var.billing_account_id
- prefix = var.prefix
- name = "lod${local.project_suffix}"
- group_iam = {
- (local.groups.data-engineers) = [
- "roles/compute.viewer",
- "roles/dataflow.admin",
- "roles/dataflow.developer",
- "roles/viewer",
- ]
- }
- iam = {
- "roles/bigquery.jobUser" = [module.load-sa-df-0.iam_email]
- "roles/dataflow.admin" = [
- module.orch-sa-cmp-0.iam_email, module.load-sa-df-0.iam_email
- ]
- "roles/dataflow.worker" = [module.load-sa-df-0.iam_email]
- "roles/storage.objectAdmin" = local.load_service_accounts
- }
+ parent = var.project_config.parent
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
+ name = var.project_config.billing_account_id == null ? var.project_config.project_ids.load : "${var.project_config.project_ids.load}${local.project_suffix}"
+ iam = var.project_config.billing_account_id != null ? local.iam_load : null
+ iam_additive = var.project_config.billing_account_id == null ? local.iam_load : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
@@ -86,8 +86,13 @@ module "load-sa-df-0" {
name = "load-df-0"
display_name = "Data platform Dataflow load service account"
iam = {
- "roles/iam.serviceAccountTokenCreator" = [local.groups_iam.data-engineers]
- "roles/iam.serviceAccountUser" = [module.orch-sa-cmp-0.iam_email]
+ "roles/iam.serviceAccountTokenCreator" = [
+ local.groups_iam.data-engineers,
+ module.orch-sa-cmp-0.iam_email
+ ],
+ "roles/iam.serviceAccountUser" = [
+ module.orch-sa-cmp-0.iam_email
+ ]
}
}
@@ -107,11 +112,11 @@ module "load-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.load-project.project_id
- name = "${var.prefix}-default"
+ name = "${var.prefix}-lod"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
- name = "default"
+ name = "${var.prefix}-lod"
region = var.region
}
]
@@ -131,7 +136,7 @@ module "load-nat" {
source = "../../../modules/net-cloudnat"
count = local.use_shared_vpc ? 0 : 1
project_id = module.load-project.project_id
- name = "${var.prefix}-default"
+ name = "${var.prefix}-lod"
region = var.region
router_network = module.load-vpc.0.name
}
diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
index a202afdd05..fc0eda12e6 100644
--- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
+++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf
@@ -15,52 +15,22 @@
# tfdoc:file:description Orchestration project and VPC.
locals {
- orch_subnet = (
- local.use_shared_vpc
- ? var.network_config.subnet_self_links.orchestration
- : values(module.orch-vpc.0.subnet_self_links)[0]
- )
- orch_vpc = (
- local.use_shared_vpc
- ? var.network_config.network_self_link
- : module.orch-vpc.0.self_link
- )
-
- # Note: This formatting is needed for output purposes since the fabric artifact registry
- # module doesn't yet expose the docker usage path of a registry folder in the needed format.
- orch_docker_path = format("%s-docker.pkg.dev/%s/%s",
- var.region, module.orch-project.project_id, module.orch-artifact-reg.name)
-}
-
-module "orch-project" {
- source = "../../../modules/project"
- parent = var.folder_id
- billing_account = var.billing_account_id
- prefix = var.prefix
- name = "orc${local.project_suffix}"
- group_iam = {
- (local.groups.data-engineers) = [
- "roles/bigquery.dataEditor",
- "roles/bigquery.jobUser",
- "roles/cloudbuild.builds.editor",
- "roles/composer.admin",
- "roles/composer.environmentAndStorageObjectAdmin",
- "roles/iap.httpsResourceAccessor",
- "roles/iam.serviceAccountUser",
- "roles/storage.objectAdmin",
- "roles/storage.admin",
- "roles/artifactregistry.admin",
- "roles/serviceusage.serviceUsageConsumer",
- ]
- }
- iam = {
+ iam_orch = {
+ "roles/artifactregistry.admin" = [local.groups_iam.data-engineers]
+ "roles/artifactregistry.reader" = [module.load-sa-df-0.iam_email]
"roles/bigquery.dataEditor" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email,
+ local.groups_iam.data-engineers
]
"roles/bigquery.jobUser" = [
module.orch-sa-cmp-0.iam_email,
+ local.groups_iam.data-engineers
]
+ "roles/cloudbuild.builds.editor" = [local.groups_iam.data-engineers]
+ "roles/cloudbuild.serviceAgent" = [module.orch-sa-df-build.iam_email]
+ "roles/composer.admin" = [local.groups_iam.data-engineers]
+ "roles/composer.environmentAndStorageObjectAdmin" = [local.groups_iam.data-engineers]
"roles/composer.ServiceAgentV2Ext" = [
"serviceAccount:${module.orch-project.service_accounts.robots.composer}"
]
@@ -68,23 +38,46 @@ module "orch-project" {
module.orch-sa-cmp-0.iam_email
]
"roles/iam.serviceAccountUser" = [
- module.orch-sa-cmp-0.iam_email
+ module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers
]
+ "roles/iap.httpsResourceAccessor" = [local.groups_iam.data-engineers]
+ "roles/serviceusage.serviceUsageConsumer" = [local.groups_iam.data-engineers]
"roles/storage.objectAdmin" = [
module.orch-sa-cmp-0.iam_email,
module.orch-sa-df-build.iam_email,
"serviceAccount:${module.orch-project.service_accounts.robots.composer}",
"serviceAccount:${module.orch-project.service_accounts.robots.cloudbuild}",
- ]
- "roles/artifactregistry.reader" = [
- module.load-sa-df-0.iam_email,
- ]
- "roles/cloudbuild.serviceAgent" = [
- module.orch-sa-df-build.iam_email,
+ local.groups_iam.data-engineers
]
"roles/storage.objectViewer" = [module.load-sa-df-0.iam_email]
}
- oslogin = false
+ orch_subnet = (
+ local.use_shared_vpc
+ ? var.network_config.subnet_self_links.orchestration
+ : values(module.orch-vpc.0.subnet_self_links)[0]
+ )
+ orch_vpc = (
+ local.use_shared_vpc
+ ? var.network_config.network_self_link
+ : module.orch-vpc.0.self_link
+ )
+
+ # Note: This formatting is needed for output purposes since the fabric artifact registry
+ # module doesn't yet expose the docker usage path of a registry folder in the needed format.
+ orch_docker_path = format("%s-docker.pkg.dev/%s/%s",
+ var.region, module.orch-project.project_id, module.orch-artifact-reg.name)
+}
+
+module "orch-project" {
+ source = "../../../modules/project"
+ parent = var.project_config.parent
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
+ name = var.project_config.billing_account_id == null ? var.project_config.project_ids.orc : "${var.project_config.project_ids.orc}${local.project_suffix}"
+ iam = var.project_config.billing_account_id != null ? local.iam_orch : null
+ iam_additive = var.project_config.billing_account_id == null ? local.iam_orch : null
+ oslogin = false
services = concat(var.project_services, [
"artifactregistry.googleapis.com",
"bigquery.googleapis.com",
@@ -132,11 +125,11 @@ module "orch-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.orch-project.project_id
- name = "${var.prefix}-default"
+ name = "${var.prefix}-orch"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
- name = "default"
+ name = "${var.prefix}-orch"
region = var.region
secondary_ip_ranges = {
pods = "10.10.8.0/22"
@@ -160,7 +153,7 @@ module "orch-nat" {
count = local.use_shared_vpc ? 0 : 1
source = "../../../modules/net-cloudnat"
project_id = module.orch-project.project_id
- name = "${var.prefix}-default"
+ name = "${var.prefix}-orch"
region = var.region
router_network = module.orch-vpc.0.name
}
diff --git a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf
index 3d3a818c57..394adedf8a 100644
--- a/blueprints/data-solutions/data-platform-foundations/04-transformation.tf
+++ b/blueprints/data-solutions/data-platform-foundations/04-transformation.tf
@@ -15,6 +15,19 @@
# tfdoc:file:description Trasformation project and VPC.
locals {
+ iam_trf = {
+ "roles/bigquery.jobUser" = [
+ module.transf-sa-bq-0.iam_email, local.groups_iam.data-engineers
+ ]
+ "roles/dataflow.admin" = [
+ module.orch-sa-cmp-0.iam_email, local.groups_iam.data-engineers
+ ]
+ "roles/dataflow.worker" = [module.transf-sa-df-0.iam_email]
+ "roles/storage.objectAdmin" = [
+ module.transf-sa-df-0.iam_email,
+ "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
+ ]
+ }
transf_subnet = (
local.use_shared_vpc
? var.network_config.subnet_self_links.orchestration
@@ -29,31 +42,13 @@ locals {
module "transf-project" {
source = "../../../modules/project"
- parent = var.folder_id
- billing_account = var.billing_account_id
- prefix = var.prefix
- name = "trf${local.project_suffix}"
- group_iam = {
- (local.groups.data-engineers) = [
- "roles/bigquery.jobUser",
- "roles/dataflow.admin",
- ]
- }
- iam = {
- "roles/bigquery.jobUser" = [
- module.transf-sa-bq-0.iam_email,
- ]
- "roles/dataflow.admin" = [
- module.orch-sa-cmp-0.iam_email,
- ]
- "roles/dataflow.worker" = [
- module.transf-sa-df-0.iam_email
- ]
- "roles/storage.objectAdmin" = [
- module.transf-sa-df-0.iam_email,
- "serviceAccount:${module.transf-project.service_accounts.robots.dataflow}"
- ]
- }
+ parent = var.project_config.parent
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
+ name = var.project_config.billing_account_id == null ? var.project_config.project_ids.trf : "${var.project_config.project_ids.trf}${local.project_suffix}"
+ iam = var.project_config.billing_account_id != null ? local.iam_trf : null
+ iam_additive = var.project_config.billing_account_id == null ? local.iam_trf : null
services = concat(var.project_services, [
"bigquery.googleapis.com",
"bigqueryreservation.googleapis.com",
@@ -131,11 +126,11 @@ module "transf-vpc" {
source = "../../../modules/net-vpc"
count = local.use_shared_vpc ? 0 : 1
project_id = module.transf-project.project_id
- name = "${var.prefix}-default"
+ name = "${var.prefix}-trf"
subnets = [
{
ip_cidr_range = "10.10.0.0/24"
- name = "default"
+ name = "${var.prefix}-trf"
region = var.region
}
]
@@ -155,7 +150,7 @@ module "transf-nat" {
source = "../../../modules/net-cloudnat"
count = local.use_shared_vpc ? 0 : 1
project_id = module.transf-project.project_id
- name = "${var.prefix}-default"
+ name = "${var.prefix}-trf"
region = var.region
router_network = module.transf-vpc.0.name
}
diff --git a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
index 0db5ce4404..67c43daefe 100644
--- a/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
+++ b/blueprints/data-solutions/data-platform-foundations/05-datawarehouse.tf
@@ -15,54 +15,48 @@
# tfdoc:file:description Data Warehouse projects.
locals {
- dwh_group_iam = {
- (local.groups.data-engineers) = [
- "roles/bigquery.dataEditor",
- "roles/storage.admin",
- ],
- (local.groups.data-analysts) = [
- "roles/bigquery.dataViewer",
- "roles/bigquery.jobUser",
- "roles/bigquery.metadataViewer",
- "roles/bigquery.user",
- "roles/datacatalog.viewer",
- "roles/datacatalog.tagTemplateViewer",
- "roles/storage.objectViewer",
- ]
- }
dwh_lnd_iam = {
"roles/bigquery.dataOwner" = [
module.load-sa-df-0.iam_email,
+ ]
+ "roles/bigquery.dataViewer" = [
module.transf-sa-df-0.iam_email,
module.transf-sa-bq-0.iam_email,
+ local.groups_iam.data-engineers
]
"roles/bigquery.jobUser" = [
- module.load-sa-df-0.iam_email,
- ]
- "roles/datacatalog.categoryAdmin" = [
- module.transf-sa-bq-0.iam_email
- ]
- "roles/storage.objectCreator" = [
- module.load-sa-df-0.iam_email,
+ module.load-sa-df-0.iam_email, local.groups_iam.data-engineers
]
+ "roles/datacatalog.categoryAdmin" = [module.transf-sa-bq-0.iam_email]
+ "roles/datacatalog.tagTemplateViewer" = [local.groups_iam.data-engineers]
+ "roles/datacatalog.viewer" = [local.groups_iam.data-engineers]
+ "roles/storage.objectCreator" = [module.load-sa-df-0.iam_email]
+ "roles/storage.objectViewer" = [local.groups_iam.data-engineers]
}
dwh_iam = {
"roles/bigquery.dataOwner" = [
module.transf-sa-df-0.iam_email,
module.transf-sa-bq-0.iam_email,
]
+ "roles/bigquery.dataViewer" = [
+ local.groups_iam.data-analysts,
+ local.groups_iam.data-engineers
+ ]
"roles/bigquery.jobUser" = [
module.transf-sa-bq-0.iam_email,
+ local.groups_iam.data-analysts,
+ local.groups_iam.data-engineers
]
- "roles/datacatalog.categoryAdmin" = [
- module.load-sa-df-0.iam_email
+ "roles/datacatalog.tagTemplateViewer" = [
+ local.groups_iam.data-analysts, local.groups_iam.data-engineers
]
- "roles/storage.objectCreator" = [
- module.transf-sa-df-0.iam_email,
+ "roles/datacatalog.viewer" = [
+ local.groups_iam.data-analysts, local.groups_iam.data-engineers
]
"roles/storage.objectViewer" = [
- module.transf-sa-df-0.iam_email,
+ local.groups_iam.data-analysts, local.groups_iam.data-engineers
]
+ "roles/storage.objectAdmin" = [module.transf-sa-df-0.iam_email]
}
dwh_services = concat(var.project_services, [
"bigquery.googleapis.com",
@@ -82,12 +76,13 @@ locals {
module "dwh-lnd-project" {
source = "../../../modules/project"
- parent = var.folder_id
- billing_account = var.billing_account_id
- prefix = var.prefix
- name = "dwh-lnd${local.project_suffix}"
- group_iam = local.dwh_group_iam
- iam = local.dwh_lnd_iam
+ parent = var.project_config.parent
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
+ name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-lnd : "${var.project_config.project_ids.dwh-lnd}${local.project_suffix}"
+ iam = var.project_config.billing_account_id != null ? local.dwh_lnd_iam : {}
+ iam_additive = var.project_config.billing_account_id == null ? local.dwh_lnd_iam : {}
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
@@ -97,12 +92,13 @@ module "dwh-lnd-project" {
module "dwh-cur-project" {
source = "../../../modules/project"
- parent = var.folder_id
- billing_account = var.billing_account_id
- prefix = var.prefix
- name = "dwh-cur${local.project_suffix}"
- group_iam = local.dwh_group_iam
- iam = local.dwh_iam
+ parent = var.project_config.parent
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
+ name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-cur : "${var.project_config.project_ids.dwh-cur}${local.project_suffix}"
+ iam = var.project_config.billing_account_id != null ? local.dwh_iam : {}
+ iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : {}
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
@@ -112,12 +108,13 @@ module "dwh-cur-project" {
module "dwh-conf-project" {
source = "../../../modules/project"
- parent = var.folder_id
- billing_account = var.billing_account_id
- prefix = var.prefix
- name = "dwh-conf${local.project_suffix}"
- group_iam = local.dwh_group_iam
- iam = local.dwh_iam
+ parent = var.project_config.parent
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
+ name = var.project_config.billing_account_id == null ? var.project_config.project_ids.dwh-conf : "${var.project_config.project_ids.dwh-conf}${local.project_suffix}"
+ iam = var.project_config.billing_account_id != null ? local.dwh_iam : null
+ iam_additive = var.project_config.billing_account_id == null ? local.dwh_iam : null
services = local.dwh_services
service_encryption_key_ids = {
bq = [try(local.service_encryption_keys.bq, null)]
@@ -138,7 +135,7 @@ module "dwh-lnd-bq-0" {
module "dwh-cur-bq-0" {
source = "../../../modules/bigquery-dataset"
project_id = module.dwh-cur-project.project_id
- id = "${replace(var.prefix, "-", "_")}_dwh_lnd_bq_0"
+ id = "${replace(var.prefix, "-", "_")}_dwh_cur_bq_0"
location = var.location
encryption_key = try(local.service_encryption_keys.bq, null)
}
diff --git a/blueprints/data-solutions/data-platform-foundations/06-common.tf b/blueprints/data-solutions/data-platform-foundations/06-common.tf
index 80451500c2..5a84ee777e 100644
--- a/blueprints/data-solutions/data-platform-foundations/06-common.tf
+++ b/blueprints/data-solutions/data-platform-foundations/06-common.tf
@@ -14,35 +14,22 @@
# tfdoc:file:description common project.
-module "common-project" {
- source = "../../../modules/project"
- parent = var.folder_id
- billing_account = var.billing_account_id
- prefix = var.prefix
- name = "cmn${local.project_suffix}"
- group_iam = {
- (local.groups.data-analysts) = [
- "roles/datacatalog.viewer",
- ]
- (local.groups.data-engineers) = [
- "roles/dlp.reader",
- "roles/dlp.user",
- "roles/dlp.estimatesAdmin",
- ]
- (local.groups.data-security) = [
- "roles/dlp.admin",
- "roles/datacatalog.admin"
- ]
- }
- iam = {
+locals {
+ iam_common = {
+ "roles/dlp.admin" = [local.groups_iam.data-security]
+ "roles/dlp.estimatesAdmin" = [local.groups_iam.data-engineers]
+ "roles/dlp.reader" = [local.groups_iam.data-engineers]
"roles/dlp.user" = [
module.load-sa-df-0.iam_email,
- module.transf-sa-df-0.iam_email
+ module.transf-sa-df-0.iam_email,
+ local.groups_iam.data-engineers
]
+ "roles/datacatalog.admin" = [local.groups_iam.data-security]
"roles/datacatalog.viewer" = [
module.load-sa-df-0.iam_email,
module.transf-sa-df-0.iam_email,
- module.transf-sa-bq-0.iam_email
+ module.transf-sa-bq-0.iam_email,
+ local.groups_iam.data-analysts
]
"roles/datacatalog.categoryFineGrainedReader" = [
module.transf-sa-df-0.iam_email,
@@ -51,6 +38,16 @@ module "common-project" {
# local.groups_iam.data-analysts
]
}
+}
+module "common-project" {
+ source = "../../../modules/project"
+ parent = var.project_config.parent
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
+ name = var.project_config.billing_account_id == null ? var.project_config.project_ids.common : "${var.project_config.project_ids.common}${local.project_suffix}"
+ iam = var.project_config.billing_account_id != null ? local.iam_common : null
+ iam_additive = var.project_config.billing_account_id == null ? local.iam_common : null
services = concat(var.project_services, [
"datacatalog.googleapis.com",
"dlp.googleapis.com",
diff --git a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf
index 030be0b832..ea8fca0947 100644
--- a/blueprints/data-solutions/data-platform-foundations/07-exposure.tf
+++ b/blueprints/data-solutions/data-platform-foundations/07-exposure.tf
@@ -16,8 +16,9 @@
module "exp-project" {
source = "../../../modules/project"
- parent = var.folder_id
- billing_account = var.billing_account_id
- prefix = var.prefix
- name = "exp${local.project_suffix}"
+ parent = var.project_config.parent
+ billing_account = var.project_config.billing_account_id
+ project_create = var.project_config.billing_account_id != null
+ prefix = var.project_config.billing_account_id == null ? null : var.prefix
+ name = var.project_config.billing_account_id == null ? var.project_config.project_ids.exp : "${var.project_config.project_ids.exp}${local.project_suffix}"
}
diff --git a/blueprints/data-solutions/data-platform-foundations/IAM.md b/blueprints/data-solutions/data-platform-foundations/IAM.md
index dd898bd750..b982f8c4ee 100644
--- a/blueprints/data-solutions/data-platform-foundations/IAM.md
+++ b/blueprints/data-solutions/data-platform-foundations/IAM.md
@@ -17,51 +17,48 @@ Legend: +
additive, •
conditional.
| members | roles |
|---|---|
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/pubsub.editor](https://cloud.google.com/iam/docs/understanding-roles#pubsub.editor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
+|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user) |
|drp-bq-0
serviceAccount|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor) |
|drp-cs-0
serviceAccount|[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
|drp-ps-0
serviceAccount|[roles/pubsub.publisher](https://cloud.google.com/iam/docs/understanding-roles#pubsub.publisher) |
-|load-df-0
serviceAccount|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
+|load-df-0
serviceAccount|[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|orc-cmp-0
serviceAccount|[roles/pubsub.subscriber](https://cloud.google.com/iam/docs/understanding-roles#pubsub.subscriber)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
## Project dwh-conf
| members | roles |
|---|---|
-|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
+|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +
|
-|load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
|trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
-|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
## Project dwh-cur
| members | roles |
|---|---|
-|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
+|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +
|
-|load-df-0
serviceAccount|[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
|trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser) |
-|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
+|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
## Project dwh-lnd
| members | roles |
|---|---|
-|gcp-data-analysts
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/bigquery.metadataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.metadataViewer)
[roles/bigquery.user](https://cloud.google.com/iam/docs/understanding-roles#bigquery.user)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
-|gcp-data-engineers
group|[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin) |
+|gcp-data-engineers
group|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/datacatalog.tagTemplateViewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.tagTemplateViewer)
[roles/datacatalog.viewer](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.viewer)
[roles/storage.objectViewer](https://cloud.google.com/iam/docs/understanding-roles#storage.objectViewer) |
|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +
|
|load-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/storage.objectCreator](https://cloud.google.com/iam/docs/understanding-roles#storage.objectCreator) |
-|trf-bq-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner)
[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
-|trf-df-0
serviceAccount|[roles/bigquery.dataOwner](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataOwner) |
+|trf-bq-0
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer)
[roles/datacatalog.categoryAdmin](https://cloud.google.com/iam/docs/understanding-roles#datacatalog.categoryAdmin) |
+|trf-df-0
serviceAccount|[roles/bigquery.dataViewer](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataViewer) |
## Project lod
| members | roles |
|---|---|
-|gcp-data-engineers
group|[roles/compute.viewer](https://cloud.google.com/iam/docs/understanding-roles#compute.viewer)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer)
[roles/viewer](https://cloud.google.com/iam/docs/understanding-roles#viewer) |
+|gcp-data-engineers
group|[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.developer](https://cloud.google.com/iam/docs/understanding-roles#dataflow.developer) |
|SERVICE_IDENTITY_dataflow-service-producer-prod
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +
|
|load-df-0
serviceAccount|[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/dataflow.admin](https://cloud.google.com/iam/docs/understanding-roles#dataflow.admin)
[roles/dataflow.worker](https://cloud.google.com/iam/docs/understanding-roles#dataflow.worker)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
@@ -71,7 +68,7 @@ Legend: +
additive, •
conditional.
| members | roles |
|---|---|
-|gcp-data-engineers
group|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin)
[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor)
[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin)
[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor)
[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer)
[roles/storage.admin](https://cloud.google.com/iam/docs/understanding-roles#storage.admin)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
+|gcp-data-engineers
group|[roles/artifactregistry.admin](https://cloud.google.com/iam/docs/understanding-roles#artifactregistry.admin)
[roles/bigquery.dataEditor](https://cloud.google.com/iam/docs/understanding-roles#bigquery.dataEditor)
[roles/bigquery.jobUser](https://cloud.google.com/iam/docs/understanding-roles#bigquery.jobUser)
[roles/cloudbuild.builds.editor](https://cloud.google.com/iam/docs/understanding-roles#cloudbuild.builds.editor)
[roles/composer.admin](https://cloud.google.com/iam/docs/understanding-roles#composer.admin)
[roles/composer.environmentAndStorageObjectAdmin](https://cloud.google.com/iam/docs/understanding-roles#composer.environmentAndStorageObjectAdmin)
[roles/iam.serviceAccountUser](https://cloud.google.com/iam/docs/understanding-roles#iam.serviceAccountUser)
[roles/iap.httpsResourceAccessor](https://cloud.google.com/iam/docs/understanding-roles#iap.httpsResourceAccessor)
[roles/serviceusage.serviceUsageConsumer](https://cloud.google.com/iam/docs/understanding-roles#serviceusage.serviceUsageConsumer)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|SERVICE_IDENTITY_cloudcomposer-accounts
serviceAccount|[roles/composer.ServiceAgentV2Ext](https://cloud.google.com/iam/docs/understanding-roles#composer.ServiceAgentV2Ext)
[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|SERVICE_IDENTITY_gcp-sa-cloudbuild
serviceAccount|[roles/storage.objectAdmin](https://cloud.google.com/iam/docs/understanding-roles#storage.objectAdmin) |
|SERVICE_IDENTITY_service-networking
serviceAccount|[roles/servicenetworking.serviceAgent](https://cloud.google.com/iam/docs/understanding-roles#servicenetworking.serviceAgent) +
|
diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md
index 08b24b2116..ad08721692 100644
--- a/blueprints/data-solutions/data-platform-foundations/README.md
+++ b/blueprints/data-solutions/data-platform-foundations/README.md
@@ -213,13 +213,15 @@ While this blueprint can be used as a standalone deployment, it can also be call
```hcl
module "data-platform" {
source = "./fabric/blueprints/data-solutions/data-platform-foundations"
- billing_account_id = var.billing_account_id
- folder_id = var.folder_id
organization_domain = "example.com"
- prefix = "myprefix"
+ project_config = {
+ billing_account_id = "123456-123456-123456"
+ parent = "folders/12345678"
+ }
+ prefix = "myprefix"
}
-# tftest modules=43 resources=297
+# tftest modules=43 resources=278
```
## Customizations
@@ -233,6 +235,14 @@ To create Cloud Key Management keys in the Data Platform you can uncomment the C
To handle multiple groups of `data-analysts` accessing the same Data Warehouse layer projects but only to the dataset belonging to a specific group, you may want to assign roles at BigQuery dataset level instead of at project-level.
To do this, you need to remove IAM binging at project-level for the `data-analysts` group and give roles at BigQuery dataset level using the `iam` variable on `bigquery-dataset` modules.
+### Project Configuration
+
+The solution can be deployed by creating projects on a given parent (organization or folder) or on existing projects. Configure variable `project_config` accordingly.
+
+When you rely on existing projects, the blueprint is designed to rely on different projects configuring IAM binding with an additive approach. For discovery or experimentation purposes, you may also configure `project_config.project_ids` to point different projects to one project with the granularity you need. For example, deploy resources from the 'load' project with resources in the 'transformation' project.
+
+Once you have identified the required project granularity for your use case, we suggest adapting the terraform script accordingly and relying on authoritative IAM binding.
+
## Demo pipeline
The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features.
@@ -244,20 +254,19 @@ You can find examples in the `[demo](./demo)` folder.
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [billing_account_id](variables.tf#L17) | Billing account id. | string
| ✓ | |
-| [folder_id](variables.tf#L122) | Folder to be used for the networking resources in folders/nnnn format. | string
| ✓ | |
-| [organization_domain](variables.tf#L166) | Organization domain. | string
| ✓ | |
-| [prefix](variables.tf#L171) | Prefix used for resource names. | string
| ✓ | |
-| [composer_config](variables.tf#L22) | Cloud Composer config. | object({…})
| | {…}
|
-| [data_catalog_tags](variables.tf#L105) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string)))
| | {…}
|
-| [data_force_destroy](variables.tf#L116) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool
| | false
|
-| [groups](variables.tf#L127) | User groups. | map(string)
| | {…}
|
-| [location](variables.tf#L137) | Location used for multi-regional resources. | string
| | "eu"
|
-| [network_config](variables.tf#L143) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…})
| | null
|
-| [project_services](variables.tf#L180) | List of core services enabled on all projects. | list(string)
| | […]
|
-| [project_suffix](variables.tf#L191) | Suffix used only for project ids. | string
| | null
|
-| [region](variables.tf#L197) | Region used for regional resources. | string
| | "europe-west1"
|
-| [service_encryption_keys](variables.tf#L203) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…})
| | null
|
+| [organization_domain](variables.tf#L156) | Organization domain. | string
| ✓ | |
+| [prefix](variables.tf#L161) | Prefix used for resource names. | string
| ✓ | |
+| [project_config](variables.tf#L170) | Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format. | object({…})
| ✓ | |
+| [composer_config](variables.tf#L17) | Cloud Composer config. | object({…})
| | {…}
|
+| [data_catalog_tags](variables.tf#L100) | List of Data Catalog Policy tags to be created with optional IAM binging configuration in {tag => {ROLE => [MEMBERS]}} format. | map(map(list(string)))
| | {…}
|
+| [data_force_destroy](variables.tf#L111) | Flag to set 'force_destroy' on data services like BiguQery or Cloud Storage. | bool
| | false
|
+| [groups](variables.tf#L117) | User groups. | map(string)
| | {…}
|
+| [location](variables.tf#L127) | Location used for multi-regional resources. | string
| | "eu"
|
+| [network_config](variables.tf#L133) | Shared VPC network configurations to use. If null networks will be created in projects with preconfigured values. | object({…})
| | null
|
+| [project_services](variables.tf#L204) | List of core services enabled on all projects. | list(string)
| | […]
|
+| [project_suffix](variables.tf#L215) | Suffix used only for project ids. | string
| | null
|
+| [region](variables.tf#L221) | Region used for regional resources. | string
| | "europe-west1"
|
+| [service_encryption_keys](variables.tf#L227) | Cloud KMS to use to encrypt different services. Key location should match service region. | object({…})
| | null
|
## Outputs
diff --git a/blueprints/data-solutions/data-platform-foundations/demo/dataflow-csv2bq/README.md b/blueprints/data-solutions/data-platform-foundations/demo/dataflow-csv2bq/README.md
index 44f178fa26..b052fab058 100644
--- a/blueprints/data-solutions/data-platform-foundations/demo/dataflow-csv2bq/README.md
+++ b/blueprints/data-solutions/data-platform-foundations/demo/dataflow-csv2bq/README.md
@@ -3,7 +3,66 @@ This demo serves as a simple example of building and launching a Flex Template D
![Dataflow pipeline overview](../../images/df_demo_pipeline.png "Dataflow pipeline overview")
-## Example build run
+
+## Local development run
+
+For local development, the pipeline can be launched from the local machine for testing purposes using different runners depending on the scope of the test.
+
+### Using the Beam DirectRunner
+The below example uses the Beam DirectRunner. The use case for this runner is mainly for quick local tests on the development environment with low volume of data.
+
+```
+CSV_FILE=gs://[TEST-BUCKET]/customers.csv
+JSON_SCHEMA=gs://[TEST-BUCKET]/customers_schema.json
+OUTPUT_TABLE=[TEST-PROJ].[TEST-DATASET].customers
+PIPELINE_STAGIN_PATH="gs://[TEST-STAGING-BUCKET]"
+
+python src/csv2bq.py \
+--runner="DirectRunner" \
+--csv_file=$CSV_FILE \
+--json_schema=$JSON_SCHEMA \
+--output_table=$OUTPUT_TABLE \
+--temp_location=$PIPELINE_STAGIN_PATH/tmp
+```
+
+*Note:* All paths mentioned can be local paths or on GCS. For cloud resources referenced (GCS and BigQuery), make sure that the user launching the command is authenticated to GCP via `gcloud auth application-default login` and has the required access privileges to those resources.
+
+### Using the DataflowRunner with a local CLI launch
+
+The below example triggers the pipeline on Dataflow from your local development environment. The use case for this is for running local tests on larger volumes of test data and verifying that the pipeline runs well on Dataflow, before compiling it into a template.
+
+```
+PROJECT_ID=[TEST-PROJECT]
+REGION=[REGION]
+SUBNET=[SUBNET-NAME]
+DEV_SERVICE_ACCOUNT=[DEV-SA]
+
+PIPELINE_STAGIN_PATH="gs://[TEST-STAGING-BUCKET]"
+CSV_FILE=gs://[TEST-BUCKET]/customers.csv
+JSON_SCHEMA=gs://[TEST-BUCKET]/customers_schema.json
+OUTPUT_TABLE=[TEST-PROJ].[TEST-DATASET].customers
+
+python src/csv2bq.py \
+--runner="Dataflow" \
+--project=$PROJECT_ID \
+--region=$REGION \
+--csv_file=$CSV_FILE \
+--json_schema=$JSON_SCHEMA \
+--output_table=$OUTPUT_TABLE \
+--temp_location=$PIPELINE_STAGIN_PATH/tmp
+--staging_location=$PIPELINE_STAGIN_PATH/stage \
+--subnetwork="regions/$REGION/subnetworks/$SUBNET" \
+--impersonate_service_account=$DEV_SERVICE_ACCOUNT \
+--no_use_public_ips
+```
+
+In terms of resource access privilege, you can choose to impersonate another service account, which could be defined for development resource access. The authenticated user launching this pipeline will need to have the role `roles/iam.serviceAccountTokenCreator`. If you choose to launch the pipeline without service account impersonation, it will use the default compute service account assigned of the target project.
+
+## Dataflow Flex Template run
+
+For production, and as outline in the Data Platform demo, we build and launch the pipeline as a Flex Template, making it available for other cloud services(such as Apache Airflow) and users to trigger launch instances of it on demand.
+
+### Build launch
Below is an example for triggering the Dataflow flex template build pipeline defined in `cloudbuild.yaml`. The Terraform output provides an example as well filled with the parameters values based on the generated resources in the data platform.
@@ -28,9 +87,9 @@ gcloud builds submit \
**Note:** For the scope of the demo, the launch of this build is manual, but in production, this build would be launched via a configured cloud build trigger when new changes are merged into the code branch of the Dataflow template.
-## Example Dataflow pipeline launch in bash (from flex template)
+### Dataflow Flex Template run
-Below is an example of launching a dataflow pipeline manually, based on the built template. When launched manually, the Dataflow pipeline would be launched via the orchestration service account, which is what the Airflow DAG is also using in the scope of this demo.
+After the build step succeeds. You can launch dataflow pipeline from CLI (outline in this example) or the API via Airflow's operator. For the use case of the data platform, the Dataflow pipeline would be launched via the orchestration service account, which is what the Airflow DAG is also using in the scope of this demo.
**Note:** In the data platform demo, the launch of this Dataflow pipeline is handled by the airflow operator (DataflowStartFlexTemplateOperator).
diff --git a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py
index 4b15eaaba5..86b8e5bbe6 100644
--- a/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py
+++ b/blueprints/data-solutions/data-platform-foundations/demo/datapipeline_dc_tags.py
@@ -123,7 +123,7 @@
task_id="upsert_table_customers",
project_id=DWH_LAND_PRJ,
dataset_id=DWH_LAND_BQ_DATASET,
- impersonation_chain=[TRF_SA_DF],
+ impersonation_chain=[LOD_SA_DF],
table_resource={
"tableReference": {"tableId": "customers"},
},
@@ -133,7 +133,7 @@
task_id="upsert_table_purchases",
project_id=DWH_LAND_PRJ,
dataset_id=DWH_LAND_BQ_DATASET,
- impersonation_chain=[TRF_SA_BQ],
+ impersonation_chain=[LOD_SA_DF],
table_resource={
"tableReference": {"tableId": "purchases"}
},
@@ -167,7 +167,7 @@
project_id=DWH_LAND_PRJ,
dataset_id=DWH_LAND_BQ_DATASET,
table_id="customers",
- impersonation_chain=[TRF_SA_BQ],
+ impersonation_chain=[LOD_SA_DF],
include_policy_tags=True,
schema_fields_updates=[
{ "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" },
@@ -182,7 +182,7 @@
project_id=DWH_LAND_PRJ,
dataset_id=DWH_LAND_BQ_DATASET,
table_id="purchases",
- impersonation_chain=[TRF_SA_BQ],
+ impersonation_chain=[LOD_SA_DF],
include_policy_tags=True,
schema_fields_updates=[
{ "mode": "REQUIRED", "name": "id", "type": "INTEGER", "description": "ID" },
diff --git a/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py b/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py
index dc0c954b14..bade038871 100644
--- a/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py
+++ b/blueprints/data-solutions/data-platform-foundations/demo/delete_table.py
@@ -122,13 +122,13 @@
delete_table_customers = BigQueryDeleteTableOperator(
task_id="delete_table_customers",
deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".customers",
- impersonation_chain=[TRF_SA_DF]
+ impersonation_chain=[LOD_SA_DF]
)
delete_table_purchases = BigQueryDeleteTableOperator(
task_id="delete_table_purchases",
deletion_dataset_table=DWH_LAND_PRJ+"."+DWH_LAND_BQ_DATASET+".purchases",
- impersonation_chain=[TRF_SA_DF]
+ impersonation_chain=[LOD_SA_DF]
)
delete_table_customer_purchase_curated = BigQueryDeleteTableOperator(
diff --git a/blueprints/data-solutions/data-platform-foundations/variables.tf b/blueprints/data-solutions/data-platform-foundations/variables.tf
index 6c25406a2f..4ec2fd7e1f 100644
--- a/blueprints/data-solutions/data-platform-foundations/variables.tf
+++ b/blueprints/data-solutions/data-platform-foundations/variables.tf
@@ -14,11 +14,6 @@
# tfdoc:file:description Terraform Variables.
-variable "billing_account_id" {
- description = "Billing account id."
- type = string
-}
-
variable "composer_config" {
description = "Cloud Composer config."
type = object({
@@ -119,11 +114,6 @@ variable "data_force_destroy" {
default = false
}
-variable "folder_id" {
- description = "Folder to be used for the networking resources in folders/nnnn format."
- type = string
-}
-
variable "groups" {
description = "User groups."
type = map(string)
@@ -177,6 +167,40 @@ variable "prefix" {
}
}
+variable "project_config" {
+ description = "Provide 'billing_account_id' value if project creation is needed, uses existing 'project_ids' if null. Parent is in 'folders/nnn' or 'organizations/nnn' format."
+ type = object({
+ billing_account_id = optional(string, null)
+ parent = string
+ project_ids = optional(object({
+ drop = string
+ load = string
+ orc = string
+ trf = string
+ dwh-lnd = string
+ dwh-cur = string
+ dwh-conf = string
+ common = string
+ exp = string
+ }), {
+ drop = "drp"
+ load = "lod"
+ orc = "orc"
+ trf = "trf"
+ dwh-lnd = "dwh-lnd"
+ dwh-cur = "dwh-cur"
+ dwh-conf = "dwh-conf"
+ common = "cmn"
+ exp = "exp"
+ }
+ )
+ })
+ validation {
+ condition = var.project_config.billing_account_id != null || var.project_config.project_ids != null
+ error_message = "At least one attribute should be set."
+ }
+}
+
variable "project_services" {
description = "List of core services enabled on all projects."
type = list(string)
diff --git a/blueprints/data-solutions/data-playground/main.tf b/blueprints/data-solutions/data-playground/main.tf
index b87e8e7301..a3cfd54eb6 100644
--- a/blueprints/data-solutions/data-playground/main.tf
+++ b/blueprints/data-solutions/data-playground/main.tf
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -83,8 +83,8 @@ module "project" {
}
org_policies = {
- # "constraints/compute.requireOsLogin" = {
- # enforce = false
+ # "compute.requireOsLogin" = {
+ # rules = [{ enforce = false }]
# }
# Example of applying a project wide policy, mainly useful for Composer 1
}
@@ -217,6 +217,11 @@ resource "google_notebooks_instance" "playground" {
service_account = module.service-account-notebook.email
+ # Remove once terraform-provider-google/issues/9164 is fixed
+ lifecycle {
+ ignore_changes = [disk_encryption, kms_key]
+ }
+
#TODO Uncomment once terraform-provider-google/issues/9273 is fixed
# tags = ["ssh"]
depends_on = [
diff --git a/blueprints/data-solutions/shielded-folder/data/org-policies/compute.yaml b/blueprints/data-solutions/shielded-folder/data/org-policies/compute.yaml
index 0d27ac426d..a3f96b1b1c 100644
--- a/blueprints/data-solutions/shielded-folder/data/org-policies/compute.yaml
+++ b/blueprints/data-solutions/shielded-folder/data/org-policies/compute.yaml
@@ -3,71 +3,90 @@
# sample subset of useful organization policies, edit to suit requirements
compute.disableGuestAttributesAccess:
- enforce: true
+ rules:
+ - enforce: true
compute.requireOsLogin:
- enforce: true
+ rules:
+ - enforce: true
compute.restrictLoadBalancerCreationForTypes:
- allow:
- values:
- - in:INTERNAL
+ rules:
+ - allow:
+ values:
+ - in:INTERNAL
compute.skipDefaultNetworkCreation:
- enforce: true
+ rules:
+ - enforce: true
compute.vmExternalIpAccess:
- deny:
- all: true
+ rules:
+ - deny:
+ all: true
# compute.disableInternetNetworkEndpointGroup:
-# enforce: true
+# rules:
+# - enforce: true
# compute.disableNestedVirtualization:
-# enforce: true
+# rules:
+# - enforce: true
# compute.disableSerialPortAccess:
-# enforce: true
+# rules:
+# - enforce: true
# compute.restrictCloudNATUsage:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictDedicatedInterconnectUsage:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictPartnerInterconnectUsage:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictProtocolForwardingCreationForTypes:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictSharedVpcHostProjects:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictSharedVpcSubnetworks:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictVpcPeering:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictVpnPeerIPs:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictXpnProjectLienRemoval:
-# enforce: true
+# rules:
+# - enforce: true
# compute.setNewProjectDefaultToZonalDNSOnly:
-# enforce: true
+# rules:
+# - enforce: true
# compute.vmCanIpForward:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
diff --git a/blueprints/data-solutions/shielded-folder/data/org-policies/iam.yaml b/blueprints/data-solutions/shielded-folder/data/org-policies/iam.yaml
index 4d83f827fe..58e0032cb3 100644
--- a/blueprints/data-solutions/shielded-folder/data/org-policies/iam.yaml
+++ b/blueprints/data-solutions/shielded-folder/data/org-policies/iam.yaml
@@ -3,10 +3,13 @@
# sample subset of useful organization policies, edit to suit requirements
iam.automaticIamGrantsForDefaultServiceAccounts:
- enforce: true
+ rules:
+ - enforce: true
iam.disableServiceAccountKeyCreation:
- enforce: true
+ rules:
+ - enforce: true
iam.disableServiceAccountKeyUpload:
- enforce: true
+ rules:
+ - enforce: true
diff --git a/blueprints/data-solutions/shielded-folder/data/org-policies/serverless.yaml b/blueprints/data-solutions/shielded-folder/data/org-policies/serverless.yaml
index de62e6c702..3efb23cdee 100644
--- a/blueprints/data-solutions/shielded-folder/data/org-policies/serverless.yaml
+++ b/blueprints/data-solutions/shielded-folder/data/org-policies/serverless.yaml
@@ -3,24 +3,29 @@
# sample subset of useful organization policies, edit to suit requirements
run.allowedIngress:
- allow:
- values:
- - is:internal
+ rules:
+ - allow:
+ values:
+ - is:internal
# run.allowedVPCEgress:
-# allow:
-# values:
+# rules:
+# - allow:
+# values:
# - is:private-ranges-only
# cloudfunctions.allowedIngressSettings:
-# allow:
-# values:
-# - is:ALLOW_INTERNAL_ONLY
+# rules:
+# - allow:
+# values:
+# - is:ALLOW_INTERNAL_ONLY
# cloudfunctions.allowedVpcConnectorEgressSettings:
-# allow:
-# values:
-# - is:PRIVATE_RANGES_ONLY
+# rules:
+# - allow:
+# values:
+# - is:PRIVATE_RANGES_ONLY
# cloudfunctions.requireVPCConnector:
-# enforce: true
+# rules:
+# - enforce: true
diff --git a/blueprints/data-solutions/shielded-folder/data/org-policies/sql.yaml b/blueprints/data-solutions/shielded-folder/data/org-policies/sql.yaml
index 88b84d9d50..0eee80453a 100644
--- a/blueprints/data-solutions/shielded-folder/data/org-policies/sql.yaml
+++ b/blueprints/data-solutions/shielded-folder/data/org-policies/sql.yaml
@@ -3,7 +3,9 @@
# sample subset of useful organization policies, edit to suit requirements
sql.restrictAuthorizedNetworks:
- enforce: true
+ rules:
+ - enforce: true
sql.restrictPublicIp:
- enforce: true
+ rules:
+ - enforce: true
diff --git a/blueprints/data-solutions/shielded-folder/data/org-policies/storage.yaml b/blueprints/data-solutions/shielded-folder/data/org-policies/storage.yaml
index 6c0a673f3a..448357b8bc 100644
--- a/blueprints/data-solutions/shielded-folder/data/org-policies/storage.yaml
+++ b/blueprints/data-solutions/shielded-folder/data/org-policies/storage.yaml
@@ -3,4 +3,5 @@
# sample subset of useful organization policies, edit to suit requirements
storage.uniformBucketLevelAccess:
- enforce: true
+ rules:
+ - enforce: true
diff --git a/blueprints/data-solutions/vertex-mlops/main.tf b/blueprints/data-solutions/vertex-mlops/main.tf
index 5f7fbc0c97..27129298af 100644
--- a/blueprints/data-solutions/vertex-mlops/main.tf
+++ b/blueprints/data-solutions/vertex-mlops/main.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -112,7 +112,7 @@ module "gcs-bucket" {
encryption_key = try(local.service_encryption_keys.storage, null)
}
-# Default bucket for Cloud Build to prevent error: "'us' violates constraint ‘constraints/gcp.resourceLocations’"
+# Default bucket for Cloud Build to prevent error: "'us' violates constraint ‘gcp.resourceLocations’"
# https://stackoverflow.com/questions/53206667/cloud-build-fails-with-resource-location-constraint
module "gcs-bucket-cloudbuild" {
source = "../../../modules/gcs"
@@ -230,8 +230,8 @@ module "project" {
org_policies = {
# Example of applying a project wide policy
- # "constraints/compute.requireOsLogin" = {
- # enforce = false
+ # "compute.requireOsLogin" = {
+ # rules = [{ enforce = false }]
# }
}
diff --git a/blueprints/factories/project-factory/README.md b/blueprints/factories/project-factory/README.md
index 2b8c3874e6..68e2e1d062 100644
--- a/blueprints/factories/project-factory/README.md
+++ b/blueprints/factories/project-factory/README.md
@@ -156,15 +156,18 @@ labels:
# [opt] Org policy overrides defined at project level
org_policies:
- constraints/compute.disableGuestAttributesAccess:
- enforce: true
- constraints/compute.trustedImageProjects:
- allow:
- values:
+ compute.disableGuestAttributesAccess:
+ rules:
+ - enforce: true
+ compute.trustedImageProjects:
+ rules:
+ - allow:
+ values:
- projects/fast-dev-iac-core-0
- constraints/compute.vmExternalIpAccess:
- deny:
- all: true
+ compute.vmExternalIpAccess:
+ rules:
+ - deny:
+ all: true
# [opt] Service account to create for the project and their roles on the project
# in name => [roles] format
@@ -223,8 +226,8 @@ vpc:
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
| [billing_account_id](variables.tf#L17) | Billing account id. | string
| ✓ | |
-| [prefix](variables.tf#L157) | Prefix used for resource names. | string
| ✓ | |
-| [project_id](variables.tf#L166) | Project id. | string
| ✓ | |
+| [prefix](variables.tf#L144) | Prefix used for resource names. | string
| ✓ | |
+| [project_id](variables.tf#L153) | Project id. | string
| ✓ | |
| [billing_alert](variables.tf#L22) | Billing alert configuration. | object({…})
| | null
|
| [defaults](variables.tf#L35) | Project factory default values. | object({…})
| | null
|
| [descriptive_name](variables.tf#L57) | Name of the project name. Used for project name instead of `name` variable. | string
| | null
|
@@ -237,15 +240,15 @@ vpc:
| [iam_additive](variables.tf#L99) | Custom additive IAM settings in role => [principal] format. | map(list(string))
| | {}
|
| [kms_service_agents](variables.tf#L105) | KMS IAM configuration in as service => [key]. | map(list(string))
| | {}
|
| [labels](variables.tf#L111) | Labels to be assigned at project level. | map(string)
| | {}
|
-| [org_policies](variables.tf#L117) | Org-policy overrides at project level. | map(object({…}))
| | {}
|
-| [service_accounts](variables.tf#L171) | Service accounts to be created, and roles assigned them on the project. | map(list(string))
| | {}
|
-| [service_accounts_additive](variables.tf#L177) | Service accounts to be created, and roles assigned them on the project additively. | map(list(string))
| | {}
|
-| [service_accounts_iam](variables.tf#L183) | IAM bindings on service account resources. Format is KEY => {ROLE => [MEMBERS]}. | map(map(list(string)))
| | {}
|
-| [service_accounts_iam_additive](variables.tf#L190) | IAM additive bindings on service account resources. Format is KEY => {ROLE => [MEMBERS]}. | map(map(list(string)))
| | {}
|
-| [service_identities_iam](variables.tf#L197) | Custom IAM settings for service identities in service => [role] format. | map(list(string))
| | {}
|
-| [service_identities_iam_additive](variables.tf#L204) | Custom additive IAM settings for service identities in service => [role] format. | map(list(string))
| | {}
|
-| [services](variables.tf#L211) | Services to be enabled for the project. | list(string)
| | []
|
-| [vpc](variables.tf#L218) | VPC configuration for the project. | object({…})
| | null
|
+| [org_policies](variables.tf#L117) | Org-policy overrides at project level. | map(object({…}))
| | {}
|
+| [service_accounts](variables.tf#L158) | Service accounts to be created, and roles assigned them on the project. | map(list(string))
| | {}
|
+| [service_accounts_additive](variables.tf#L164) | Service accounts to be created, and roles assigned them on the project additively. | map(list(string))
| | {}
|
+| [service_accounts_iam](variables.tf#L170) | IAM bindings on service account resources. Format is KEY => {ROLE => [MEMBERS]}. | map(map(list(string)))
| | {}
|
+| [service_accounts_iam_additive](variables.tf#L177) | IAM additive bindings on service account resources. Format is KEY => {ROLE => [MEMBERS]}. | map(map(list(string)))
| | {}
|
+| [service_identities_iam](variables.tf#L184) | Custom IAM settings for service identities in service => [role] format. | map(list(string))
| | {}
|
+| [service_identities_iam_additive](variables.tf#L191) | Custom additive IAM settings for service identities in service => [role] format. | map(list(string))
| | {}
|
+| [services](variables.tf#L198) | Services to be enabled for the project. | list(string)
| | []
|
+| [vpc](variables.tf#L205) | VPC configuration for the project. | object({…})
| | null
|
## Outputs
diff --git a/blueprints/factories/project-factory/sample-data/projects/project.yaml b/blueprints/factories/project-factory/sample-data/projects/project.yaml
index 0344991380..cd7b18374b 100644
--- a/blueprints/factories/project-factory/sample-data/projects/project.yaml
+++ b/blueprints/factories/project-factory/sample-data/projects/project.yaml
@@ -48,15 +48,18 @@ labels:
# [opt] Org policy overrides defined at project level
org_policies:
- constraints/compute.disableGuestAttributesAccess:
- enforce: true
- constraints/compute.trustedImageProjects:
- allow:
- values:
+ compute.disableGuestAttributesAccess:
+ rules:
+ - enforce: true
+ compute.trustedImageProjects:
+ rules:
+ - allow:
+ values:
- projects/fast-dev-iac-core-0
- constraints/compute.vmExternalIpAccess:
- deny:
- all: true
+ compute.vmExternalIpAccess:
+ rules:
+ - deny:
+ all: true
# [opt] Prefix - overrides default if set
prefix: test1
diff --git a/blueprints/factories/project-factory/variables.tf b/blueprints/factories/project-factory/variables.tf
index 3aa3fa36b4..a2089bcfec 100644
--- a/blueprints/factories/project-factory/variables.tf
+++ b/blueprints/factories/project-factory/variables.tf
@@ -119,19 +119,6 @@ variable "org_policies" {
type = map(object({
inherit_from_parent = optional(bool) # for list policies only.
reset = optional(bool)
-
- # default (unconditional) values
- allow = optional(object({
- all = optional(bool)
- values = optional(list(string))
- }))
- deny = optional(object({
- all = optional(bool)
- values = optional(list(string))
- }))
- enforce = optional(bool, true) # for boolean policies only.
-
- # conditional values
rules = optional(list(object({
allow = optional(object({
all = optional(bool)
@@ -141,13 +128,13 @@ variable "org_policies" {
all = optional(bool)
values = optional(list(string))
}))
- enforce = optional(bool, true) # for boolean policies only.
- condition = object({
+ enforce = optional(bool) # for boolean policies only.
+ condition = optional(object({
description = optional(string)
expression = optional(string)
location = optional(string)
title = optional(string)
- })
+ }), {})
})), [])
}))
default = {}
diff --git a/blueprints/gke/multitenant-fleet/main.tf b/blueprints/gke/multitenant-fleet/main.tf
index 588d6c5b62..4079db99af 100644
--- a/blueprints/gke/multitenant-fleet/main.tf
+++ b/blueprints/gke/multitenant-fleet/main.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -64,10 +64,10 @@ module "gke-project-0" {
}
# specify project-level org policies here if you need them
# policy_boolean = {
- # "constraints/compute.disableGuestAttributesAccess" = true
+ # "compute.disableGuestAttributesAccess" = true
# }
# policy_list = {
- # "constraints/compute.trustedImageProjects" = {
+ # "compute.trustedImageProjects" = {
# inherit_from_parent = null
# suggested_value = null
# status = true
diff --git a/blueprints/networking/filtering-proxy/main.tf b/blueprints/networking/filtering-proxy/main.tf
index 06efa81475..b36f0140b5 100644
--- a/blueprints/networking/filtering-proxy/main.tf
+++ b/blueprints/networking/filtering-proxy/main.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -227,8 +227,8 @@ module "folder-apps" {
name = "apps"
org_policies = {
# prevent VMs with public IPs in the apps folder
- "constraints/compute.vmExternalIpAccess" = {
- deny = { all = true }
+ "compute.vmExternalIpAccess" = {
+ rules = [{ deny = { all = true } }]
}
}
}
diff --git a/fast/extras/0-cicd-github/README.md b/fast/extras/0-cicd-github/README.md
index 58407b5e4e..0bd0b5be1d 100644
--- a/fast/extras/0-cicd-github/README.md
+++ b/fast/extras/0-cicd-github/README.md
@@ -39,6 +39,16 @@ modules_config = {
# tftest skip
```
+If the modules are located in a non modules only repository, use the module_prefix attribute to set the location of your modules within the repository:
+
+```hcl
+modules_config = {
+ repository_name = "GoogleCloudPlatform/cloud-foundation-fabric"
+ module_prefix = "modules/"
+}
+# tftest skip
+```
+
In the above example, no key options are set so it's assumed modules will be fetched from a public repository. If modules repository authentication is needed the `key_config` attribute also needs to be set.
If no keypair path is specified an internally generated key will be stored as an access key in the modules repository, and as secrets in the stage repositories:
@@ -125,10 +135,10 @@ Finally, a `commit_config` variable is optional: it can be used to configure aut
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [organization](variables.tf#L50) | GitHub organization. | string
| ✓ | |
+| [organization](variables.tf#L51) | GitHub organization. | string
| ✓ | |
| [commmit_config](variables.tf#L17) | Configure commit metadata. | object({…})
| | {}
|
-| [modules_config](variables.tf#L28) | Configure access to repository module via key, and replacement for modules sources in stage repositories. | object({…})
| | null
|
-| [repositories](variables.tf#L55) | Repositories to create. | map(object({…}))
| | {}
|
+| [modules_config](variables.tf#L28) | Configure access to repository module via key, and replacement for modules sources in stage repositories. | object({…})
| | null
|
+| [repositories](variables.tf#L56) | Repositories to create. | map(object({…}))
| | {}
|
## Outputs
diff --git a/fast/extras/0-cicd-github/main.tf b/fast/extras/0-cicd-github/main.tf
index d91ab970c5..3c42b5cf6d 100644
--- a/fast/extras/0-cicd-github/main.tf
+++ b/fast/extras/0-cicd-github/main.tf
@@ -18,6 +18,7 @@ locals {
_repository_files = flatten([
for k, v in var.repositories : [
for f in concat(
+ [for f in fileset(path.module, "${v.populate_from}/*.svg") : f],
[for f in fileset(path.module, "${v.populate_from}/*.md") : f],
[for f in fileset(path.module, "${v.populate_from}/*.tf") : f]
) : {
@@ -32,7 +33,8 @@ locals {
? ""
: "?ref=${var.modules_config.source_ref}"
)
- modules_repo = try(var.modules_config.repository_name, null)
+ modules_repo = try(var.modules_config.repository_name, null)
+ module_prefix = try(var.modules_config.module_prefix, null)
repositories = {
for k, v in var.repositories :
k => v.create_options == null ? k : github_repository.default[k].name
@@ -143,8 +145,8 @@ resource "github_repository_file" "default" {
endswith(each.value.name, ".tf") && local.modules_repo != null
? replace(
file(each.value.file),
- "/source\\s*=\\s*\"../../../modules/([^/\"]+)\"/",
- "source = \"git@github.com:${local.modules_repo}.git//$1${local.modules_ref}\"" # "
+ "/source(\\s*)=\\s*\"../../../modules/([^/\"]+)\"/",
+ "source$1= \"git@github.com:${local.modules_repo}.git//${local.module_prefix}$2${local.modules_ref}\"" # "
)
: file(each.value.file)
)
diff --git a/fast/extras/0-cicd-github/variables.tf b/fast/extras/0-cicd-github/variables.tf
index 8e5d0832ff..ea378ee789 100644
--- a/fast/extras/0-cicd-github/variables.tf
+++ b/fast/extras/0-cicd-github/variables.tf
@@ -30,6 +30,7 @@ variable "modules_config" {
type = object({
repository_name = string
source_ref = optional(string)
+ module_prefix = optional(string, "")
key_config = optional(object({
create_key = optional(bool, false)
create_secrets = optional(bool, false)
diff --git a/fast/stage-links.sh b/fast/stage-links.sh
index 79d1973fa1..52c9e5ae6d 100755
--- a/fast/stage-links.sh
+++ b/fast/stage-links.sh
@@ -78,6 +78,11 @@ case $STAGE_NAME in
TFVARS="tfvars/0-bootstrap.auto.tfvars.json
tfvars/1-resman.auto.tfvars.json"
;;
+"2-security"*)
+ PROVIDER="providers/2-security-providers.tf"
+ TFVARS="tfvars/0-bootstrap.auto.tfvars.json
+ tfvars/1-resman.auto.tfvars.json"
+ ;;
*)
# check for a "dev" stage 3
echo "no stage found, trying for parent stage 3..."
diff --git a/fast/stages-multitenant/1-resman-tenant/branch-sandbox.tf b/fast/stages-multitenant/1-resman-tenant/branch-sandbox.tf
index 6f3d526c85..39ab03ed7f 100644
--- a/fast/stages-multitenant/1-resman-tenant/branch-sandbox.tf
+++ b/fast/stages-multitenant/1-resman-tenant/branch-sandbox.tf
@@ -28,8 +28,8 @@ module "branch-sandbox-folder" {
"roles/resourcemanager.projectCreator" = [local.automation_sas_iam.sandbox]
}
org_policies = {
- "constraints/sql.restrictPublicIp" = { enforce = false }
- "constraints/compute.vmExternalIpAccess" = { allow = { all = true } }
+ "sql.restrictPublicIp" = { rules = [{ enforce = false }] }
+ "compute.vmExternalIpAccess" = { rules = [{ allow = { all = true } }] }
}
tag_bindings = {
context = var.tags.values["${var.tags.names.context}/sandbox"]
diff --git a/fast/stages-multitenant/1-resman-tenant/data/org-policies/compute.yaml b/fast/stages-multitenant/1-resman-tenant/data/org-policies/compute.yaml
index 0d27ac426d..a3f96b1b1c 100644
--- a/fast/stages-multitenant/1-resman-tenant/data/org-policies/compute.yaml
+++ b/fast/stages-multitenant/1-resman-tenant/data/org-policies/compute.yaml
@@ -3,71 +3,90 @@
# sample subset of useful organization policies, edit to suit requirements
compute.disableGuestAttributesAccess:
- enforce: true
+ rules:
+ - enforce: true
compute.requireOsLogin:
- enforce: true
+ rules:
+ - enforce: true
compute.restrictLoadBalancerCreationForTypes:
- allow:
- values:
- - in:INTERNAL
+ rules:
+ - allow:
+ values:
+ - in:INTERNAL
compute.skipDefaultNetworkCreation:
- enforce: true
+ rules:
+ - enforce: true
compute.vmExternalIpAccess:
- deny:
- all: true
+ rules:
+ - deny:
+ all: true
# compute.disableInternetNetworkEndpointGroup:
-# enforce: true
+# rules:
+# - enforce: true
# compute.disableNestedVirtualization:
-# enforce: true
+# rules:
+# - enforce: true
# compute.disableSerialPortAccess:
-# enforce: true
+# rules:
+# - enforce: true
# compute.restrictCloudNATUsage:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictDedicatedInterconnectUsage:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictPartnerInterconnectUsage:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictProtocolForwardingCreationForTypes:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictSharedVpcHostProjects:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictSharedVpcSubnetworks:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictVpcPeering:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictVpnPeerIPs:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
# compute.restrictXpnProjectLienRemoval:
-# enforce: true
+# rules:
+# - enforce: true
# compute.setNewProjectDefaultToZonalDNSOnly:
-# enforce: true
+# rules:
+# - enforce: true
# compute.vmCanIpForward:
-# deny:
-# all: true
+# rules:
+# - deny:
+# all: true
diff --git a/fast/stages-multitenant/1-resman-tenant/data/org-policies/iam.yaml b/fast/stages-multitenant/1-resman-tenant/data/org-policies/iam.yaml
index 4d83f827fe..58e0032cb3 100644
--- a/fast/stages-multitenant/1-resman-tenant/data/org-policies/iam.yaml
+++ b/fast/stages-multitenant/1-resman-tenant/data/org-policies/iam.yaml
@@ -3,10 +3,13 @@
# sample subset of useful organization policies, edit to suit requirements
iam.automaticIamGrantsForDefaultServiceAccounts:
- enforce: true
+ rules:
+ - enforce: true
iam.disableServiceAccountKeyCreation:
- enforce: true
+ rules:
+ - enforce: true
iam.disableServiceAccountKeyUpload:
- enforce: true
+ rules:
+ - enforce: true
diff --git a/fast/stages-multitenant/1-resman-tenant/data/org-policies/serverless.yaml b/fast/stages-multitenant/1-resman-tenant/data/org-policies/serverless.yaml
index de62e6c702..3efb23cdee 100644
--- a/fast/stages-multitenant/1-resman-tenant/data/org-policies/serverless.yaml
+++ b/fast/stages-multitenant/1-resman-tenant/data/org-policies/serverless.yaml
@@ -3,24 +3,29 @@
# sample subset of useful organization policies, edit to suit requirements
run.allowedIngress:
- allow:
- values:
- - is:internal
+ rules:
+ - allow:
+ values:
+ - is:internal
# run.allowedVPCEgress:
-# allow:
-# values:
+# rules:
+# - allow:
+# values:
# - is:private-ranges-only
# cloudfunctions.allowedIngressSettings:
-# allow:
-# values:
-# - is:ALLOW_INTERNAL_ONLY
+# rules:
+# - allow:
+# values:
+# - is:ALLOW_INTERNAL_ONLY
# cloudfunctions.allowedVpcConnectorEgressSettings:
-# allow:
-# values:
-# - is:PRIVATE_RANGES_ONLY
+# rules:
+# - allow:
+# values:
+# - is:PRIVATE_RANGES_ONLY
# cloudfunctions.requireVPCConnector:
-# enforce: true
+# rules:
+# - enforce: true
diff --git a/fast/stages-multitenant/1-resman-tenant/data/org-policies/sql.yaml b/fast/stages-multitenant/1-resman-tenant/data/org-policies/sql.yaml
index 88b84d9d50..0eee80453a 100644
--- a/fast/stages-multitenant/1-resman-tenant/data/org-policies/sql.yaml
+++ b/fast/stages-multitenant/1-resman-tenant/data/org-policies/sql.yaml
@@ -3,7 +3,9 @@
# sample subset of useful organization policies, edit to suit requirements
sql.restrictAuthorizedNetworks:
- enforce: true
+ rules:
+ - enforce: true
sql.restrictPublicIp:
- enforce: true
+ rules:
+ - enforce: true
diff --git a/fast/stages-multitenant/1-resman-tenant/data/org-policies/storage.yaml b/fast/stages-multitenant/1-resman-tenant/data/org-policies/storage.yaml
index 6c0a673f3a..448357b8bc 100644
--- a/fast/stages-multitenant/1-resman-tenant/data/org-policies/storage.yaml
+++ b/fast/stages-multitenant/1-resman-tenant/data/org-policies/storage.yaml
@@ -3,4 +3,5 @@
# sample subset of useful organization policies, edit to suit requirements
storage.uniformBucketLevelAccess:
- enforce: true
+ rules:
+ - enforce: true
diff --git a/fast/stages/0-bootstrap/README.md b/fast/stages/0-bootstrap/README.md
index 2cab11510d..88bdceb3fb 100644
--- a/fast/stages/0-bootstrap/README.md
+++ b/fast/stages/0-bootstrap/README.md
@@ -14,6 +14,28 @@ Use the following diagram as a simple high level reference for the following sec
string
| ✓ | |
-| [project_id](variables.tf#L100) | Id of the project where datasets will be created. | string
| ✓ | |
+| [project_id](variables.tf#L97) | Id of the project where datasets will be created. | string
| ✓ | |
| [access](variables.tf#L17) | Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`. | map(object({…}))
| | {}
|
| [access_identities](variables.tf#L33) | Map of access identities used for basic access roles. View identities have the format 'project_id\|dataset_id\|table_id'. | map(string)
| | {}
|
| [dataset_access](variables.tf#L39) | Set access in the dataset resource instead of using separate resources. | bool
| | false
|
@@ -188,9 +189,9 @@ module "bigquery-dataset" {
| [iam](variables.tf#L63) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string))
| | {}
|
| [labels](variables.tf#L74) | Dataset labels. | map(string)
| | {}
|
| [location](variables.tf#L80) | Dataset location. | string
| | "EU"
|
-| [options](variables.tf#L86) | Dataset options. | object({…})
| | {…}
|
-| [tables](variables.tf#L105) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…}))
| | {}
|
-| [views](variables.tf#L133) | View definitions. | map(object({…}))
| | {}
|
+| [options](variables.tf#L86) | Dataset options. | object({…})
| | {}
|
+| [tables](variables.tf#L102) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…}))
| | {}
|
+| [views](variables.tf#L130) | View definitions. | map(object({…}))
| | {}
|
## Outputs
diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf
index 47f8fcb53b..f832cd85c3 100644
--- a/modules/bigquery-dataset/main.tf
+++ b/modules/bigquery-dataset/main.tf
@@ -42,7 +42,7 @@ resource "google_bigquery_dataset" "default" {
delete_contents_on_destroy = var.options.delete_contents_on_destroy
default_table_expiration_ms = var.options.default_table_expiration_ms
default_partition_expiration_ms = var.options.default_partition_expiration_ms
-
+ max_time_travel_hours = var.options.max_time_travel_hours
dynamic "access" {
for_each = var.dataset_access ? local.access_domain : {}
content {
diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf
index 5f8028abfb..b44b66585b 100644
--- a/modules/bigquery-dataset/variables.tf
+++ b/modules/bigquery-dataset/variables.tf
@@ -86,15 +86,12 @@ variable "location" {
variable "options" {
description = "Dataset options."
type = object({
- default_table_expiration_ms = number
- default_partition_expiration_ms = number
- delete_contents_on_destroy = bool
+ default_table_expiration_ms = optional(number, null)
+ default_partition_expiration_ms = optional(number, null)
+ delete_contents_on_destroy = optional(bool, false)
+ max_time_travel_hours = optional(number, 168)
})
- default = {
- default_table_expiration_ms = null
- default_partition_expiration_ms = null
- delete_contents_on_destroy = false
- }
+ default = {}
}
variable "project_id" {
diff --git a/modules/folder/README.md b/modules/folder/README.md
index e1ad6809e2..8addd48ef2 100644
--- a/modules/folder/README.md
+++ b/modules/folder/README.md
@@ -42,40 +42,46 @@ module "folder" {
name = "Folder name"
org_policies = {
"compute.disableGuestAttributesAccess" = {
- enforce = true
+ rules = [{ enforce = true }]
}
- "constraints/compute.skipDefaultNetworkCreation" = {
- enforce = true
+ "compute.skipDefaultNetworkCreation" = {
+ rules = [{ enforce = true }]
}
"iam.disableServiceAccountKeyCreation" = {
- enforce = true
+ rules = [{ enforce = true }]
}
"iam.disableServiceAccountKeyUpload" = {
- enforce = false
rules = [
{
condition = {
- expression = "resource.matchTagId(\"tagKeys/1234\", \"tagValues/1234\")"
+ expression = "resource.matchTagId('tagKeys/1234', 'tagValues/1234')"
title = "condition"
description = "test condition"
location = "somewhere"
}
enforce = true
+ },
+ {
+ enforce = false
}
]
}
- "constraints/iam.allowedPolicyMemberDomains" = {
- allow = {
- values = ["C0xxxxxxx", "C0yyyyyyy"]
- }
+ "iam.allowedPolicyMemberDomains" = {
+ rules = [{
+ allow = {
+ values = ["C0xxxxxxx", "C0yyyyyyy"]
+ }
+ }]
}
- "constraints/compute.trustedImageProjects" = {
- allow = {
- values = ["projects/my-project"]
- }
+ "compute.trustedImageProjects" = {
+ rules = [{
+ allow = {
+ values = ["projects/my-project"]
+ }
+ }]
}
- "constraints/compute.vmExternalIpAccess" = {
- deny = { all = true }
+ "compute.vmExternalIpAccess" = {
+ rules = [{ deny = { all = true } }]
}
}
}
@@ -340,10 +346,10 @@ module "folder" {
| [logging_exclusions](variables.tf#L98) | Logging exclusions for this folder in the form {NAME -> FILTER}. | map(string)
| | {}
|
| [logging_sinks](variables.tf#L105) | Logging sinks to create for the organization. | map(object({…}))
| | {}
|
| [name](variables.tf#L135) | Folder name. | string
| | null
|
-| [org_policies](variables.tf#L141) | Organization policies applied to this folder keyed by policy name. | map(object({…}))
| | {}
|
-| [org_policies_data_path](variables.tf#L181) | Path containing org policies in YAML format. | string
| | null
|
-| [parent](variables.tf#L187) | Parent in folders/folder_id or organizations/org_id format. | string
| | null
|
-| [tag_bindings](variables.tf#L197) | Tag bindings for this folder, in key => tag value id format. | map(string)
| | null
|
+| [org_policies](variables.tf#L141) | Organization policies applied to this folder keyed by policy name. | map(object({…}))
| | {}
|
+| [org_policies_data_path](variables.tf#L168) | Path containing org policies in YAML format. | string
| | null
|
+| [parent](variables.tf#L174) | Parent in folders/folder_id or organizations/org_id format. | string
| | null
|
+| [tag_bindings](variables.tf#L184) | Tag bindings for this folder, in key => tag value id format. | map(string)
| | null
|
## Outputs
diff --git a/modules/folder/organization-policies.tf b/modules/folder/organization-policies.tf
index 47532f21be..2bf79c4ab6 100644
--- a/modules/folder/organization-policies.tf
+++ b/modules/folder/organization-policies.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,16 +28,6 @@ locals {
k => {
inherit_from_parent = try(v.inherit_from_parent, null)
reset = try(v.reset, null)
- allow = can(v.allow) ? {
- all = try(v.allow.all, null)
- values = try(v.allow.values, null)
- } : null
- deny = can(v.deny) ? {
- all = try(v.deny.all, null)
- values = try(v.deny.values, null)
- } : null
- enforce = try(v.enforce, true)
-
rules = [
for r in try(v.rules, []) : {
allow = can(r.allow) ? {
@@ -48,7 +38,7 @@ locals {
all = try(r.deny.all, null)
values = try(r.deny.values, null)
} : null
- enforce = try(r.enforce, true)
+ enforce = try(r.enforce, null)
condition = {
description = try(r.condition.description, null)
expression = try(r.condition.expression, null)
@@ -67,8 +57,9 @@ locals {
k => merge(v, {
name = "${local.folder.name}/policies/${k}"
parent = local.folder.name
-
- is_boolean_policy = v.allow == null && v.deny == null
+ is_boolean_policy = (
+ alltrue([for r in v.rules : r.allow == null && r.deny == null])
+ )
has_values = (
length(coalesce(try(v.allow.values, []), [])) > 0 ||
length(coalesce(try(v.deny.values, []), [])) > 0
@@ -90,11 +81,9 @@ resource "google_org_policy_policy" "default" {
for_each = local.org_policies
name = each.value.name
parent = each.value.parent
-
spec {
inherit_from_parent = each.value.inherit_from_parent
reset = each.value.reset
-
dynamic "rules" {
for_each = each.value.rules
iterator = rule
@@ -106,11 +95,14 @@ resource "google_org_policy_policy" "default" {
? upper(tostring(rule.value.enforce))
: null
)
- condition {
- description = rule.value.condition.description
- expression = rule.value.condition.expression
- location = rule.value.condition.location
- title = rule.value.condition.title
+ dynamic "condition" {
+ for_each = rule.value.condition.expression != null ? [1] : []
+ content {
+ description = rule.value.condition.description
+ expression = rule.value.condition.expression
+ location = rule.value.condition.location
+ title = rule.value.condition.title
+ }
}
dynamic "values" {
for_each = rule.value.has_values ? [1] : []
@@ -121,22 +113,5 @@ resource "google_org_policy_policy" "default" {
}
}
}
-
- rules {
- allow_all = try(each.value.allow.all, null) == true ? "TRUE" : null
- deny_all = try(each.value.deny.all, null) == true ? "TRUE" : null
- enforce = (
- each.value.is_boolean_policy && each.value.enforce != null
- ? upper(tostring(each.value.enforce))
- : null
- )
- dynamic "values" {
- for_each = each.value.has_values ? [1] : []
- content {
- allowed_values = try(each.value.allow.values, null)
- denied_values = try(each.value.deny.values, null)
- }
- }
- }
}
}
diff --git a/modules/folder/variables.tf b/modules/folder/variables.tf
index a93ea1aae9..e0abc612c1 100644
--- a/modules/folder/variables.tf
+++ b/modules/folder/variables.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -143,19 +143,6 @@ variable "org_policies" {
type = map(object({
inherit_from_parent = optional(bool) # for list policies only.
reset = optional(bool)
-
- # default (unconditional) values
- allow = optional(object({
- all = optional(bool)
- values = optional(list(string))
- }))
- deny = optional(object({
- all = optional(bool)
- values = optional(list(string))
- }))
- enforce = optional(bool, true) # for boolean policies only.
-
- # conditional values
rules = optional(list(object({
allow = optional(object({
all = optional(bool)
@@ -165,13 +152,13 @@ variable "org_policies" {
all = optional(bool)
values = optional(list(string))
}))
- enforce = optional(bool, true) # for boolean policies only.
- condition = object({
+ enforce = optional(bool) # for boolean policies only.
+ condition = optional(object({
description = optional(string)
expression = optional(string)
location = optional(string)
title = optional(string)
- })
+ }), {})
})), [])
}))
default = {}
diff --git a/modules/gke-cluster/README.md b/modules/gke-cluster/README.md
index 2e09aeb113..2d60e487c4 100644
--- a/modules/gke-cluster/README.md
+++ b/modules/gke-cluster/README.md
@@ -91,8 +91,7 @@ module "cluster-autopilot" {
master_ipv4_cidr_block = "192.168.0.0/28"
}
enable_features = {
- autopilot = true
- workload_identity = false
+ autopilot = true
}
}
# tftest modules=1 resources=1 inventory=autopilot.yaml
@@ -162,5 +161,6 @@ module "cluster-1" {
| [name](outputs.tf#L49) | Cluster name. | |
| [notifications](outputs.tf#L54) | GKE PubSub notifications topic. | |
| [self_link](outputs.tf#L59) | Cluster self link. | ✓ |
+| [workload_identity_pool](outputs.tf#L65) | Workload identity pool. | |
diff --git a/modules/gke-cluster/main.tf b/modules/gke-cluster/main.tf
index 0079dd8d88..107d8341cf 100644
--- a/modules/gke-cluster/main.tf
+++ b/modules/gke-cluster/main.tf
@@ -379,7 +379,7 @@ resource "google_container_cluster" "cluster" {
}
dynamic "workload_identity_config" {
- for_each = var.enable_features.workload_identity ? [""] : []
+ for_each = (var.enable_features.workload_identity && !var.enable_features.autopilot) ? [""] : []
content {
workload_pool = "${var.project_id}.svc.id.goog"
}
diff --git a/modules/gke-cluster/outputs.tf b/modules/gke-cluster/outputs.tf
index f98f4f54c7..c02c9be2b6 100644
--- a/modules/gke-cluster/outputs.tf
+++ b/modules/gke-cluster/outputs.tf
@@ -61,3 +61,11 @@ output "self_link" {
sensitive = true
value = google_container_cluster.cluster.self_link
}
+
+output "workload_identity_pool" {
+ description = "Workload identity pool."
+ value = "${var.project_id}.svc.id.goog"
+ depends_on = [
+ google_container_cluster.cluster
+ ]
+}
\ No newline at end of file
diff --git a/modules/gke-nodepool/main.tf b/modules/gke-nodepool/main.tf
index ad0c053f35..9ae4cf2844 100644
--- a/modules/gke-nodepool/main.tf
+++ b/modules/gke-nodepool/main.tf
@@ -115,9 +115,9 @@ resource "google_container_node_pool" "nodepool" {
dynamic "network_config" {
for_each = var.pod_range != null ? [""] : []
content {
- create_pod_range = var.pod_range.create
- pod_ipv4_cidr_block = var.pod_range.cidr
- pod_range = var.pod_range.name
+ create_pod_range = var.pod_range.secondary_pod_range.create
+ pod_ipv4_cidr_block = var.pod_range.secondary_pod_range.cidr
+ pod_range = var.pod_range.secondary_pod_range.name
}
}
diff --git a/modules/net-ilb-l7/README.md b/modules/net-ilb-l7/README.md
index b5862f31e6..b6436e92d5 100644
--- a/modules/net-ilb-l7/README.md
+++ b/modules/net-ilb-l7/README.md
@@ -326,8 +326,10 @@ module "ilb-l7" {
group = "my-neg"
max_rate = { per_endpoint = 1 }
}]
+ health_checks = []
}
}
+ health_check_configs = {}
neg_configs = {
my-neg = {
cloudrun = {
@@ -343,7 +345,7 @@ module "ilb-l7" {
subnetwork = var.subnet.self_link
}
}
-# tftest modules=1 resources=6
+# tftest modules=1 resources=5
```
### URL Map
diff --git a/modules/net-ilb-l7/backend-service.tf b/modules/net-ilb-l7/backend-service.tf
index a517bd08c8..ea758835bd 100644
--- a/modules/net-ilb-l7/backend-service.tf
+++ b/modules/net-ilb-l7/backend-service.tf
@@ -46,7 +46,7 @@ resource "google_compute_region_backend_service" "default" {
description = var.description
affinity_cookie_ttl_sec = each.value.affinity_cookie_ttl_sec
connection_draining_timeout_sec = each.value.connection_draining_timeout_sec
- health_checks = [
+ health_checks = length(each.value.health_checks) == 0 ? null : [
for k in each.value.health_checks : lookup(local.hc_ids, k, k)
] # not for internet / serverless NEGs
locality_lb_policy = each.value.locality_lb_policy
diff --git a/modules/net-vpc/README.md b/modules/net-vpc/README.md
index dbd8550221..bd5675d239 100644
--- a/modules/net-vpc/README.md
+++ b/modules/net-vpc/README.md
@@ -314,7 +314,7 @@ module "vpc" {
name = "my-network"
data_folder = "config/subnets"
}
-# tftest modules=1 resources=3 files=subnet-simple,subnet-detailed inventory=factory.yaml
+# tftest modules=1 resources=4 files=subnet-simple,subnet-detailed inventory=factory.yaml
```
```yaml
diff --git a/modules/net-vpc/subnets.tf b/modules/net-vpc/subnets.tf
index ae094ecfa0..7c03bfca3b 100644
--- a/modules/net-vpc/subnets.tf
+++ b/modules/net-vpc/subnets.tf
@@ -31,6 +31,9 @@ locals {
flow_logs_config = try(v.flow_logs, null)
ipv6 = try(v.ipv6, null)
secondary_ip_ranges = try(v.secondary_ip_ranges, null)
+ iam_groups = try(v.iam_groups, [])
+ iam_users = try(v.iam_users, [])
+ iam_service_accounts = try(v.iam_service_accounts, [])
}
}
_factory_subnets_iam = [
diff --git a/modules/organization/README.md b/modules/organization/README.md
index b6caa3cd03..39b5ff29e7 100644
--- a/modules/organization/README.md
+++ b/modules/organization/README.md
@@ -25,50 +25,77 @@ module "org" {
iam_additive_members = {
"user:compute@example.org" = ["roles/compute.admin", "roles/container.viewer"]
}
-
+ tags = {
+ allowexternal = {
+ description = "Allow external identities."
+ values = {
+ true = {}, false = {}
+ }
+ }
+ }
org_policies = {
"custom.gkeEnableAutoUpgrade" = {
- enforce = true
+ rules = [{ enforce = true }]
}
"compute.disableGuestAttributesAccess" = {
- enforce = true
+ rules = [{ enforce = true }]
}
- "constraints/compute.skipDefaultNetworkCreation" = {
- enforce = true
+ "compute.skipDefaultNetworkCreation" = {
+ rules = [{ enforce = true }]
}
"iam.disableServiceAccountKeyCreation" = {
- enforce = true
+ rules = [{ enforce = true }]
}
"iam.disableServiceAccountKeyUpload" = {
- enforce = false
rules = [
{
condition = {
- expression = "resource.matchTagId(\"tagKeys/1234\", \"tagValues/1234\")"
+ expression = "resource.matchTagId('tagKeys/1234', 'tagValues/1234')"
title = "condition"
description = "test condition"
location = "somewhere"
}
enforce = true
+ },
+ {
+ enforce = false
}
]
}
- "constraints/iam.allowedPolicyMemberDomains" = {
- allow = {
- values = ["C0xxxxxxx", "C0yyyyyyy"]
- }
+ "iam.allowedPolicyMemberDomains" = {
+ rules = [
+ {
+ allow = { all = true }
+ condition = {
+ expression = "resource.matchTag('1234567890/allowexternal', 'true')"
+ title = "Allow external identities"
+ description = "Allow external identities when resource has the `allowexternal` tag set to true."
+ }
+ },
+ {
+ allow = { values = ["C0xxxxxxx", "C0yyyyyyy"] }
+ condition = {
+ expression = "!resource.matchTag('1234567890/allowexternal', 'true')"
+ title = ""
+ description = "For any resource without allowexternal=true, only allow identities from restricted domains."
+ }
+ }
+ ]
}
- "constraints/compute.trustedImageProjects" = {
- allow = {
- values = ["projects/my-project"]
- }
+
+ "compute.trustedImageProjects" = {
+ rules = [{
+ allow = {
+ values = ["projects/my-project"]
+ }
+ }]
}
- "constraints/compute.vmExternalIpAccess" = {
- deny = { all = true }
+ "compute.vmExternalIpAccess" = {
+ rules = [{ deny = { all = true } }]
}
}
}
-# tftest modules=1 resources=13 inventory=basic.yaml
+# tftest modules=1 resources=16 inventory=basic.yaml
```
## IAM
@@ -111,7 +138,7 @@ module "org" {
# not necessarily to enforce on the org level, policy may be applied on folder/project levels
org_policies = {
"custom.gkeEnableAutoUpgrade" = {
- enforce = true
+ rules = [{ enforce = true }]
}
}
}
@@ -131,7 +158,7 @@ module "org" {
org_policy_custom_constraints_data_path = "configs/custom-constraints"
org_policies = {
"custom.gkeEnableAutoUpgrade" = {
- enforce = true
+ rules = [{ enforce = true }]
}
}
}
@@ -447,7 +474,7 @@ module "org" {
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [organization_id](variables.tf#L246) | Organization id in organizations/nnnnnn format. | string
| ✓ | |
+| [organization_id](variables.tf#L234) | Organization id in organizations/nnnnnn format. | string
| ✓ | |
| [contacts](variables.tf#L17) | List of essential contacts for this resource. Must be in the form EMAIL -> [NOTIFICATION_TYPES]. Valid notification types are ALL, SUSPENSION, SECURITY, TECHNICAL, BILLING, LEGAL, PRODUCT_UPDATES. | map(list(string))
| | {}
|
| [custom_roles](variables.tf#L24) | Map of role name => list of permissions to create in this project. | map(list(string))
| | {}
|
| [firewall_policies](variables.tf#L31) | Hierarchical firewall policy rules created in the organization. | map(map(object({…})))
| | {}
|
@@ -463,12 +490,12 @@ module "org" {
| [logging_exclusions](variables.tf#L122) | Logging exclusions for this organization in the form {NAME -> FILTER}. | map(string)
| | {}
|
| [logging_sinks](variables.tf#L129) | Logging sinks to create for the organization. | map(object({…}))
| | {}
|
| [network_tags](variables.tf#L159) | Network tags by key name. If `id` is provided, key creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…}))
| | {}
|
-| [org_policies](variables.tf#L181) | Organization policies applied to this organization keyed by policy name. | map(object({…}))
| | {}
|
-| [org_policies_data_path](variables.tf#L220) | Path containing org policies in YAML format. | string
| | null
|
-| [org_policy_custom_constraints](variables.tf#L226) | Organization policiy custom constraints keyed by constraint name. | map(object({…}))
| | {}
|
-| [org_policy_custom_constraints_data_path](variables.tf#L240) | Path containing org policy custom constraints in YAML format. | string
| | null
|
-| [tag_bindings](variables.tf#L255) | Tag bindings for this organization, in key => tag value id format. | map(string)
| | null
|
-| [tags](variables.tf#L261) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…}))
| | {}
|
+| [org_policies](variables.tf#L181) | Organization policies applied to this organization keyed by policy name. | map(object({…}))
| | {}
|
+| [org_policies_data_path](variables.tf#L208) | Path containing org policies in YAML format. | string
| | null
|
+| [org_policy_custom_constraints](variables.tf#L214) | Organization policiy custom constraints keyed by constraint name. | map(object({…}))
| | {}
|
+| [org_policy_custom_constraints_data_path](variables.tf#L228) | Path containing org policy custom constraints in YAML format. | string
| | null
|
+| [tag_bindings](variables.tf#L243) | Tag bindings for this organization, in key => tag value id format. | map(string)
| | null
|
+| [tags](variables.tf#L249) | Tags by key name. If `id` is provided, key or value creation is skipped. The `iam` attribute behaves like the similarly named one at module level. | map(object({…}))
| | {}
|
## Outputs
diff --git a/modules/organization/organization-policies.tf b/modules/organization/organization-policies.tf
index 1a99ef9a1c..b43c5955c6 100644
--- a/modules/organization/organization-policies.tf
+++ b/modules/organization/organization-policies.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,16 +28,6 @@ locals {
k => {
inherit_from_parent = try(v.inherit_from_parent, null)
reset = try(v.reset, null)
- allow = can(v.allow) ? {
- all = try(v.allow.all, null)
- values = try(v.allow.values, null)
- } : null
- deny = can(v.deny) ? {
- all = try(v.deny.all, null)
- values = try(v.deny.values, null)
- } : null
- enforce = try(v.enforce, true)
-
rules = [
for r in try(v.rules, []) : {
allow = can(r.allow) ? {
@@ -48,7 +38,7 @@ locals {
all = try(r.deny.all, null)
values = try(r.deny.values, null)
} : null
- enforce = try(r.enforce, true)
+ enforce = try(r.enforce, null)
condition = {
description = try(r.condition.description, null)
expression = try(r.condition.expression, null)
@@ -67,8 +57,9 @@ locals {
k => merge(v, {
name = "${var.organization_id}/policies/${k}"
parent = var.organization_id
-
- is_boolean_policy = v.allow == null && v.deny == null
+ is_boolean_policy = (
+ alltrue([for r in v.rules : r.allow == null && r.deny == null])
+ )
has_values = (
length(coalesce(try(v.allow.values, []), [])) > 0 ||
length(coalesce(try(v.deny.values, []), [])) > 0
@@ -90,11 +81,9 @@ resource "google_org_policy_policy" "default" {
for_each = local.org_policies
name = each.value.name
parent = each.value.parent
-
spec {
inherit_from_parent = each.value.inherit_from_parent
reset = each.value.reset
-
dynamic "rules" {
for_each = each.value.rules
iterator = rule
@@ -106,11 +95,14 @@ resource "google_org_policy_policy" "default" {
? upper(tostring(rule.value.enforce))
: null
)
- condition {
- description = rule.value.condition.description
- expression = rule.value.condition.expression
- location = rule.value.condition.location
- title = rule.value.condition.title
+ dynamic "condition" {
+ for_each = rule.value.condition.expression != null ? [1] : []
+ content {
+ description = rule.value.condition.description
+ expression = rule.value.condition.expression
+ location = rule.value.condition.location
+ title = rule.value.condition.title
+ }
}
dynamic "values" {
for_each = rule.value.has_values ? [1] : []
@@ -121,25 +113,7 @@ resource "google_org_policy_policy" "default" {
}
}
}
-
- rules {
- allow_all = try(each.value.allow.all, null) == true ? "TRUE" : null
- deny_all = try(each.value.deny.all, null) == true ? "TRUE" : null
- enforce = (
- each.value.is_boolean_policy && each.value.enforce != null
- ? upper(tostring(each.value.enforce))
- : null
- )
- dynamic "values" {
- for_each = each.value.has_values ? [1] : []
- content {
- allowed_values = try(each.value.allow.values, null)
- denied_values = try(each.value.deny.values, null)
- }
- }
- }
}
-
depends_on = [
google_organization_iam_audit_config.config,
google_organization_iam_binding.authoritative,
diff --git a/modules/organization/variables.tf b/modules/organization/variables.tf
index ced5cad3d2..619056a0af 100644
--- a/modules/organization/variables.tf
+++ b/modules/organization/variables.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -183,18 +183,6 @@ variable "org_policies" {
type = map(object({
inherit_from_parent = optional(bool) # for list policies only.
reset = optional(bool)
-
- # default (unconditional) values
- allow = optional(object({
- all = optional(bool)
- values = optional(list(string))
- }))
- deny = optional(object({
- all = optional(bool)
- values = optional(list(string))
- }))
- enforce = optional(bool, true) # for boolean policies only.
- # conditional values
rules = optional(list(object({
allow = optional(object({
all = optional(bool)
@@ -204,13 +192,13 @@ variable "org_policies" {
all = optional(bool)
values = optional(list(string))
}))
- enforce = optional(bool, true) # for boolean policies only.
- condition = object({
+ enforce = optional(bool) # for boolean policies only.
+ condition = optional(object({
description = optional(string)
expression = optional(string)
location = optional(string)
title = optional(string)
- })
+ }), {})
})), [])
}))
default = {}
diff --git a/modules/project/README.md b/modules/project/README.md
index fbc4ab294d..730fe19080 100644
--- a/modules/project/README.md
+++ b/modules/project/README.md
@@ -138,6 +138,29 @@ module "project" {
# tftest modules=1 resources=2
```
+### Using shortcodes for Service Identities in additive IAM
+Most Service Identities contains project number in their e-mail address and this prevents additive IAM to work, as these values are not known at moment of execution of `terraform plan` (its not an issue for authoritative IAM). To refer current project Service Identities you may use shortcodes for Service Identities similarly as for `service_identity_iam` when configuring Shared VPC.
+
+```hcl
+module "project" {
+ source = "./fabric/modules/project"
+ name = "project-example"
+
+ services = [
+ "run.googleapis.com",
+ "container.googleapis.com",
+ ]
+
+ iam_additive = {
+ "roles/editor" = ["cloudservices"]
+ "roles/vpcaccess.user" = ["cloudrun"]
+ "roles/container.hostServiceAgentUser" = ["container-engine"]
+ }
+}
+# tftest modules=1 resources=6
+```
+
+
### Service identities requiring manual IAM grants
The module will create service identities at project creation instead of creating of them at the time of first use. This allows granting these service identities roles in other projects, something which is usually necessary in a Shared VPC context.
@@ -190,7 +213,6 @@ module "service-project" {
source = "./fabric/modules/project"
name = "my-service-project"
shared_vpc_service_config = {
- attach = true
host_project = module.host-project.project_id
service_identity_iam = {
"roles/compute.networkUser" = [
@@ -221,40 +243,46 @@ module "project" {
prefix = "foo"
org_policies = {
"compute.disableGuestAttributesAccess" = {
- enforce = true
+ rules = [{ enforce = true }]
}
- "constraints/compute.skipDefaultNetworkCreation" = {
- enforce = true
+ "compute.skipDefaultNetworkCreation" = {
+ rules = [{ enforce = true }]
}
"iam.disableServiceAccountKeyCreation" = {
- enforce = true
+ rules = [{ enforce = true }]
}
"iam.disableServiceAccountKeyUpload" = {
- enforce = false
rules = [
{
condition = {
- expression = "resource.matchTagId(\"tagKeys/1234\", \"tagValues/1234\")"
+ expression = "resource.matchTagId('tagKeys/1234', 'tagValues/1234')"
title = "condition"
description = "test condition"
location = "somewhere"
}
enforce = true
+ },
+ {
+ enforce = false
}
]
}
- "constraints/iam.allowedPolicyMemberDomains" = {
- allow = {
- values = ["C0xxxxxxx", "C0yyyyyyy"]
- }
+ "iam.allowedPolicyMemberDomains" = {
+ rules = [{
+ allow = {
+ values = ["C0xxxxxxx", "C0yyyyyyy"]
+ }
+ }]
}
- "constraints/compute.trustedImageProjects" = {
- allow = {
- values = ["projects/my-project"]
- }
+ "compute.trustedImageProjects" = {
+ rules = [{
+ allow = {
+ values = ["projects/my-project"]
+ }
+ }]
}
- "constraints/compute.vmExternalIpAccess" = {
- deny = { all = true }
+ "compute.vmExternalIpAccess" = {
+ rules = [{ deny = { all = true } }]
}
}
}
@@ -284,36 +312,42 @@ module "project" {
```yaml
# tftest-file id=boolean path=configs/org-policies/boolean.yaml
compute.disableGuestAttributesAccess:
- enforce: true
-constraints/compute.skipDefaultNetworkCreation:
- enforce: true
+ rules:
+ - enforce: true
+compute.skipDefaultNetworkCreation:
+ rules:
+ - enforce: true
iam.disableServiceAccountKeyCreation:
- enforce: true
+ rules:
+ - enforce: true
iam.disableServiceAccountKeyUpload:
- enforce: false
rules:
- condition:
description: test condition
- expression: resource.matchTagId("tagKeys/1234", "tagValues/1234")
+ expression: resource.matchTagId('tagKeys/1234', 'tagValues/1234')
location: somewhere
title: condition
enforce: true
+ - enforce: false
```
```yaml
# tftest-file id=list path=configs/org-policies/list.yaml
-constraints/compute.trustedImageProjects:
- allow:
- values:
- - projects/my-project
-constraints/compute.vmExternalIpAccess:
- deny:
- all: true
-constraints/iam.allowedPolicyMemberDomains:
- allow:
- values:
- - C0xxxxxxx
- - C0yyyyyyy
+compute.trustedImageProjects:
+ rules:
+ - allow:
+ values:
+ - projects/my-project
+compute.vmExternalIpAccess:
+ rules:
+ - deny:
+ all: true
+iam.allowedPolicyMemberDomains:
+ rules:
+ - allow:
+ values:
+ - C0xxxxxxx
+ - C0yyyyyyy
```
@@ -500,23 +534,23 @@ output "compute_robot" {
| [logging_exclusions](variables.tf#L95) | Logging exclusions for this project in the form {NAME -> FILTER}. | map(string)
| | {}
|
| [logging_sinks](variables.tf#L102) | Logging sinks to create for this project. | map(object({…}))
| | {}
|
| [metric_scopes](variables.tf#L133) | List of projects that will act as metric scopes for this project. | list(string)
| | []
|
-| [org_policies](variables.tf#L145) | Organization policies applied to this project keyed by policy name. | map(object({…}))
| | {}
|
-| [org_policies_data_path](variables.tf#L185) | Path containing org policies in YAML format. | string
| | null
|
-| [oslogin](variables.tf#L191) | Enable OS Login. | bool
| | false
|
-| [oslogin_admins](variables.tf#L197) | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string)
| | []
|
-| [oslogin_users](variables.tf#L205) | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string)
| | []
|
-| [parent](variables.tf#L212) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string
| | null
|
-| [prefix](variables.tf#L222) | Optional prefix used to generate project id and name. | string
| | null
|
-| [project_create](variables.tf#L232) | Create project. When set to false, uses a data source to reference existing project. | bool
| | true
|
-| [service_config](variables.tf#L238) | Configure service API activation. | object({…})
| | {…}
|
-| [service_encryption_key_ids](variables.tf#L250) | Cloud KMS encryption key in {SERVICE => [KEY_URL]} format. | map(list(string))
| | {}
|
-| [service_perimeter_bridges](variables.tf#L257) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | list(string)
| | null
|
-| [service_perimeter_standard](variables.tf#L264) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | string
| | null
|
-| [services](variables.tf#L270) | Service APIs to enable. | list(string)
| | []
|
-| [shared_vpc_host_config](variables.tf#L276) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…})
| | null
|
-| [shared_vpc_service_config](variables.tf#L285) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…})
| | null
|
-| [skip_delete](variables.tf#L295) | Allows the underlying resources to be destroyed without destroying the project itself. | bool
| | false
|
-| [tag_bindings](variables.tf#L301) | Tag bindings for this project, in key => tag value id format. | map(string)
| | null
|
+| [org_policies](variables.tf#L145) | Organization policies applied to this project keyed by policy name. | map(object({…}))
| | {}
|
+| [org_policies_data_path](variables.tf#L172) | Path containing org policies in YAML format. | string
| | null
|
+| [oslogin](variables.tf#L178) | Enable OS Login. | bool
| | false
|
+| [oslogin_admins](variables.tf#L184) | List of IAM-style identities that will be granted roles necessary for OS Login administrators. | list(string)
| | []
|
+| [oslogin_users](variables.tf#L192) | List of IAM-style identities that will be granted roles necessary for OS Login users. | list(string)
| | []
|
+| [parent](variables.tf#L199) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string
| | null
|
+| [prefix](variables.tf#L209) | Optional prefix used to generate project id and name. | string
| | null
|
+| [project_create](variables.tf#L219) | Create project. When set to false, uses a data source to reference existing project. | bool
| | true
|
+| [service_config](variables.tf#L225) | Configure service API activation. | object({…})
| | {…}
|
+| [service_encryption_key_ids](variables.tf#L237) | Cloud KMS encryption key in {SERVICE => [KEY_URL]} format. | map(list(string))
| | {}
|
+| [service_perimeter_bridges](variables.tf#L244) | Name of VPC-SC Bridge perimeters to add project into. See comment in the variables file for format. | list(string)
| | null
|
+| [service_perimeter_standard](variables.tf#L251) | Name of VPC-SC Standard perimeter to add project into. See comment in the variables file for format. | string
| | null
|
+| [services](variables.tf#L257) | Service APIs to enable. | list(string)
| | []
|
+| [shared_vpc_host_config](variables.tf#L263) | Configures this project as a Shared VPC host project (mutually exclusive with shared_vpc_service_project). | object({…})
| | null
|
+| [shared_vpc_service_config](variables.tf#L272) | Configures this project as a Shared VPC service project (mutually exclusive with shared_vpc_host_config). | object({…})
| | null
|
+| [skip_delete](variables.tf#L282) | Allows the underlying resources to be destroyed without destroying the project itself. | bool
| | false
|
+| [tag_bindings](variables.tf#L288) | Tag bindings for this project, in key => tag value id format. | map(string)
| | null
|
## Outputs
diff --git a/modules/project/iam.tf b/modules/project/iam.tf
index 69925cc767..3ed2d2a6fd 100644
--- a/modules/project/iam.tf
+++ b/modules/project/iam.tf
@@ -47,7 +47,18 @@ locals {
}
iam_additive = {
for pair in concat(local._iam_additive_pairs, local._iam_additive_member_pairs) :
- "${pair.role}-${pair.member}" => pair
+ "${pair.role}-${pair.member}" => {
+ role = pair.role
+ member = (
+ pair.member == "cloudservices"
+ ? "serviceAccount:${local.service_account_cloud_services}"
+ : pair.member == "default-compute"
+ ? "serviceAccount:${local.service_accounts_default.compute}"
+ : pair.member == "default-gae"
+ ? "serviceAccount:${local.service_accounts_default.gae}"
+ : try("serviceAccount:${local.service_accounts_robots[pair.member]}", pair.member)
+ )
+ }
}
}
diff --git a/modules/project/organization-policies.tf b/modules/project/organization-policies.tf
index 4ff5bb9922..37e6f2531f 100644
--- a/modules/project/organization-policies.tf
+++ b/modules/project/organization-policies.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,16 +28,6 @@ locals {
k => {
inherit_from_parent = try(v.inherit_from_parent, null)
reset = try(v.reset, null)
- allow = can(v.allow) ? {
- all = try(v.allow.all, null)
- values = try(v.allow.values, null)
- } : null
- deny = can(v.deny) ? {
- all = try(v.deny.all, null)
- values = try(v.deny.values, null)
- } : null
- enforce = try(v.enforce, true)
-
rules = [
for r in try(v.rules, []) : {
allow = can(r.allow) ? {
@@ -48,7 +38,7 @@ locals {
all = try(r.deny.all, null)
values = try(r.deny.values, null)
} : null
- enforce = try(r.enforce, true)
+ enforce = try(r.enforce, null)
condition = {
description = try(r.condition.description, null)
expression = try(r.condition.expression, null)
@@ -67,8 +57,9 @@ locals {
k => merge(v, {
name = "projects/${local.project.project_id}/policies/${k}"
parent = "projects/${local.project.project_id}"
-
- is_boolean_policy = v.allow == null && v.deny == null
+ is_boolean_policy = (
+ alltrue([for r in v.rules : r.allow == null && r.deny == null])
+ )
has_values = (
length(coalesce(try(v.allow.values, []), [])) > 0 ||
length(coalesce(try(v.deny.values, []), [])) > 0
@@ -90,11 +81,9 @@ resource "google_org_policy_policy" "default" {
for_each = local.org_policies
name = each.value.name
parent = each.value.parent
-
spec {
inherit_from_parent = each.value.inherit_from_parent
reset = each.value.reset
-
dynamic "rules" {
for_each = each.value.rules
iterator = rule
@@ -106,11 +95,14 @@ resource "google_org_policy_policy" "default" {
? upper(tostring(rule.value.enforce))
: null
)
- condition {
- description = rule.value.condition.description
- expression = rule.value.condition.expression
- location = rule.value.condition.location
- title = rule.value.condition.title
+ dynamic "condition" {
+ for_each = rule.value.condition.expression != null ? [1] : []
+ content {
+ description = rule.value.condition.description
+ expression = rule.value.condition.expression
+ location = rule.value.condition.location
+ title = rule.value.condition.title
+ }
}
dynamic "values" {
for_each = rule.value.has_values ? [1] : []
@@ -121,22 +113,5 @@ resource "google_org_policy_policy" "default" {
}
}
}
-
- rules {
- allow_all = try(each.value.allow.all, null) == true ? "TRUE" : null
- deny_all = try(each.value.deny.all, null) == true ? "TRUE" : null
- enforce = (
- each.value.is_boolean_policy && each.value.enforce != null
- ? upper(tostring(each.value.enforce))
- : null
- )
- dynamic "values" {
- for_each = each.value.has_values ? [1] : []
- content {
- allowed_values = try(each.value.allow.values, null)
- denied_values = try(each.value.deny.values, null)
- }
- }
- }
}
}
diff --git a/modules/project/variables.tf b/modules/project/variables.tf
index 3769a1fb4e..ede3a8c6ea 100644
--- a/modules/project/variables.tf
+++ b/modules/project/variables.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -147,19 +147,6 @@ variable "org_policies" {
type = map(object({
inherit_from_parent = optional(bool) # for list policies only.
reset = optional(bool)
-
- # default (unconditional) values
- allow = optional(object({
- all = optional(bool)
- values = optional(list(string))
- }))
- deny = optional(object({
- all = optional(bool)
- values = optional(list(string))
- }))
- enforce = optional(bool, true) # for boolean policies only.
-
- # conditional values
rules = optional(list(object({
allow = optional(object({
all = optional(bool)
@@ -169,13 +156,13 @@ variable "org_policies" {
all = optional(bool)
values = optional(list(string))
}))
- enforce = optional(bool, true) # for boolean policies only.
- condition = object({
+ enforce = optional(bool) # for boolean policies only.
+ condition = optional(object({
description = optional(string)
expression = optional(string)
location = optional(string)
title = optional(string)
- })
+ }), {})
})), [])
}))
default = {}
diff --git a/modules/projects-data-source/README.md b/modules/projects-data-source/README.md
index e48ac9770b..5d35f1ab0c 100644
--- a/modules/projects-data-source/README.md
+++ b/modules/projects-data-source/README.md
@@ -1,9 +1,14 @@
# Projects Data Source Module
-This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects and folders under a specific `parent` recursively.
+This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects under a specific `parent` recursively with only one API call against [Cloud Asset Inventory](https://cloud.google.com/asset-inventory) service.
A good usage pattern would be when we want all the projects under a specific folder (including nested subfolders) to be included into [VPC Service Controls](../vpc-sc/). Instead of manually maintaining the list of project numbers as an input to the `vpc-sc` module we can use that module to retrieve all the project numbers dynamically.
+### IAM Permissions required
+
+- `roles/cloudasset.viewer` on the `parent` level or above
+
+
## Examples
### All projects in my org
@@ -14,12 +19,8 @@ module "my-org" {
parent = "organizations/123456789"
}
-output "projects" {
- value = module.my-org.projects
-}
-
-output "folders" {
- value = module.my-org.folders
+output "project_numbers" {
+ value = module.my-org.project_numbers
}
# tftest skip (uses data sources)
@@ -31,18 +32,46 @@ output "folders" {
module "my-dev" {
source = "./fabric/modules/projects-data-source"
parent = "folders/123456789"
- filter = "labels.env:DEV lifecycleState:ACTIVE"
+ query = "labels.env:DEV state:ACTIVE"
}
output "dev-projects" {
value = module.my-dev.projects
}
-output "dev-folders" {
- value = module.my-dev.folders
+# tftest skip (uses data sources)
+```
+
+### Projects under org with folder/project exclusions
+```hcl
+module "my-filtered" {
+ source = "./fabric/modules/projects-data-source"
+ parent = "organizations/123456789"
+ ignore_projects = [
+ "sandbox-*", # wildcard ignore
+ "project-full-id", # specific project id
+ "0123456789" # specific project number
+ ]
+
+ include_projects = [
+ "sandbox-114", # include specific project which was excluded by wildcard
+ "415216609246" # include specific project which was excluded by wildcard (by project number)
+ ]
+
+ ignore_folders = [ # subfolders are ingoner as well
+ "343991594985",
+ "437102807785",
+ "345245235245"
+ ]
+ query = "state:ACTIVE"
+}
+
+output "filtered-projects" {
+ value = module.my-filtered.projects
}
# tftest skip (uses data sources)
+
```
@@ -50,15 +79,17 @@ output "dev-folders" {
| name | description | type | required | default |
|---|---|:---:|:---:|:---:|
-| [parent](variables.tf#L23) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string
| ✓ | |
-| [filter](variables.tf#L17) | A string filter as defined in the [REST API](https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#query-parameters). | string
| | "lifecycleState:ACTIVE"
|
+| [parent](variables.tf#L55) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string
| ✓ | |
+| [ignore_folders](variables.tf#L17) | A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable. | list(string)
| | []
|
+| [ignore_projects](variables.tf#L28) | A list of project IDs, numbers or prefixes to exclude matching projects from the module output. | list(string)
| | []
|
+| [include_projects](variables.tf#L41) | A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries. | list(string)
| | []
|
+| [query](variables.tf#L64) | A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax). | string
| | "state:ACTIVE"
|
## Outputs
| name | description | sensitive |
|---|---|:---:|
-| [folders](outputs.tf#L17) | Map of folders attributes keyed by folder id. | |
-| [project_numbers](outputs.tf#L22) | List of project numbers. | |
-| [projects](outputs.tf#L27) | Map of projects attributes keyed by projects id. | |
+| [project_numbers](outputs.tf#L17) | List of project numbers. | |
+| [projects](outputs.tf#L22) | List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format. | |
diff --git a/modules/projects-data-source/main.tf b/modules/projects-data-source/main.tf
index 76df425e5c..6bd5631ccf 100644
--- a/modules/projects-data-source/main.tf
+++ b/modules/projects-data-source/main.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,129 +15,27 @@
*/
locals {
- folders_l1_map = { for item in data.google_folders.folders_l1.folders : item.name => item }
-
- folders_l2_map = merge([
- for _, v in data.google_folders.folders_l2 :
- { for item in v.folders : item.name => item }
- ]...)
-
- folders_l3_map = merge([
- for _, v in data.google_folders.folders_l3 :
- { for item in v.folders : item.name => item }
- ]...)
-
- folders_l4_map = merge([
- for _, v in data.google_folders.folders_l4 :
- { for item in v.folders : item.name => item }
- ]...)
-
- folders_l5_map = merge([
- for _, v in data.google_folders.folders_l5 :
- { for item in v.folders : item.name => item }
- ]...)
-
- folders_l6_map = merge([
- for _, v in data.google_folders.folders_l6 :
- { for item in v.folders : item.name => item }
- ]...)
-
- folders_l7_map = merge([
- for _, v in data.google_folders.folders_l7 :
- { for item in v.folders : item.name => item }
- ]...)
-
- folders_l8_map = merge([
- for _, v in data.google_folders.folders_l8 :
- { for item in v.folders : item.name => item }
- ]...)
-
- folders_l9_map = merge([
- for _, v in data.google_folders.folders_l9 :
- { for item in v.folders : item.name => item }
- ]...)
-
- folders_l10_map = merge([
- for _, v in data.google_folders.folders_l10 :
- { for item in v.folders : item.name => item }
- ]...)
-
- all_folders = merge(
- local.folders_l1_map,
- local.folders_l2_map,
- local.folders_l3_map,
- local.folders_l4_map,
- local.folders_l5_map,
- local.folders_l6_map,
- local.folders_l7_map,
- local.folders_l8_map,
- local.folders_l9_map,
- local.folders_l10_map
+ _ignore_folder_numbers = [for folder_id in var.ignore_folders : trimprefix(folder_id, "folders/")]
+ _ignore_folders_query = join(" AND NOT folders:", concat([""], local._ignore_folder_numbers))
+ query = var.query != "" ? (
+ format("%s%s", var.query, local._ignore_folders_query)
+ ) : (
+ format("%s%s", var.query, trimprefix(local._ignore_folders_query, " AND "))
)
- parent_ids = toset(concat(
- [split("/", var.parent)[1]],
- [for k, _ in local.all_folders : split("/", k)[1]]
- ))
-
- projects = merge([
- for _, v in data.google_projects.projects :
- { for item in v.projects : item.project_id => item }
- ]...)
-}
-
-# 10 datasources are used to cover 10 possible nested layers in GCP organization hirerarcy.
-data "google_folders" "folders_l1" {
- parent_id = var.parent
-}
-
-data "google_folders" "folders_l2" {
- for_each = local.folders_l1_map
- parent_id = each.value.name
-}
-
-data "google_folders" "folders_l3" {
- for_each = local.folders_l2_map
- parent_id = each.value.name
-}
-
-data "google_folders" "folders_l4" {
- for_each = local.folders_l3_map
- parent_id = each.value.name
-}
-
-data "google_folders" "folders_l5" {
- for_each = local.folders_l4_map
- parent_id = each.value.name
-}
-
-data "google_folders" "folders_l6" {
- for_each = local.folders_l5_map
- parent_id = each.value.name
-}
-
-data "google_folders" "folders_l7" {
- for_each = local.folders_l6_map
- parent_id = each.value.name
-}
-
-data "google_folders" "folders_l8" {
- for_each = local.folders_l7_map
- parent_id = each.value.name
-}
-
-data "google_folders" "folders_l9" {
- for_each = local.folders_l8_map
- parent_id = each.value.name
-}
-
-data "google_folders" "folders_l10" {
- for_each = local.folders_l9_map
- parent_id = each.value.name
-}
-
-# Getting all projects parented by any of the folders in the tree including root prg/folder provided by `parent` variable.
-data "google_projects" "projects" {
- for_each = local.parent_ids
- filter = "parent.id:${each.value} ${var.filter}"
+ ignore_patterns = [for item in var.ignore_projects : "^${replace(item, "*", ".*")}$"]
+ ignore_regexp = length(local.ignore_patterns) > 0 ? join("|", local.ignore_patterns) : "^NO_PROJECTS_TO_IGNORE$"
+ projects_after_ignore = [for item in data.google_cloud_asset_resources_search_all.projects.results : item if(
+ length(concat(try(regexall(local.ignore_regexp, trimprefix(item.project, "projects/")), []), try(regexall(local.ignore_regexp, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")), []))) == 0
+ ) || contains(var.include_projects, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")) || contains(var.include_projects, trimprefix(item.project, "projects/"))
+ ]
+}
+
+data "google_cloud_asset_resources_search_all" "projects" {
+ provider = google-beta
+ scope = var.parent
+ asset_types = [
+ "cloudresourcemanager.googleapis.com/Project"
+ ]
+ query = local.query
}
diff --git a/modules/projects-data-source/outputs.tf b/modules/projects-data-source/outputs.tf
index b7e38ae2cf..b1710fa20b 100644
--- a/modules/projects-data-source/outputs.tf
+++ b/modules/projects-data-source/outputs.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,17 +14,12 @@
* limitations under the License.
*/
-output "folders" {
- description = "Map of folders attributes keyed by folder id."
- value = local.all_folders
-}
-
output "project_numbers" {
description = "List of project numbers."
- value = [for _, v in local.projects : v.number]
+ value = [for item in local.projects_after_ignore : trimprefix(item.project, "projects/")]
}
output "projects" {
- description = "Map of projects attributes keyed by projects id."
- value = local.projects
+ description = "List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format."
+ value = local.projects_after_ignore
}
diff --git a/modules/projects-data-source/variables.tf b/modules/projects-data-source/variables.tf
index a7f393d335..9fef35ab6c 100644
--- a/modules/projects-data-source/variables.tf
+++ b/modules/projects-data-source/variables.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -14,10 +14,42 @@
* limitations under the License.
*/
-variable "filter" {
- description = "A string filter as defined in the [REST API](https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#query-parameters)."
- type = string
- default = "lifecycleState:ACTIVE"
+variable "ignore_folders" {
+ description = "A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable."
+ type = list(string)
+ default = []
+ # example exlusing a folder
+ # ignore_folders = [
+ # "folders/0123456789",
+ # "2345678901"
+ # ]
+}
+
+variable "ignore_projects" {
+ description = "A list of project IDs, numbers or prefixes to exclude matching projects from the module output."
+ type = list(string)
+ default = []
+ # example
+ #ignore_projects = [
+ # "dev-proj-1",
+ # "uat-proj-2",
+ # "0123456789",
+ # "prd-proj-*"
+ #]
+}
+
+variable "include_projects" {
+ description = "A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries."
+ type = list(string)
+ default = []
+ # example excluding all the projects starting with "prf-" except "prd-123457"
+ #ignore_projects = [
+ # "prd-*"
+ #]
+ #include_projects = [
+ # "prd-123457",
+ # "0123456789"
+ #]
}
variable "parent" {
@@ -28,3 +60,9 @@ variable "parent" {
error_message = "Parent must be of the form folders/folder_id or organizations/organization_id."
}
}
+
+variable "query" {
+ description = "A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax)."
+ type = string
+ default = "state:ACTIVE"
+}
diff --git a/modules/projects-data-source/versions.tf b/modules/projects-data-source/versions.tf
index 08492c6f95..d9c1d37c73 100644
--- a/modules/projects-data-source/versions.tf
+++ b/modules/projects-data-source/versions.tf
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
diff --git a/modules/vpc-sc/README.md b/modules/vpc-sc/README.md
index 8e412bcfa6..7ad0cba53f 100644
--- a/modules/vpc-sc/README.md
+++ b/modules/vpc-sc/README.md
@@ -147,8 +147,8 @@ module "test" {
from = {
identities = [
"serviceAccount:test-tf@myproject.iam.gserviceaccount.com",
- ],
- source_access_levels = ["*"]
+ ]
+ access_levels = ["*"]
}
to = {
operations = [{ service_name = "*" }]
diff --git a/tests/blueprints/cloud_operations/terraform_enterprise_wif/__init__.py b/tests/blueprints/cloud_operations/terraform_cloud_dynamic_credentials/__init__.py
similarity index 100%
rename from tests/blueprints/cloud_operations/terraform_enterprise_wif/__init__.py
rename to tests/blueprints/cloud_operations/terraform_cloud_dynamic_credentials/__init__.py
diff --git a/tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/__init__.py b/tests/blueprints/cloud_operations/terraform_cloud_dynamic_credentials/gcp_workload_identity_provider/__init__.py
similarity index 100%
rename from tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/__init__.py
rename to tests/blueprints/cloud_operations/terraform_cloud_dynamic_credentials/gcp_workload_identity_provider/__init__.py
diff --git a/tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/fixture/main.tf b/tests/blueprints/cloud_operations/terraform_cloud_dynamic_credentials/gcp_workload_identity_provider/fixture/main.tf
similarity index 81%
rename from tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/fixture/main.tf
rename to tests/blueprints/cloud_operations/terraform_cloud_dynamic_credentials/gcp_workload_identity_provider/fixture/main.tf
index 3552740c2a..800201f011 100644
--- a/tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/fixture/main.tf
+++ b/tests/blueprints/cloud_operations/terraform_cloud_dynamic_credentials/gcp_workload_identity_provider/fixture/main.tf
@@ -1,5 +1,5 @@
/**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,13 +15,13 @@
*/
module "test" {
- source = "../../../../../../blueprints/cloud-operations/terraform-enterprise-wif/gcp-workload-identity-provider"
+ source = "../../../../../../blueprints/cloud-operations/terraform-cloud-dynamic-credentials/gcp-workload-identity-provider"
billing_account = var.billing_account
project_create = var.project_create
project_id = var.project_id
parent = var.parent
- tfe_organization_id = var.tfe_organization_id
- tfe_workspace_id = var.tfe_workspace_id
+ tfc_organization_id = var.tfc_organization_id
+ tfc_workspace_id = var.tfc_workspace_id
workload_identity_pool_id = var.workload_identity_pool_id
workload_identity_pool_provider_id = var.workload_identity_pool_provider_id
issuer_uri = var.issuer_uri
diff --git a/tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/fixture/variables.tf b/tests/blueprints/cloud_operations/terraform_cloud_dynamic_credentials/gcp_workload_identity_provider/fixture/variables.tf
similarity index 83%
rename from tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/fixture/variables.tf
rename to tests/blueprints/cloud_operations/terraform_cloud_dynamic_credentials/gcp_workload_identity_provider/fixture/variables.tf
index d99981c0cf..8d7c27197a 100644
--- a/tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/fixture/variables.tf
+++ b/tests/blueprints/cloud_operations/terraform_cloud_dynamic_credentials/gcp_workload_identity_provider/fixture/variables.tf
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -37,14 +37,14 @@ variable "parent" {
}
}
-variable "tfe_organization_id" {
- description = "TFE organization id."
+variable "tfc_organization_id" {
+ description = "TFC organization id."
type = string
default = "org-123"
}
-variable "tfe_workspace_id" {
- description = "TFE workspace id."
+variable "tfc_workspace_id" {
+ description = "TFC workspace id."
type = string
default = "ws-123"
}
@@ -52,17 +52,17 @@ variable "tfe_workspace_id" {
variable "workload_identity_pool_id" {
description = "Workload identity pool id."
type = string
- default = "tfe-pool"
+ default = "tfc-pool"
}
variable "workload_identity_pool_provider_id" {
description = "Workload identity pool provider id."
type = string
- default = "tfe-provider"
+ default = "tfc-provider"
}
variable "issuer_uri" {
- description = "Terraform Enterprise uri. Replace the uri if a self hosted instance is used."
+ description = "Terraform Cloud uri. Replace the uri if a self hosted instance is used."
type = string
default = "https://app.terraform.io/"
}
diff --git a/tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/test_plan.py b/tests/blueprints/cloud_operations/terraform_cloud_dynamic_credentials/gcp_workload_identity_provider/test_plan.py
similarity index 100%
rename from tests/blueprints/cloud_operations/terraform_enterprise_wif/gcp_workload_identity_provider/test_plan.py
rename to tests/blueprints/cloud_operations/terraform_cloud_dynamic_credentials/gcp_workload_identity_provider/test_plan.py
diff --git a/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf b/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf
index 52317d6f5f..5acb29e835 100644
--- a/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf
+++ b/tests/blueprints/data_solutions/data_platform_foundations/fixture/main.tf
@@ -17,7 +17,9 @@
module "test" {
source = "../../../../../blueprints/data-solutions/data-platform-foundations/"
organization_domain = "example.com"
- billing_account_id = "123456-123456-123456"
- folder_id = "folders/12345678"
- prefix = "prefix"
+ project_config = {
+ billing_account_id = "123456-123456-123456"
+ parent = "folders/12345678"
+ }
+ prefix = "prefix"
}
diff --git a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py
index 785f470537..630944f26d 100644
--- a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py
+++ b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py
@@ -23,4 +23,4 @@ def test_resources(e2e_plan_runner):
modules, resources = e2e_plan_runner(FIXTURES_DIR)
assert len(modules) == 42
- assert len(resources) == 296
+ assert len(resources) == 277
diff --git a/tests/blueprints/factories/project_factory/fixture/projects/project.yaml b/tests/blueprints/factories/project_factory/fixture/projects/project.yaml
index a158198484..b8d6e6639b 100644
--- a/tests/blueprints/factories/project_factory/fixture/projects/project.yaml
+++ b/tests/blueprints/factories/project_factory/fixture/projects/project.yaml
@@ -48,15 +48,14 @@ labels:
# [opt] Org policy overrides defined at project level
org_policies:
- policy_boolean:
- constraints/compute.disableGuestAttributesAccess: true
- policy_list:
- constraints/compute.trustedImageProjects:
- inherit_from_parent: null
- status: true
- suggested_value: null
- values:
- - projects/fast-prod-iac-core-0
+ compute.disableGuestAttributesAccess:
+ rules:
+ - enforce: true
+ compute.trustedImageProjects:
+ rules:
+ - allow:
+ values:
+ - projects/fast-prod-iac-core-0
# [opt] Prefix - overrides default if set
prefix: test1
diff --git a/tests/fast/stages/s3_project_factory/data/projects/project.yaml b/tests/fast/stages/s3_project_factory/data/projects/project.yaml
index d988d9d50c..90354a2ac7 100644
--- a/tests/fast/stages/s3_project_factory/data/projects/project.yaml
+++ b/tests/fast/stages/s3_project_factory/data/projects/project.yaml
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -60,15 +60,14 @@ labels:
# [opt] Org policy overrides defined at project level
org_policies:
- policy_boolean:
- constraints/compute.disableGuestAttributesAccess: true
- policy_list:
- constraints/compute.trustedImageProjects:
- inherit_from_parent: null
- status: true
- suggested_value: null
- values:
- - projects/fast-prod-iac-core-0
+ compute.disableGuestAttributesAccess:
+ rules:
+ - enforce: true
+ compute.trustedImageProjects:
+ rules:
+ - allow:
+ values:
+ - projects/fast-prod-iac-core-0
# [opt] Service account to create for the project and their roles on the project
# in name => [roles] format
diff --git a/tests/modules/folder/examples/org-policies.yaml b/tests/modules/folder/examples/org-policies.yaml
index f8bf41879a..c7bee1239e 100644
--- a/tests/modules/folder/examples/org-policies.yaml
+++ b/tests/modules/folder/examples/org-policies.yaml
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -26,7 +26,7 @@ values:
deny_all: null
enforce: 'TRUE'
values: []
- module.folder.google_org_policy_policy.default["constraints/compute.skipDefaultNetworkCreation"]:
+ module.folder.google_org_policy_policy.default["compute.skipDefaultNetworkCreation"]:
spec:
- inherit_from_parent: null
reset: null
@@ -36,7 +36,7 @@ values:
deny_all: null
enforce: 'TRUE'
values: []
- module.folder.google_org_policy_policy.default["constraints/compute.trustedImageProjects"]:
+ module.folder.google_org_policy_policy.default["compute.trustedImageProjects"]:
spec:
- inherit_from_parent: null
reset: null
@@ -49,7 +49,7 @@ values:
- allowed_values:
- projects/my-project
denied_values: null
- module.folder.google_org_policy_policy.default["constraints/compute.vmExternalIpAccess"]:
+ module.folder.google_org_policy_policy.default["compute.vmExternalIpAccess"]:
spec:
- inherit_from_parent: null
reset: null
@@ -59,7 +59,7 @@ values:
deny_all: 'TRUE'
enforce: null
values: []
- module.folder.google_org_policy_policy.default["constraints/iam.allowedPolicyMemberDomains"]:
+ module.folder.google_org_policy_policy.default["iam.allowedPolicyMemberDomains"]:
spec:
- inherit_from_parent: null
reset: null
@@ -91,7 +91,7 @@ values:
- allow_all: null
condition:
- description: test condition
- expression: resource.matchTagId("tagKeys/1234", "tagValues/1234")
+ expression: resource.matchTagId('tagKeys/1234', 'tagValues/1234')
location: somewhere
title: condition
deny_all: null
diff --git a/tests/modules/folder/org_policies_boolean.tfvars b/tests/modules/folder/org_policies_boolean.tfvars
index eceafe6d29..cf5047a205 100644
--- a/tests/modules/folder/org_policies_boolean.tfvars
+++ b/tests/modules/folder/org_policies_boolean.tfvars
@@ -1,9 +1,8 @@
org_policies = {
"iam.disableServiceAccountKeyCreation" = {
- enforce = true
+ rules = [{ enforce = true }]
}
"iam.disableServiceAccountKeyUpload" = {
- enforce = false
rules = [
{
condition = {
@@ -13,6 +12,9 @@ org_policies = {
location = "xxx"
}
enforce = true
+ },
+ {
+ enforce = false
}
]
}
diff --git a/tests/modules/folder/org_policies_list.tfvars b/tests/modules/folder/org_policies_list.tfvars
index 7380717336..2c83de47e3 100644
--- a/tests/modules/folder/org_policies_list.tfvars
+++ b/tests/modules/folder/org_policies_list.tfvars
@@ -1,14 +1,15 @@
org_policies = {
"compute.vmExternalIpAccess" = {
- deny = { all = true }
+ rules = [{ deny = { all = true } }]
}
"iam.allowedPolicyMemberDomains" = {
- allow = {
- values = ["C0xxxxxxx", "C0yyyyyyy"]
- }
+ rules = [{
+ allow = {
+ values = ["C0xxxxxxx", "C0yyyyyyy"]
+ }
+ }]
}
"compute.restrictLoadBalancerCreationForTypes" = {
- deny = { values = ["in:EXTERNAL"] }
rules = [
{
condition = {
@@ -31,6 +32,9 @@ org_policies = {
allow = {
all = true
}
+ },
+ {
+ deny = { values = ["in:EXTERNAL"] }
}
]
}
diff --git a/tests/modules/net_vpc/examples/factory.yaml b/tests/modules/net_vpc/examples/factory.yaml
index 48671c2922..0724b5970e 100644
--- a/tests/modules/net_vpc/examples/factory.yaml
+++ b/tests/modules/net_vpc/examples/factory.yaml
@@ -44,7 +44,18 @@ values:
region: europe-west4
role: null
secondary_ip_range: []
+ module.vpc.google_compute_subnetwork_iam_binding.binding["europe-west1/subnet-detailed.roles/compute.networkUser"]:
+ condition: []
+ members:
+ - group:lorem@example.com
+ - serviceAccount:fbz@prj.iam.gserviceaccount.com
+ - user:foobar@example.com
+ project: my-project
+ region: europe-west1
+ role: roles/compute.networkUser
+ subnetwork: subnet-detailed
counts:
google_compute_network: 1
google_compute_subnetwork: 2
+ google_compute_subnetwork_iam_binding: 1
diff --git a/tests/modules/organization/examples/basic.yaml b/tests/modules/organization/examples/basic.yaml
index f7b63a1d41..9960a71297 100644
--- a/tests/modules/organization/examples/basic.yaml
+++ b/tests/modules/organization/examples/basic.yaml
@@ -25,8 +25,8 @@ values:
deny_all: null
enforce: 'TRUE'
values: []
- module.org.google_org_policy_policy.default["constraints/compute.skipDefaultNetworkCreation"]:
- name: organizations/1234567890/policies/constraints/compute.skipDefaultNetworkCreation
+ module.org.google_org_policy_policy.default["compute.skipDefaultNetworkCreation"]:
+ name: organizations/1234567890/policies/compute.skipDefaultNetworkCreation
parent: organizations/1234567890
spec:
- inherit_from_parent: null
@@ -37,8 +37,8 @@ values:
deny_all: null
enforce: 'TRUE'
values: []
- module.org.google_org_policy_policy.default["constraints/compute.trustedImageProjects"]:
- name: organizations/1234567890/policies/constraints/compute.trustedImageProjects
+ module.org.google_org_policy_policy.default["compute.trustedImageProjects"]:
+ name: organizations/1234567890/policies/compute.trustedImageProjects
parent: organizations/1234567890
spec:
- inherit_from_parent: null
@@ -52,8 +52,8 @@ values:
- allowed_values:
- projects/my-project
denied_values: null
- module.org.google_org_policy_policy.default["constraints/compute.vmExternalIpAccess"]:
- name: organizations/1234567890/policies/constraints/compute.vmExternalIpAccess
+ module.org.google_org_policy_policy.default["compute.vmExternalIpAccess"]:
+ name: organizations/1234567890/policies/compute.vmExternalIpAccess
parent: organizations/1234567890
spec:
- inherit_from_parent: null
@@ -64,15 +64,30 @@ values:
deny_all: 'TRUE'
enforce: null
values: []
- module.org.google_org_policy_policy.default["constraints/iam.allowedPolicyMemberDomains"]:
- name: organizations/1234567890/policies/constraints/iam.allowedPolicyMemberDomains
+ module.org.google_org_policy_policy.default["iam.allowedPolicyMemberDomains"]:
+ name: organizations/1234567890/policies/iam.allowedPolicyMemberDomains
parent: organizations/1234567890
spec:
- inherit_from_parent: null
reset: null
rules:
+ - allow_all: 'TRUE'
+ condition:
+ - description: Allow external identities when resource has the `allowexternal`
+ tag set to true.
+ expression: resource.matchTag('1234567890/allowexternal', 'true')
+ location: null
+ title: Allow external identities
+ deny_all: null
+ enforce: null
+ values: []
- allow_all: null
- condition: []
+ condition:
+ - description: For any resource without allowexternal=true, only allow identities
+ from restricted domains.
+ expression: '!resource.matchTag(''1234567890/allowexternal'', ''true'')'
+ location: null
+ title: ''
deny_all: null
enforce: null
values:
@@ -102,7 +117,7 @@ values:
- allow_all: null
condition:
- description: test condition
- expression: resource.matchTagId("tagKeys/1234", "tagValues/1234")
+ expression: resource.matchTagId('tagKeys/1234', 'tagValues/1234')
location: somewhere
title: condition
deny_all: null
@@ -141,6 +156,20 @@ values:
member: user:compute@example.org
org_id: '1234567890'
role: roles/container.viewer
+ module.org.google_tags_tag_key.default["allowexternal"]:
+ description: Allow external identities.
+ parent: organizations/1234567890
+ purpose: null
+ purpose_data: null
+ short_name: allowexternal
+ module.org.google_tags_tag_value.default["allowexternal/false"]:
+ short_name: 'false'
+ module.org.google_tags_tag_value.default["allowexternal/true"]:
+ short_name: 'true'
+
counts:
google_org_policy_policy: 8
google_organization_iam_binding: 3
+ google_organization_iam_member: 2
+ google_tags_tag_key: 1
+ google_tags_tag_value: 2
diff --git a/tests/modules/organization/org_policies_boolean.tfvars b/tests/modules/organization/org_policies_boolean.tfvars
index eceafe6d29..cf5047a205 100644
--- a/tests/modules/organization/org_policies_boolean.tfvars
+++ b/tests/modules/organization/org_policies_boolean.tfvars
@@ -1,9 +1,8 @@
org_policies = {
"iam.disableServiceAccountKeyCreation" = {
- enforce = true
+ rules = [{ enforce = true }]
}
"iam.disableServiceAccountKeyUpload" = {
- enforce = false
rules = [
{
condition = {
@@ -13,6 +12,9 @@ org_policies = {
location = "xxx"
}
enforce = true
+ },
+ {
+ enforce = false
}
]
}
diff --git a/tests/modules/organization/org_policies_list.tfvars b/tests/modules/organization/org_policies_list.tfvars
index f9de8dbabe..d03f8530f1 100644
--- a/tests/modules/organization/org_policies_list.tfvars
+++ b/tests/modules/organization/org_policies_list.tfvars
@@ -1,15 +1,17 @@
org_policies = {
"compute.vmExternalIpAccess" = {
- deny = { all = true }
+ rules = [{ deny = { all = true } }]
}
"iam.allowedPolicyMemberDomains" = {
inherit_from_parent = true
- allow = {
- values = ["C0xxxxxxx", "C0yyyyyyy"]
- }
+ rules = [{
+ allow = {
+ values = ["C0xxxxxxx", "C0yyyyyyy"]
+ }
+ }]
+
}
"compute.restrictLoadBalancerCreationForTypes" = {
- deny = { values = ["in:EXTERNAL"] }
rules = [
{
condition = {
@@ -32,6 +34,9 @@ org_policies = {
allow = {
all = true
}
+ },
+ {
+ deny = { values = ["in:EXTERNAL"] }
}
]
}
diff --git a/tests/modules/organization/test_plan_org_policies_modules.py b/tests/modules/organization/test_plan_org_policies_modules.py
index 1d19ee1eb6..30881d993e 100644
--- a/tests/modules/organization/test_plan_org_policies_modules.py
+++ b/tests/modules/organization/test_plan_org_policies_modules.py
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -26,61 +26,35 @@ def test_policy_implementation():
path = modules_path / module / 'organization-policies.tf'
lines[module] = path.open().readlines()
- diff1 = difflib.unified_diff(lines['project'], lines['folder'])
+ diff1 = difflib.unified_diff(lines['project'], lines['folder'], 'project',
+ 'folder', n=0)
assert list(diff1) == [
- '--- \n',
- '+++ \n',
- '@@ -14,7 +14,7 @@\n',
- ' * limitations under the License.\n',
- ' */\n',
- ' \n',
+ '--- project\n',
+ '+++ folder\n',
+ '@@ -17 +17 @@\n',
'-# tfdoc:file:description Project-level organization policies.\n',
'+# tfdoc:file:description Folder-level organization policies.\n',
- ' \n',
- ' locals {\n',
- ' _factory_data_raw = merge([\n',
- '@@ -65,8 +65,8 @@\n',
- ' org_policies = {\n',
- ' for k, v in local._org_policies :\n',
- ' k => merge(v, {\n',
+ '@@ -58,2 +58,2 @@\n',
'- name = "projects/${local.project.project_id}/policies/${k}"\n',
'- parent = "projects/${local.project.project_id}"\n',
'+ name = "${local.folder.name}/policies/${k}"\n',
'+ parent = local.folder.name\n',
- ' \n',
- ' is_boolean_policy = v.allow == null && v.deny == null\n',
- ' has_values = (\n',
]
- diff2 = difflib.unified_diff(lines['folder'], lines['organization'])
+ diff2 = difflib.unified_diff(lines['folder'], lines['organization'], 'folder',
+ 'organization', n=0)
assert list(diff2) == [
- '--- \n',
- '+++ \n',
- '@@ -14,7 +14,7 @@\n',
- ' * limitations under the License.\n',
- ' */\n',
- ' \n',
+ '--- folder\n',
+ '+++ organization\n',
+ '@@ -17 +17 @@\n',
'-# tfdoc:file:description Folder-level organization policies.\n',
'+# tfdoc:file:description Organization-level organization policies.\n',
- ' \n',
- ' locals {\n',
- ' _factory_data_raw = merge([\n',
- '@@ -65,8 +65,8 @@\n',
- ' org_policies = {\n',
- ' for k, v in local._org_policies :\n',
- ' k => merge(v, {\n',
+ '@@ -58,2 +58,2 @@\n',
'- name = "${local.folder.name}/policies/${k}"\n',
'- parent = local.folder.name\n',
'+ name = "${var.organization_id}/policies/${k}"\n',
'+ parent = var.organization_id\n',
- ' \n',
- ' is_boolean_policy = v.allow == null && v.deny == null\n',
- ' has_values = (\n',
- '@@ -139,4 +139,13 @@\n',
- ' }\n',
- ' }\n',
- ' }\n',
- '+\n',
+ '@@ -116,0 +117,8 @@\n',
'+ depends_on = [\n',
'+ google_organization_iam_audit_config.config,\n',
'+ google_organization_iam_binding.authoritative,\n',
@@ -89,5 +63,4 @@ def test_policy_implementation():
'+ google_organization_iam_policy.authoritative,\n',
'+ google_org_policy_custom_constraint.constraint,\n',
'+ ]\n',
- ' }\n',
]
diff --git a/tests/modules/project/examples/iam-additive-members.yaml b/tests/modules/project/examples/iam-additive-members.yaml
index 5832e4dcaa..6a517a4a1d 100644
--- a/tests/modules/project/examples/iam-additive-members.yaml
+++ b/tests/modules/project/examples/iam-additive-members.yaml
@@ -17,17 +17,14 @@ values:
project_id: project-example
module.project.google_project_iam_member.additive["roles/editor-user:two@example.org"]:
condition: []
- member: user:two@example.org
project: project-example
role: roles/editor
module.project.google_project_iam_member.additive["roles/owner-user:one@example.org"]:
condition: []
- member: user:one@example.org
project: project-example
role: roles/owner
module.project.google_project_iam_member.additive["roles/owner-user:two@example.org"]:
condition: []
- member: user:two@example.org
project: project-example
role: roles/owner
diff --git a/tests/modules/project/examples/iam-additive.yaml b/tests/modules/project/examples/iam-additive.yaml
index f07b0df66e..5bab822321 100644
--- a/tests/modules/project/examples/iam-additive.yaml
+++ b/tests/modules/project/examples/iam-additive.yaml
@@ -16,22 +16,18 @@ values:
module.project.google_project.project[0]: {}
module.project.google_project_iam_member.additive["roles/owner-group:three@example.org"]:
condition: []
- member: group:three@example.org
project: project-example
role: roles/owner
module.project.google_project_iam_member.additive["roles/storage.objectAdmin-group:two@example.org"]:
condition: []
- member: group:two@example.org
project: project-example
role: roles/storage.objectAdmin
module.project.google_project_iam_member.additive["roles/viewer-group:one@example.org"]:
condition: []
- member: group:one@example.org
project: project-example
role: roles/viewer
module.project.google_project_iam_member.additive["roles/viewer-group:two@xample.org"]:
condition: []
- member: group:two@xample.org
project: project-example
role: roles/viewer
diff --git a/tests/modules/project/examples/org-policies.yaml b/tests/modules/project/examples/org-policies.yaml
index 8841dedee8..d4dddc75bf 100644
--- a/tests/modules/project/examples/org-policies.yaml
+++ b/tests/modules/project/examples/org-policies.yaml
@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -25,8 +25,8 @@ values:
deny_all: null
enforce: 'TRUE'
values: []
- module.project.google_org_policy_policy.default["constraints/compute.skipDefaultNetworkCreation"]:
- name: projects/foo-project-example/policies/constraints/compute.skipDefaultNetworkCreation
+ module.project.google_org_policy_policy.default["compute.skipDefaultNetworkCreation"]:
+ name: projects/foo-project-example/policies/compute.skipDefaultNetworkCreation
parent: projects/foo-project-example
spec:
- inherit_from_parent: null
@@ -37,8 +37,8 @@ values:
deny_all: null
enforce: 'TRUE'
values: []
- module.project.google_org_policy_policy.default["constraints/compute.trustedImageProjects"]:
- name: projects/foo-project-example/policies/constraints/compute.trustedImageProjects
+ module.project.google_org_policy_policy.default["compute.trustedImageProjects"]:
+ name: projects/foo-project-example/policies/compute.trustedImageProjects
parent: projects/foo-project-example
spec:
- inherit_from_parent: null
@@ -52,8 +52,8 @@ values:
- allowed_values:
- projects/my-project
denied_values: null
- module.project.google_org_policy_policy.default["constraints/compute.vmExternalIpAccess"]:
- name: projects/foo-project-example/policies/constraints/compute.vmExternalIpAccess
+ module.project.google_org_policy_policy.default["compute.vmExternalIpAccess"]:
+ name: projects/foo-project-example/policies/compute.vmExternalIpAccess
parent: projects/foo-project-example
spec:
- inherit_from_parent: null
@@ -64,8 +64,8 @@ values:
deny_all: 'TRUE'
enforce: null
values: []
- module.project.google_org_policy_policy.default["constraints/iam.allowedPolicyMemberDomains"]:
- name: projects/foo-project-example/policies/constraints/iam.allowedPolicyMemberDomains
+ module.project.google_org_policy_policy.default["iam.allowedPolicyMemberDomains"]:
+ name: projects/foo-project-example/policies/iam.allowedPolicyMemberDomains
parent: projects/foo-project-example
spec:
- inherit_from_parent: null
@@ -102,7 +102,7 @@ values:
- allow_all: null
condition:
- description: test condition
- expression: resource.matchTagId("tagKeys/1234", "tagValues/1234")
+ expression: resource.matchTagId('tagKeys/1234', 'tagValues/1234')
location: somewhere
title: condition
deny_all: null
diff --git a/tests/modules/project/org_policies_boolean.tfvars b/tests/modules/project/org_policies_boolean.tfvars
index eceafe6d29..cf5047a205 100644
--- a/tests/modules/project/org_policies_boolean.tfvars
+++ b/tests/modules/project/org_policies_boolean.tfvars
@@ -1,9 +1,8 @@
org_policies = {
"iam.disableServiceAccountKeyCreation" = {
- enforce = true
+ rules = [{ enforce = true }]
}
"iam.disableServiceAccountKeyUpload" = {
- enforce = false
rules = [
{
condition = {
@@ -13,6 +12,9 @@ org_policies = {
location = "xxx"
}
enforce = true
+ },
+ {
+ enforce = false
}
]
}
diff --git a/tests/modules/project/org_policies_list.tfvars b/tests/modules/project/org_policies_list.tfvars
index f9de8dbabe..4889547d2a 100644
--- a/tests/modules/project/org_policies_list.tfvars
+++ b/tests/modules/project/org_policies_list.tfvars
@@ -1,15 +1,16 @@
org_policies = {
"compute.vmExternalIpAccess" = {
- deny = { all = true }
+ rules = [{ deny = { all = true } }]
}
"iam.allowedPolicyMemberDomains" = {
inherit_from_parent = true
- allow = {
- values = ["C0xxxxxxx", "C0yyyyyyy"]
- }
+ rules = [{
+ allow = {
+ values = ["C0xxxxxxx", "C0yyyyyyy"]
+ }
+ }]
}
"compute.restrictLoadBalancerCreationForTypes" = {
- deny = { values = ["in:EXTERNAL"] }
rules = [
{
condition = {
@@ -32,6 +33,9 @@ org_policies = {
allow = {
all = true
}
+ },
+ {
+ deny = { values = ["in:EXTERNAL"] }
}
]
}