diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 09fdc4d6b9..939f76a9ec 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -34,7 +34,7 @@ jobs: - name: Set up Terraform uses: hashicorp/setup-terraform@v2 with: - terraform_version: 1.7.0 + terraform_version: 1.7.4 - name: Install dependencies run: | diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3c83a8b1a2..8adef331ca 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,7 +25,7 @@ env: PYTEST_ADDOPTS: "--color=yes" PYTHON_VERSION: "3.10" TF_PLUGIN_CACHE_DIR: "/home/runner/.terraform.d/plugin-cache" - TF_VERSION: 1.7.0 + TF_VERSION: 1.7.4 TFTEST_COPY: 1 jobs: diff --git a/CHANGELOG.md b/CHANGELOG.md index f3157936cd..3fb9351ad1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1653,7 +1653,7 @@ All notable changes to this project will be documented in this file. - add support for VPC-SC perimeters in Data Foundation end to end example - fix `vpc-sc` module - new networking example showing how to use [Private Service Connect to call a Cloud Function from on-premises](./blueprints/networking/private-cloud-function-from-onprem/) -- new networking example showing how to organize [decentralized firewall](./blueprints/networking/decentralized-firewall/) management on GCP +- new networking example showing how to organize decentralized firewall management on GCP ## [5.0.0] - 2021-06-17 diff --git a/README.md b/README.md index a1d91a2d2b..aeb8d026fa 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ The current list of modules supports most of the core foundational and networkin Currently available modules: - **foundational** - [billing account](./modules/billing-account), [Cloud Identity group](./modules/cloud-identity-group/), [folder](./modules/folder), [service accounts](./modules/iam-service-account), [logging bucket](./modules/logging-bucket), [organization](./modules/organization), [project](./modules/project), [projects-data-source](./modules/projects-data-source) +- **process factories** - [project factory](./modules/project-factory/README.md) - **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [VLAN Attachment](./modules/net-vlan-attachment/), [External Application LB](./modules/net-lb-app-ext/), [External Passthrough Network LB](./modules/net-lb-ext), [External Regional Application Load Balancer](./modules/net-lb-app-ext-regional/), [Firewall policy](./modules/net-firewall-policy), [Internal Application LB](./modules/net-lb-app-int), [Cross-region Internal Application LB](./modules/net-lb-app-int-cross-region), [Internal Passthrough Network LB](./modules/net-lb-int), [Internal Proxy Network LB](./modules/net-lb-proxy-int), [IPSec over Interconnect](./modules/net-ipsec-over-interconnect), [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory), [Secure Web Proxy](./modules/net-swp) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid), [GKE cluster](./modules/gke-cluster-standard), [GKE hub](./modules/gke-hub), [GKE nodepool](./modules/gke-nodepool), [GCVE private cloud](./modules/gcve-private-cloud) - **data** - [Analytics Hub](./modules/analytics-hub), [BigQuery dataset](./modules/bigquery-dataset), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex DataScan](./modules/dataplex-datascan/), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Data Catalog Tag](./modules/data-catalog-tag), [Data Catalog Tag Template](./modules/data-catalog-tag-template), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub), [Dataform Repository](./modules/dataform-repository/) diff --git a/blueprints/README.md b/blueprints/README.md index 5d5a5d02a1..8a995facad 100644 --- a/blueprints/README.md +++ b/blueprints/README.md @@ -7,9 +7,9 @@ Currently available blueprints: - **apigee** - [Apigee Hybrid on GKE](./apigee/hybrid-gke/), [Apigee X analytics in BigQuery](./apigee/bigquery-analytics), [Apigee network patterns](./apigee/network-patterns/) - **cloud operations** - [Active Directory Federation Services](./cloud-operations/adfs), [Cloud Asset Inventory feeds for resource change tracking and remediation](./cloud-operations/asset-inventory-feed-remediation), [Fine-grained Cloud DNS IAM via Service Directory](./cloud-operations/dns-fine-grained-iam), [Cloud DNS & Shared VPC design](./cloud-operations/dns-shared-vpc), [Delegated Role Grants](./cloud-operations/iam-delegated-role-grants), [Network Quota Monitoring](./cloud-operations/network-quota-monitoring), [Managing on-prem service account keys by uploading public keys](./cloud-operations/onprem-sa-key-management), [Compute Image builder with Hashicorp Packer](./cloud-operations/packer-image-builder), [Packer example](./cloud-operations/packer-image-builder/packer), [Compute Engine quota monitoring](./cloud-operations/compute-quota-monitoring), [Scheduled Cloud Asset Inventory Export to Bigquery](./cloud-operations/scheduled-asset-inventory-export-bq), [Configuring workload identity federation with Terraform Cloud/Enterprise workflows](./cloud-operations/terraform-cloud-dynamic-credentials), [TCP healthcheck and restart for unmanaged GCE instances](./cloud-operations/unmanaged-instances-healthcheck), [Migrate for Compute Engine (v5) blueprints](./cloud-operations/vm-migration), [Configuring workload identity federation to access Google Cloud resources from apps running on Azure](./cloud-operations/workload-identity-federation) - **data solutions** - [GCE and GCS CMEK via centralized Cloud KMS](./data-solutions/cmek-via-centralized-kms), [Cloud Composer version 2 private instance, supporting Shared VPC and external CMEK key](./data-solutions/composer-2), [Cloud SQL instance with multi-region read replicas](./data-solutions/cloudsql-multiregion), [Data Platform](./data-solutions/data-platform-foundations), [Minimal Data Platform](./data-solutions/data-platform-minimal), [Spinning up a foundation data pipeline on Google Cloud using Cloud Storage, Dataflow and BigQuery](./data-solutions/gcs-to-bq-with-least-privileges), [#SQL Server Always On Groups blueprint](./data-solutions/sqlserver-alwayson), [Data Playground](./data-solutions/data-playground), [MLOps with Vertex AI](./data-solutions/vertex-mlops), [Shielded Folder](./data-solutions/shielded-folder), [BigQuery ML and Vertex AI Pipeline](./data-solutions/bq-ml) -- **factories** - [The why and the how of Resource Factories](./factories), [Google Cloud Identity Group Factory](./factories/cloud-identity-group-factory), [Google Cloud BQ Factory](./factories/bigquery-factory), [Google Cloud VPC Firewall Factory](./factories/net-vpc-firewall-yaml), [Minimal Project Factory](./factories/project-factory) +- **factories** - [Fabric resource factories](./factories) - **GKE** - [Binary Authorization Pipeline Blueprint](./gke/binauthz), [Storage API](./gke/binauthz/image), [Multi-cluster mesh on GKE (fleet API)](./gke/multi-cluster-mesh-gke-fleet-api), [GKE Multitenant Blueprint](./gke/multitenant-fleet), [Shared VPC with GKE support](./networking/shared-vpc-gke/), [GKE Autopilot](./gke/autopilot) -- **networking** - [Calling a private Cloud Function from On-premises](./networking/private-cloud-function-from-onprem), [Decentralized firewall management](./networking/decentralized-firewall), [Decentralized firewall validator](./networking/decentralized-firewall/validator), [HA VPN over Interconnect](./networking/ha-vpn-over-interconnect/), [GLB and multi-regional daisy-chaining through hybrid NEGs](./networking/glb-hybrid-neg-internal), [Hybrid connectivity to on-premise services through PSC](./networking/psc-hybrid), [HTTP Load Balancer with Cloud Armor](./networking/glb-and-armor), [Hub and Spoke via VPN](./networking/hub-and-spoke-vpn), [Hub and Spoke via VPC Peering](./networking/hub-and-spoke-peering), [Internal Load Balancer as Next Hop](./networking/ilb-next-hop), On-prem DNS and Google Private Access, [PSC Producer](./networking/psc-hybrid/psc-producer), [PSC Consumer](./networking/psc-hybrid/psc-consumer), [Shared VPC with optional GKE cluster](./networking/shared-vpc-gke), [VPC Connectivity Lab](./networking/vpc-connectivity-lab/) +- **networking** - [Calling a private Cloud Function from On-premises](./networking/private-cloud-function-from-onprem), [HA VPN over Interconnect](./networking/ha-vpn-over-interconnect/), [GLB and multi-regional daisy-chaining through hybrid NEGs](./networking/glb-hybrid-neg-internal), [Hybrid connectivity to on-premise services through PSC](./networking/psc-hybrid), [HTTP Load Balancer with Cloud Armor](./networking/glb-and-armor), [Hub and Spoke via VPN](./networking/hub-and-spoke-vpn), [Hub and Spoke via VPC Peering](./networking/hub-and-spoke-peering), [Internal Load Balancer as Next Hop](./networking/ilb-next-hop), On-prem DNS and Google Private Access, [PSC Producer](./networking/psc-hybrid/psc-producer), [PSC Consumer](./networking/psc-hybrid/psc-consumer), [Shared VPC with optional GKE cluster](./networking/shared-vpc-gke), [VPC Connectivity Lab](./networking/vpc-connectivity-lab/) - **serverless** - [Cloud Run series](./serverless/cloud-run-explore) - **third party solutions** - [OpenShift on GCP user-provisioned infrastructure](./third-party-solutions/openshift), [Wordpress deployment on Cloud Run](./third-party-solutions/wordpress/cloudrun) diff --git a/blueprints/data-solutions/shielded-folder/main.tf b/blueprints/data-solutions/shielded-folder/main.tf index 4524999adb..ba111d2c79 100644 --- a/blueprints/data-solutions/shielded-folder/main.tf +++ b/blueprints/data-solutions/shielded-folder/main.tf @@ -94,7 +94,7 @@ module "firewall-policy" { source = "../../../modules/net-firewall-policy" name = "default" parent_id = module.folder.id - rules_factory_config = var.data_dir == null ? {} : { + factories_config = var.data_dir == null ? {} : { cidr_file_path = "${var.data_dir}/firewall-policies/cidrs.yaml" ingress_rules_file_path = "${var.data_dir}/firewall-policies/hierarchical-ingress-rules.yaml" } diff --git a/blueprints/factories/README.md b/blueprints/factories/README.md index d2eeb0b2b4..6579786500 100644 --- a/blueprints/factories/README.md +++ b/blueprints/factories/README.md @@ -1,44 +1,79 @@ -# The why and the how of Resource Factories +# Resource Factories -Terraform modules can be designed - where it makes sense - to implement a resource factory, which is a configuration-driven approach to resource creation meant to: +This README explains the rationale and high level approach for resource factories, a pattern that is widely used in this repository across modules and in the FAST framework. It also collects pointers to all the different factories implemented in modules to simplify discovery. -- accelerate and rationalize the repetitive creation of common resources, such as firewall rules and subnets -- enable teams without Terraform specific knowledge to leverage IaC via human-friendly and machine-parseable YAML files -- make it simple to implement specific requirements and best practices (e.g. "always enable PGA for GCP subnets", or "only allow using regions `europe-west1` and `europe-west3`") -- codify and centralise business logics and policies (e.g. labels and naming conventions) -- allow to easily parse and understand sets of specific resources, for documentation purposes + +- [The why](#the-why) +- [The how](#the-how) +- [Factory implementations](#factory-implementations) + - [Module-level factory interfaces](#module-level-factory-interfaces) + - [Standalone factories](#standalone-factories) + -Generally speaking, the configurations for a resource factory consists in one or more YaML files, optionally grouped in folders, that describe resources following a well defined, validable schema, such as in the example below for the subnet factory of the [`net-vpc`](../../modules/net-vpc) module, which allows for the massive creation of subnets for a given VPC. +## The why -```yaml -region: europe-west3 -ip_cidr_range: 10.0.0.0/24 -description: Sample Subnet in project project-prod-a, vpc-alpha -secondary_ip_ranges: - secondary-range-a: 192.168.0.0/24 - secondary-range-b: 192.168.1.0/24 -``` +Managing large sets of uniform resources with Terraform usually involves different teams collaborating on the same codebase, complex authorization processes and checks managed via CI/CD approvals, or even integrating with external systems that manage digital workflows. -Terraform natively supports YaML, JSON and CSV parsing - however Fabric has decided to embrace YaML for the following reasons: +Factories are a way to simplify all above use cases, by moving repetitive resource definitions out of the Terraform codebase and into sets of files that leverage different formats. -- YaML is easier to parse for a human, and allows for comments and nested, complex structures -- JSON and CSV can't include comments, which can be used to document configurations, but are often useful to bridge from other systems in automated pipelines -- JSON is more verbose (reads: longer) and harder to parse visually for humans -- CSV isn't often expressive enough (e.g. dit doesn't allow for nested structures) +Using factories, repetive resource creation and management becomes easier -If needed, converting factories to consume JSON is a matter of switching from `yamldecode()` to `jsondecode()` in the right place on each module. +- for humans who have no direct experience with Terraform, by exposing filesystem hierarchies and YAML-based configuration data +- for connected systems, by accepting well know data exchange formats like JSON or CSV +- for external code that needs to enforce checks or policies, by eliminating the need to parse HCL code or Terraform outputs +- to implement authorization processes or workwflows in CI/CD, by removing the dependency on Terraform and HCL knowledge for the teams involved -## Resource factories in Fabric +## The how -### Fabric Modules +Fabric resource-level factories can be broadly split into two different types -- [folder](../../modules/folder/README.md#firewall-policy-factory) and [organization](../../modules/organization/README.md#firewall-policy-factory) implement factories for [hierarchical firewall policies](https://cloud.google.com/vpc/docs/firewall-policies) -- [net-vpc](../../modules/net-vpc/README.md#subnet-factory) for subnets creation -- [net-vpc-firewall](../../modules/net-vpc-firewall/README.md#rules-factory) for massive rules creation +- simple factories that manage one simple resource type (firewalls, VPC-SC policies) +- complex factories that manage a set of connected resources to implement a complex flow that is usually perceived as a single unit (project creation) -### Dedicated Factories +The first factory type is implemented at the module level, where one module exposes one or more factories for some of the resources that depend on the main module resource (e.g. firewall rules for a VPC). The main goal with this approach is to simplify resource management at scale by removing the dependency on Terraform and HCL. -- [cloud-identity-group-factory](cloud-identity-group-factory/README.md) for Cloud Identity group -- [net-vpc-firewall-yaml](net-vpc-firewall-yaml/README.md) for VPC firewall rules across different projects/VPCs -- [project-factory](project-factory/README.md) for projects - +These factories are often designed as module-level interfaces which are then exposed by any module that manages a specific type of resource. All these factories leverage a single `factory_configs` variable, that allows passing in the paths for all the different factories supported in the module. + +The second factory type is implemented as a standalone module that internally references other modules, and implements complex management of different resource sets as part of a single process implemented via the factory. The typical example is the project factory, that brings together the project, service accounts, and billing accounts modules to cover all the main aspects of project creation as a single unit. + +## Factory implementations + +### Module-level factory interfaces + +- **BigQuery Analicts Hub rules** + - `analytics-hub` +- **billing budgets** + - `billing-account` +- **Data Catalog tags** + - `data-catalog-tag` +- **Data Catalog tag templates** + - `data-catalog-tag-template` +- **Dataplex Datascan rules** + - `dataplex-datascan` +- **firewall policy rules** + - `net-firewall-policy` +- **hierarchical firewall policies** + - `folder` + - `project` +- **IAM custom roles** + - `organization` + - `project` +- **organization policies** + - `organization` + - `folder` + - `project` +- **organization policy custom constraints** + - `organization` +- **DNS response policy rules** + - `dns-response-policy` +- **VPC firewall rules** + - `net-vpc-firewall` +- **VPC subnets** + - `net-vpc` +- **VPC-SC access levels and policies** + - `vpc-sc` + +### Standalone factories + +- **projects** + - `project-factory` diff --git a/blueprints/factories/net-vpc-firewall-yaml/README.md b/blueprints/factories/net-vpc-firewall-yaml/README.md deleted file mode 100644 index e385a68e87..0000000000 --- a/blueprints/factories/net-vpc-firewall-yaml/README.md +++ /dev/null @@ -1,201 +0,0 @@ -# Google Cloud VPC Firewall Factory - -This module allows creation and management of different types of firewall rules by defining them in well formatted `yaml` files. - -Yaml abstraction for FW rules can simplify users onboarding and also makes rules definition simpler and clearer comparing to HCL. - -Nested folder structure for yaml configurations is optionally supported, which allows better and structured code management for multiple teams and environments. - -## Example - -### Terraform code - -```hcl -module "prod-firewall" { - source = "./fabric/blueprints/factories/net-vpc-firewall-yaml" - - project_id = "my-prod-project" - network = "my-prod-network" - config_directories = [ - "./firewall/prod", - "./firewall/common" - ] - - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } -} - -module "dev-firewall" { - source = "./fabric/blueprints/factories/net-vpc-firewall-yaml" - - project_id = "my-dev-project" - network = "my-dev-network" - config_directories = [ - "./firewall/dev", - "./firewall/common" - ] -} -# tftest modules=2 resources=16 files=common,dev,prod inventory=example.yaml -``` - -```yaml -# tftest-file id=common path=firewall/common/common.yaml - ---- -# Terraform will be unable to decode this file if it does not contain valid YAML -# You can retain `---` (start of the document) to indicate an empty document. - -# allow ingress from GCLB to all instances in the network -lb-health-checks: - allow: - - ports: [] - protocol: tcp - direction: INGRESS - priority: 1001 - source_ranges: - - 35.191.0.0/16 - - 130.211.0.0/22 - -# deny all egress -deny-all: - deny: - - ports: [] - protocol: all - direction: EGRESS - priority: 65535 - destination_ranges: - - 0.0.0.0/0 -``` - -```yaml -# tftest-file id=dev path=firewall/dev/app.yaml - ---- -# Terraform will be unable to decode this file if it does not contain valid YAML -# You can retain `---` (start of the document) to indicate an empty document. - -# Myapp egress -web-app-dev-egress: - allow: - - ports: [443] - protocol: tcp - direction: EGRESS - destination_ranges: - - 192.168.0.0/24 - target_service_accounts: - - myapp@myproject-dev.iam.gserviceaccount.com -# Myapp ingress -web-app-dev-ingress: - allow: - - ports: [1234] - protocol: tcp - direction: INGRESS - source_service_accounts: - - frontend-sa@myproject-dev.iam.gserviceaccount.com - target_service_accounts: - - web-app-a@myproject-dev.iam.gserviceaccount.com -``` - -```yaml -# tftest-file id=prod path=firewall/prod/app.yaml - ---- -# Terraform will be unable to decode this file if it does not contain valid YAML -# You can retain `---` (start of the document) to indicate an empty document. - -# Myapp egress -web-app-prod-egress: - allow: - - ports: [443] - protocol: tcp - direction: EGRESS - destination_ranges: - - 192.168.10.0/24 - target_service_accounts: - - myapp@myproject-prod.iam.gserviceaccount.com -# Myapp ingress -web-app-prod-ingress: - allow: - - ports: [1234] - protocol: tcp - direction: INGRESS - source_service_accounts: - - frontend-sa@myproject-prod.iam.gserviceaccount.com - target_service_accounts: - - web-app-a@myproject-prod.iam.gserviceaccount.com -``` - -### Configuration Structure - -```bash -├── common -│ ├── default-egress.yaml -│   ├── lb-rules.yaml -│   └── iap-ingress.yaml -├── dev -│   ├── team-a -│   │   ├── databases.yaml -│   │   └── webb-app-a.yaml -│   └── team-b -│   ├── backend.yaml -│   └── frontend.yaml -└── prod - ├── team-a - │   ├── databases.yaml - │   └── webb-app-a.yaml - └── team-b - ├── backend.yaml - └── frontend.yaml -``` - -### Rule definition format and structure - -Firewall rules configuration should be placed in a set of yaml files in a folder/s. Firewall rule entry structure is following: - -```yaml - ---- -# Terraform will be unable to decode this file if it does not contain valid YAML -# You can retain `---` (start of the document) to indicate an empty document. - -rule-name: # descriptive name, naming convention is adjusted by the module - allow: # `allow` or `deny` - - ports: ['443', '80'] # ports for a specific protocol, keep empty list `[]` for all ports - protocol: tcp # protocol, put `all` for any protocol - direction: EGRESS # EGRESS or INGRESS - disabled: false # `false` or `true`, FW rule is disabled when `true`, default value is `false` - priority: 1000 # rule priority value, default value is 1000 - source_ranges: # list of source ranges, should be specified only for `INGRESS` rule - - 0.0.0.0/0 - destination_ranges: # list of destination ranges, should be specified only for `EGRESS` rule - - 0.0.0.0/0 - source_tags: ['some-tag'] # list of source tags, should be specified only for `INGRESS` rule - source_service_accounts: # list of source service accounts, should be specified only for `INGRESS` rule, cannot be specified together with `source_tags` or `target_tags` - - myapp@myproject-id.iam.gserviceaccount.com - target_tags: ['some-tag'] # list of target tags - target_service_accounts: # list of target service accounts, , cannot be specified together with `source_tags` or `target_tags` - - myapp@myproject-id.iam.gserviceaccount.com -``` - - - -## Variables - -| name | description | type | required | default | -|---|---|:---:|:---:|:---:| -| [config_directories](variables.tf#L17) | List of paths to folders where firewall configs are stored in yaml format. Folder may include subfolders with configuration files. Files suffix must be `.yaml`. | list(string) | ✓ | | -| [network](variables.tf#L30) | Name of the network this set of firewall rules applies to. | string | ✓ | | -| [project_id](variables.tf#L35) | Project Id. | string | ✓ | | -| [log_config](variables.tf#L22) | Log configuration. Possible values for `metadata` are `EXCLUDE_ALL_METADATA` and `INCLUDE_ALL_METADATA`. Set to `null` for disabling firewall logging. | object({…}) | | null | - -## Outputs - -| name | description | sensitive | -|---|---|:---:| -| [egress_allow_rules](outputs.tf#L17) | Egress rules with allow blocks. | | -| [egress_deny_rules](outputs.tf#L25) | Egress rules with allow blocks. | | -| [ingress_allow_rules](outputs.tf#L33) | Ingress rules with allow blocks. | | -| [ingress_deny_rules](outputs.tf#L41) | Ingress rules with deny blocks. | | - - diff --git a/blueprints/factories/net-vpc-firewall-yaml/main.tf b/blueprints/factories/net-vpc-firewall-yaml/main.tf deleted file mode 100644 index 0cfacf8a62..0000000000 --- a/blueprints/factories/net-vpc-firewall-yaml/main.tf +++ /dev/null @@ -1,113 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - firewall_rule_files = flatten( - [ - for config_path in var.config_directories : - concat( - [ - for config_file in fileset(config_path, "**/*.yaml") : - "${config_path}/${config_file}" - ] - ) - - ] - ) - - firewall_rules = merge( - [ - for config_file in local.firewall_rule_files : - yamldecode(file(config_file)) - ]... - ) -} - -resource "time_static" "timestamp" { - for_each = local.firewall_rules - triggers = { - name = md5(jsonencode(each.value)) - } -} - -resource "google_compute_firewall" "rules" { - for_each = local.firewall_rules - project = var.project_id - name = format( - "fwr-%s-%s-%s-%s", - var.network, - (try(each.value.target_service_accounts, null) != null ? "sac" : try(each.value.target_tags, null) != null ? "vpc" : "all"), - substr(lower(each.value.direction), 0, 1), - each.key - ) - description = format( - "%s rule in network %s for %s created at %s", - each.value.direction, - var.network, - each.key, - time_static.timestamp[each.key].rfc3339 - ) - - network = var.network - direction = each.value.direction - priority = try(each.value.priority, 1000) - disabled = try(each.value.disabled, null) - - source_ranges = try(each.value.source_ranges, each.value.direction == "INGRESS" ? [] : null) - source_tags = try(each.value.source_tags, null) - source_service_accounts = try(each.value.source_service_accounts, null) - - destination_ranges = try(each.value.destination_ranges, each.value.direction == "EGRESS" ? [] : null) - target_tags = try(each.value.target_tags, null) - target_service_accounts = try(each.value.target_service_accounts, null) - - dynamic "allow" { - for_each = { for block in try(each.value.allow, []) : - "${block.protocol}-${join("-", block.ports)}" => { - ports = [for port in block.ports : tostring(port)] - protocol = block.protocol - } - } - content { - protocol = allow.value.protocol - ports = allow.value.ports - } - } - - dynamic "deny" { - for_each = { for block in try(each.value.deny, []) : - "${block.protocol}-${join("-", block.ports)}" => { - ports = [for port in block.ports : tostring(port)] - protocol = block.protocol - } - } - content { - protocol = deny.value.protocol - ports = deny.value.ports - } - } - - dynamic "log_config" { - for_each = var.log_config != null ? [""] : [] - content { - metadata = var.log_config.metadata - } - } - - lifecycle { - create_before_destroy = true - } -} diff --git a/blueprints/factories/net-vpc-firewall-yaml/outputs.tf b/blueprints/factories/net-vpc-firewall-yaml/outputs.tf deleted file mode 100644 index f60e9c92fa..0000000000 --- a/blueprints/factories/net-vpc-firewall-yaml/outputs.tf +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "egress_allow_rules" { - description = "Egress rules with allow blocks." - value = [ - for rule in google_compute_firewall.rules : - rule if rule.direction == "EGRESS" && length(rule.allow) > 0 - ] -} - -output "egress_deny_rules" { - description = "Egress rules with allow blocks." - value = [ - for rule in google_compute_firewall.rules : - rule if rule.direction == "EGRESS" && length(rule.deny) > 0 - ] -} - -output "ingress_allow_rules" { - description = "Ingress rules with allow blocks." - value = [ - for rule in google_compute_firewall.rules : - rule if rule.direction == "INGRESS" && length(rule.allow) > 0 - ] -} - -output "ingress_deny_rules" { - description = "Ingress rules with deny blocks." - value = [ - for rule in google_compute_firewall.rules : - rule if rule.direction == "INGRESS" && length(rule.deny) > 0 - ] -} diff --git a/blueprints/factories/net-vpc-firewall-yaml/variables.tf b/blueprints/factories/net-vpc-firewall-yaml/variables.tf deleted file mode 100644 index b41eb57ce1..0000000000 --- a/blueprints/factories/net-vpc-firewall-yaml/variables.tf +++ /dev/null @@ -1,38 +0,0 @@ -/** - * Copyright 2022 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "config_directories" { - description = "List of paths to folders where firewall configs are stored in yaml format. Folder may include subfolders with configuration files. Files suffix must be `.yaml`." - type = list(string) -} - -variable "log_config" { - description = "Log configuration. Possible values for `metadata` are `EXCLUDE_ALL_METADATA` and `INCLUDE_ALL_METADATA`. Set to `null` for disabling firewall logging." - type = object({ - metadata = string - }) - default = null -} - -variable "network" { - description = "Name of the network this set of firewall rules applies to." - type = string -} - -variable "project_id" { - description = "Project Id." - type = string -} diff --git a/blueprints/gke/patterns/autopilot-cluster/versions.tf b/blueprints/gke/patterns/autopilot-cluster/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/blueprints/gke/patterns/autopilot-cluster/versions.tf +++ b/blueprints/gke/patterns/autopilot-cluster/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/blueprints/gke/patterns/batch/versions.tf b/blueprints/gke/patterns/batch/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/blueprints/gke/patterns/batch/versions.tf +++ b/blueprints/gke/patterns/batch/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/blueprints/gke/patterns/kafka/versions.tf b/blueprints/gke/patterns/kafka/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/blueprints/gke/patterns/kafka/versions.tf +++ b/blueprints/gke/patterns/kafka/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/blueprints/gke/patterns/mysql/versions.tf b/blueprints/gke/patterns/mysql/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/blueprints/gke/patterns/mysql/versions.tf +++ b/blueprints/gke/patterns/mysql/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/blueprints/gke/patterns/redis-cluster/versions.tf b/blueprints/gke/patterns/redis-cluster/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/blueprints/gke/patterns/redis-cluster/versions.tf +++ b/blueprints/gke/patterns/redis-cluster/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/blueprints/networking/README.md b/blueprints/networking/README.md index 8a4c40a08b..8a4f958129 100644 --- a/blueprints/networking/README.md +++ b/blueprints/networking/README.md @@ -18,12 +18,6 @@ They are meant to be used as minimal but complete starting points to create actu
-### Decentralized firewall management - - This [blueprint](./decentralized-firewall/) shows how a decentralized firewall management can be organized using the [firewall factory](../factories/net-vpc-firewall-yaml/). - -
- ### GLB and multi-regional daisy-chaining through hybrid NEGs This [blueprint](./glb-hybrid-neg-internal/) shows the experimental use of hybrid NEGs behind external Global Load Balancers (GLBs) to connect to GCP instances living in spoke VPCs and behind Network Virtual Appliances (NVAs). diff --git a/blueprints/networking/decentralized-firewall/README.md b/blueprints/networking/decentralized-firewall/README.md deleted file mode 100644 index 2fd89640c1..0000000000 --- a/blueprints/networking/decentralized-firewall/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Decentralized firewall management - -This example shows how a decentralized firewall management can be organized using the [firewall factory](../../factories/net-vpc-firewall-yaml/README.md). - -This approach is a good fit when Shared VPCs are used across multiple application/infrastructure teams. A central repository keeps environment/team -specific folders with firewall definitions in `yaml` format. - -In the current blueprint multiple teams can define their [VPC Firewall Rules](https://cloud.google.com/vpc/docs/firewalls) -for [dev](./firewall/dev) and [prod](./firewall/prod) environments using team specific subfolders. Rules defined in the -[common](./firewall/common) folder are applied to both dev and prod environments. - -> **_NOTE:_** Common rules are meant to be used for situations where [hierarchical rules](https://cloud.google.com/vpc/docs/firewall-policies) -do not map precisely to requirements (e.g. SA, etc.) - -This is the high level diagram: - -![High-level diagram](diagram.png "High-level diagram") - -The rules can be validated either using an automated process or a manual process (or a combination of -the two). There is an blueprint of a YAML-based validator using [Yamale](https://github.com/23andMe/Yamale) -in the [`validator/`](validator/) subdirectory, which can be integrated as part of a CI/CD pipeline. - - -## Variables - -| name | description | type | required | default | -|---|---|:---:|:---:|:---:| -| [billing_account_id](variables.tf#L15) | Billing account id used as default for new projects. | string | ✓ | | -| [prefix](variables.tf#L29) | Prefix used for resource names. | string | ✓ | | -| [root_node](variables.tf#L54) | Hierarchy node where projects will be created, 'organizations/org_id' or 'folders/folder_id'. | string | ✓ | | -| [ip_ranges](variables.tf#L20) | Subnet IP CIDR ranges. | map(string) | | {…} | -| [project_services](variables.tf#L38) | Service APIs enabled by default in new projects. | list(string) | | […] | -| [region](variables.tf#L48) | Region used. | string | | "europe-west1" | - -## Outputs - -| name | description | sensitive | -|---|---|:---:| -| [fw_rules](outputs.tf#L15) | Firewall rules. | | -| [projects](outputs.tf#L33) | Project ids. | | -| [vpc](outputs.tf#L41) | Shared VPCs. | | - - - -## Test -```hcl -module "test" { - source = "./fabric/blueprints/networking/decentralized-firewall" - billing_account_id = "ABCDE-12345-ABCDE" - prefix = "prefix" - root_node = "organizations/0123456789" -} - -# tftest modules=9 resources=54 -``` diff --git a/blueprints/networking/decentralized-firewall/backend.tf.sample b/blueprints/networking/decentralized-firewall/backend.tf.sample deleted file mode 100644 index e1bb8eaf50..0000000000 --- a/blueprints/networking/decentralized-firewall/backend.tf.sample +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -terraform { - backend "gcs" { - bucket = "" - } -} diff --git a/blueprints/networking/decentralized-firewall/diagram.png b/blueprints/networking/decentralized-firewall/diagram.png deleted file mode 100644 index e96aa1c3f8..0000000000 Binary files a/blueprints/networking/decentralized-firewall/diagram.png and /dev/null differ diff --git a/blueprints/networking/decentralized-firewall/firewall/common/common-egress.yaml b/blueprints/networking/decentralized-firewall/firewall/common/common-egress.yaml deleted file mode 100644 index 0af388b8dc..0000000000 --- a/blueprints/networking/decentralized-firewall/firewall/common/common-egress.yaml +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -# Terraform will be unable to decode this file if it does not contain valid YAML -# You can retain `---` (start of the document) to indicate an empty document. - -# Deny all egress (egress traffic is allowed by default) -deny-all: - deny: - - ports: [] - protocol: all - direction: EGRESS - priority: 65535 - destination_ranges: - - 0.0.0.0/0 - -# Allow access to GCP APIs via Private Google Access -# https://cloud.google.com/vpc/docs/access-apis-external-ip#config -gcp-pga-apis: - allow: - - ports: [443] - protocol: tcp - direction: EGRESS - priority: 500 - destination_ranges: - - 199.36.153.8/30 - -# Allow egress to internal networks -internal-egress: - allow: - - ports: [] - protocol: tcp - direction: EGRESS - destination_ranges: - - 10.0.0.0/16 diff --git a/blueprints/networking/decentralized-firewall/firewall/common/iap-access.yaml b/blueprints/networking/decentralized-firewall/firewall/common/iap-access.yaml deleted file mode 100644 index b8565473c1..0000000000 --- a/blueprints/networking/decentralized-firewall/firewall/common/iap-access.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -# Terraform will be unable to decode this file if it does not contain valid YAML -# You can retain `---` (start of the document) to indicate an empty document. - -# Access via SSH from IAP to all instancess https://cloud.google.com/iap/docs/using-tcp-forwarding#create-firewall-rule -iap-ssh-access: - allow: - - ports: [22] - protocol: tcp - direction: INGRESS - priority: 1001 - source_ranges: - - 35.235.240.0/20 diff --git a/blueprints/networking/decentralized-firewall/firewall/common/lb-access.yaml b/blueprints/networking/decentralized-firewall/firewall/common/lb-access.yaml deleted file mode 100644 index ca5c859aab..0000000000 --- a/blueprints/networking/decentralized-firewall/firewall/common/lb-access.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -# Terraform will be unable to decode this file if it does not contain valid YAML -# You can retain `---` (start of the document) to indicate an empty document. - -# Access from GCP LBs https://cloud.google.com/load-balancing/docs/https/#firewall_rules -lb-health-checks: - allow: - - ports: [] - protocol: tcp - direction: INGRESS - priority: 1001 - source_ranges: - - 35.191.0.0/16 - - 130.211.0.0/22 diff --git a/blueprints/networking/decentralized-firewall/firewall/dev/app-1/app1-rules.yaml b/blueprints/networking/decentralized-firewall/firewall/dev/app-1/app1-rules.yaml deleted file mode 100644 index 6b625b7949..0000000000 --- a/blueprints/networking/decentralized-firewall/firewall/dev/app-1/app1-rules.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -# Terraform will be unable to decode this file if it does not contain valid YAML -# You can retain `---` (start of the document) to indicate an empty document. - -# Allow traffic from the frontend VMs -app1-backend: - allow: - - ports: ["443", "80"] - protocol: tcp - direction: INGRESS - source_tags: ["app1-frontend"] - target_tags: ["app1-backend"] - -# Allow traffic to MySQL Servers from App1 backend -app1-db: - allow: - - ports: ["3306"] - protocol: tcp - direction: INGRESS - source_tags: ["app1-backend"] - target_tags: ["mysql-server"] diff --git a/blueprints/networking/decentralized-firewall/firewall/dev/app-2/app2-rules.yaml b/blueprints/networking/decentralized-firewall/firewall/dev/app-2/app2-rules.yaml deleted file mode 100644 index 82dd355ec9..0000000000 --- a/blueprints/networking/decentralized-firewall/firewall/dev/app-2/app2-rules.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -# Terraform will be unable to decode this file if it does not contain valid YAML -# You can retain `---` (start of the document) to indicate an empty document. - -# Allow traffic from app1 frontend -app2-backend: - allow: - - ports: ["443", "80"] - protocol: tcp - direction: INGRESS - source_tags: ["app1-frontend"] - target_tags: ["app2-backend"] - -# Allow traffic to MySQL servers from App2 backend -app2-db: - allow: - - ports: ["3306"] - protocol: tcp - direction: INGRESS - source_tags: ["app2-backend"] - target_tags: ["mysql-server"] diff --git a/blueprints/networking/decentralized-firewall/firewall/prod/app-1/app1-rules.yaml b/blueprints/networking/decentralized-firewall/firewall/prod/app-1/app1-rules.yaml deleted file mode 100644 index 6b625b7949..0000000000 --- a/blueprints/networking/decentralized-firewall/firewall/prod/app-1/app1-rules.yaml +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - ---- -# Terraform will be unable to decode this file if it does not contain valid YAML -# You can retain `---` (start of the document) to indicate an empty document. - -# Allow traffic from the frontend VMs -app1-backend: - allow: - - ports: ["443", "80"] - protocol: tcp - direction: INGRESS - source_tags: ["app1-frontend"] - target_tags: ["app1-backend"] - -# Allow traffic to MySQL Servers from App1 backend -app1-db: - allow: - - ports: ["3306"] - protocol: tcp - direction: INGRESS - source_tags: ["app1-backend"] - target_tags: ["mysql-server"] diff --git a/blueprints/networking/decentralized-firewall/main.tf b/blueprints/networking/decentralized-firewall/main.tf deleted file mode 100644 index 97ff9773ab..0000000000 --- a/blueprints/networking/decentralized-firewall/main.tf +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -############################################################################### -# Shared VPC Host projects # -############################################################################### - -module "project-host-prod" { - source = "../../../modules/project" - parent = var.root_node - billing_account = var.billing_account_id - prefix = var.prefix - name = "prod-host" - services = var.project_services - - shared_vpc_host_config = { - enabled = true - } -} - -module "project-host-dev" { - source = "../../../modules/project" - parent = var.root_node - billing_account = var.billing_account_id - prefix = var.prefix - name = "dev-host" - services = var.project_services - - shared_vpc_host_config = { - enabled = true - } -} - -################################################################################ -# Networking # -################################################################################ - -module "vpc-prod" { - source = "../../../modules/net-vpc" - project_id = module.project-host-prod.project_id - name = "prod-vpc" - subnets = [ - { - ip_cidr_range = var.ip_ranges.prod - name = "prod" - region = var.region - } - ] -} - -module "vpc-dev" { - source = "../../../modules/net-vpc" - project_id = module.project-host-dev.project_id - name = "dev-vpc" - subnets = [ - { - ip_cidr_range = var.ip_ranges.dev - name = "dev" - region = var.region - } - ] -} - -############################################################################### -# Private Google Access DNS # -############################################################################### - -module "dns-api-prod" { - source = "../../../modules/dns" - project_id = module.project-host-prod.project_id - name = "googleapis" - zone_config = { - domain = "googleapis.com." - private = { - client_networks = [module.vpc-prod.self_link] - } - } - recordsets = { - "CNAME *" = { records = ["private.googleapis.com."] } - } -} - -module "dns-api-dev" { - source = "../../../modules/dns" - project_id = module.project-host-dev.project_id - name = "googleapis" - zone_config = { - domain = "googleapis.com." - private = { - client_networks = [module.vpc-dev.self_link] - } - } - recordsets = { - "CNAME *" = { records = ["private.googleapis.com."] } - } -} - -############################################################################### -# Distributed Firewall # -############################################################################### - -module "vpc-firewall-prod" { - source = "../../factories/net-vpc-firewall-yaml" - - project_id = module.project-host-prod.project_id - network = module.vpc-prod.name - config_directories = [ - "${path.module}/firewall/common", - "${path.module}/firewall/prod" - ] - - # Enable Firewall Logging for the production fwl rules - log_config = { - metadata = "INCLUDE_ALL_METADATA" - } -} - -module "vpc-firewall-dev" { - source = "../../factories/net-vpc-firewall-yaml" - - project_id = module.project-host-dev.project_id - network = module.vpc-dev.name - config_directories = [ - "${path.module}/firewall/common", - "${path.module}/firewall/dev" - ] -} diff --git a/blueprints/networking/decentralized-firewall/outputs.tf b/blueprints/networking/decentralized-firewall/outputs.tf deleted file mode 100644 index be9138830d..0000000000 --- a/blueprints/networking/decentralized-firewall/outputs.tf +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -output "fw_rules" { - description = "Firewall rules." - value = { - prod = { - ingress_allow_rules = module.vpc-firewall-prod.ingress_allow_rules - ingress_deny_rules = module.vpc-firewall-prod.ingress_deny_rules - egress_allow_rules = module.vpc-firewall-prod.egress_allow_rules - egress_deny_rules = module.vpc-firewall-prod.egress_deny_rules - } - dev = { - ingress_allow_rules = module.vpc-firewall-dev.ingress_allow_rules - ingress_deny_rules = module.vpc-firewall-dev.ingress_deny_rules - egress_allow_rules = module.vpc-firewall-dev.egress_allow_rules - egress_deny_rules = module.vpc-firewall-dev.egress_deny_rules - } - } -} - -output "projects" { - description = "Project ids." - value = { - prod-host = module.project-host-prod.project_id - dev-host = module.project-host-dev.project_id - } -} - -output "vpc" { - description = "Shared VPCs." - value = { - prod = { - name = module.vpc-prod.name - subnets = module.vpc-prod.subnet_ips - } - dev = { - name = module.vpc-dev.name - subnets = module.vpc-dev.subnet_ips - } - } -} diff --git a/blueprints/networking/decentralized-firewall/validator/Dockerfile b/blueprints/networking/decentralized-firewall/validator/Dockerfile deleted file mode 100644 index 82d9a523a9..0000000000 --- a/blueprints/networking/decentralized-firewall/validator/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -FROM python:3.9-slim - -RUN mkdir /validator -COPY requirements.txt /validator/requirements.txt -RUN pip install -r /validator/requirements.txt -COPY validator.py /validator/validator.py - -RUN mkdir /schemas -COPY firewallSchema.yaml /schemas/firewallSchema.yaml -COPY firewallSchemaAutoApprove.yaml /schemas/firewallAutoApprove.yaml -COPY firewallSchemaSettings.yaml /schemas/firewallSchemaSettings.yaml - -RUN mkdir /rules - -CMD ["/rules/**/*.yaml"] -ENTRYPOINT ["python3", "/validator/validator.py"] \ No newline at end of file diff --git a/blueprints/networking/decentralized-firewall/validator/README.md b/blueprints/networking/decentralized-firewall/validator/README.md deleted file mode 100644 index fd58803702..0000000000 --- a/blueprints/networking/decentralized-firewall/validator/README.md +++ /dev/null @@ -1,80 +0,0 @@ -# Decentralized firewall validator - -The decentralized firewall validator is a Python scripts that utilizes [Yamale](https://github.com/23andMe/Yamale) schema -validation library to validate the configured firewall rules. - -## Configuring schemas - -There are three configuration files: -- [firewallSchema.yaml](firewallSchema.yaml), where the basic validation schema is configured -- [firewallSchemaAutoApprove.yaml](firewallSchemaAutoApprove.yaml), where the a different schema for auto-approval - can be configured (in case more validation is required than what is available in the schema settings) -- [firewallSchemaSettings.yaml](firewallSchemaSettings.yaml), configures list of allowed and approved - source and destination ranges, ports, network tags and service accounts. - -## Building the container - -You can build the container like this: - -```sh -docker build -t eu.gcr.io/YOUR-PROJECT/firewall-validator:latest . -docker push eu.gcr.io/YOUR-PROJECT/firewall-validator:latest -``` - -## Running the validator - -Example: - -```sh -docker run -v $(pwd)/firewall:/rules/ -t eu.gcr.io/YOUR-PROJECT/firewall-validator:latest -``` - -Output is JSON with keys `ok` and `errors` (if any were found). - -## Using as a GitHub action - -An `action.yml` is provided for this validator to be used as a GitHub action. - -Example of being used in a pipeline: - -```yaml - - uses: actions/checkout@v2 - - - name: Get changed files - if: ${{ github.event_name == 'pull_request' }} - id: changed-files - uses: tj-actions/changed-files@v1.1.2 - - - uses: ./.github/actions/validate-firewall - if: ${{ github.event_name == 'pull_request' }} - id: validation - with: - files: ${{ steps.changed-files.outputs.all_modified_files }} - - - uses: actions/github-script@v3 - if: ${{ github.event_name == 'pull_request' && steps.validation.outputs.ok != 'true' }} - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - var comments = []; - var errors = JSON.parse(process.env.ERRORS); - for (const filename in errors) { - var fn = filename.replace('/github/workspace/', ''); - comments.push({ - path: fn, - body: "```\n" + errors[filename].join("\n") + "\n```\n", - position: 1, - }); - } - github.pulls.createReview({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.issue.number, - event: "REQUEST_CHANGES", - body: "Firewall rule validation failed.", - comments: comments, - }); - core.setFailed("Firewall validation failed"); - env: - ERRORS: '${{ steps.validation.outputs.errors }}' -``` diff --git a/blueprints/networking/decentralized-firewall/validator/action.yml b/blueprints/networking/decentralized-firewall/validator/action.yml deleted file mode 100644 index b7817d9510..0000000000 --- a/blueprints/networking/decentralized-firewall/validator/action.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -name: "Validate firewall rules" -description: "Validate firewall rule YAML files" -inputs: - files: - description: "Files to scan (supports wildcards)" - required: false - default: "/github/workspace/firewall/**/*.yaml" - mode: - description: "Mode (validate or approve)" - required: false - default: "validate" - schema: - description: "Schema" - required: false - default: "/schemas/firewallSchema.yaml" -outputs: - ok: - description: "Validation successful" - errors: - description: "Validation results" -runs: - using: "docker" - image: "Dockerfile" - args: - - ${{ inputs.files }} - - "--mode" - - ${{ inputs.mode }} - - "--schema" - - ${{ inputs.schema }} - - "--github" diff --git a/blueprints/networking/decentralized-firewall/validator/firewallSchema.yaml b/blueprints/networking/decentralized-firewall/validator/firewallSchema.yaml deleted file mode 100644 index 8d11f13871..0000000000 --- a/blueprints/networking/decentralized-firewall/validator/firewallSchema.yaml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -map(include('rule'), key=str(min=3, max=30)) ---- -rule: - disabled: bool(required=False) - deny: list(include('trafficSpec'), required=False) - allow: list(include('trafficSpec'), required=False) - direction: enum('ingress', 'INGRESS', 'egress', 'EGRESS') - priority: int(min=1, max=65535, required=False) - destination_ranges: list(netmask(type='destination'), max=256, required=False) - source_ranges: list(netmask(type='source'), max=256, required=False) - source_tags: list(networktag(), max=30, required=False) - target_tags: list(networktag(), max=70, required=False) - source_service_accounts: list(serviceaccount(), max=10, required=False) - target_service_accounts: list(serviceaccount(), max=10, required=False) ---- -trafficSpec: - ports: list(networkports()) - protocol: enum('all', 'tcp', 'udp', 'icmp', 'esp', 'ah', 'ipip', 'sctp') diff --git a/blueprints/networking/decentralized-firewall/validator/firewallSchemaAutoApprove.yaml b/blueprints/networking/decentralized-firewall/validator/firewallSchemaAutoApprove.yaml deleted file mode 100644 index e6a9ae9cf9..0000000000 --- a/blueprints/networking/decentralized-firewall/validator/firewallSchemaAutoApprove.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -map(include('ingress'), include('egress'), key=str(min=3, max=30)) ---- -ingress: - disabled: bool(required=False) - deny: list(include('trafficSpec'), required=False) - allow: list(include('trafficSpec'), required=False) - direction: enum('ingress', 'INGRESS') - priority: int(min=1, max=65535, required=False) - source_ranges: list(netmask(type='source'), max=256, required=False) - source_tags: list(networktag(), max=30, required=False) - target_tags: list(networktag(), max=70, required=False) - source_service_accounts: list(serviceaccount(), max=10, required=False) - target_service_accounts: list(serviceaccount(), max=10, required=False) ---- -egress: - disabled: bool(required=False) - deny: list(include('trafficSpec'), required=False) - allow: list(include('trafficSpec'), required=False) - direction: enum('egress', 'EGRESS') - priority: int(min=1, max=65535, required=False) - destination_ranges: list(netmask(type='destination'), max=256, required=False) - source_tags: list(networktag(), max=30, required=False) - target_tags: list(networktag(), max=70, required=False) - source_service_accounts: list(serviceaccount(), max=10, required=False) - target_service_accounts: list(serviceaccount(), max=10, required=False) ---- -trafficSpec: - ports: list() - protocol: enum('all', 'tcp', 'udp', 'icmp', 'esp', 'ah', 'ipip', 'sctp') diff --git a/blueprints/networking/decentralized-firewall/validator/firewallSchemaSettings.yaml b/blueprints/networking/decentralized-firewall/validator/firewallSchemaSettings.yaml deleted file mode 100644 index 822dcc1ee2..0000000000 --- a/blueprints/networking/decentralized-firewall/validator/firewallSchemaSettings.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -allowedPorts: - - ports: 22 # SSH - approved: false - - ports: 80 # HTTP - approved: true - - ports: 443 # HTTPS - approved: true - - ports: 3306 # MySQL - approved: false - - ports: 8000-8999 - approved: true - -allowedSourceRanges: - - cidr: 10.0.0.0/8 # Example on-premise range - approved: true - - cidr: 35.191.0.0/16 # Load balancing & health checks - approved: true - - cidr: 130.211.0.0/22 # Load balancing & health checks - approved: false - - cidr: 35.235.240.0/20 # IAP source range - approved: true - -allowedDestinationRanges: - - cidr: 10.0.0.0/8 - approved: true - - cidr: 0.0.0.0/0 - approved: false - -allowedNetworkTags: - - tag: "*" - approved: true - -allowedServiceAccounts: - - serviceAccount: "*" - approved: true diff --git a/blueprints/networking/decentralized-firewall/validator/requirements.txt b/blueprints/networking/decentralized-firewall/validator/requirements.txt deleted file mode 100644 index 7084426b62..0000000000 --- a/blueprints/networking/decentralized-firewall/validator/requirements.txt +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -yamale~=3.0.0 -PyYAML~=5.4.0 -click~=7.1.0 \ No newline at end of file diff --git a/blueprints/networking/decentralized-firewall/validator/validator.py b/blueprints/networking/decentralized-firewall/validator/validator.py deleted file mode 100644 index ce6e87c42b..0000000000 --- a/blueprints/networking/decentralized-firewall/validator/validator.py +++ /dev/null @@ -1,256 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import glob -import ipaddress -import json -import sys - -import click -import yaml -import yamale - -from fnmatch import fnmatch -from types import SimpleNamespace -from yamale.validators import DefaultValidators, Validator - - -class Netmask(Validator): - """ Custom netmask validator """ - tag = 'netmask' - settings = {} - mode = None - _type = None - - def __init__(self, *args, **kwargs): - self._type = kwargs.pop('type', 'source-or-dest') - super().__init__(*args, **kwargs) - - def fail(self, value): - dir_str = 'source or destination' - mode_str = 'allowed' - if self._type == 'source': - dir_str = 'source' - elif self._type == 'destination': - dir_str = 'destination' - if self.mode == 'approve': - mode_str = 'automatically approved' - return '\'%s\' is not an %s %s network.' % (value, mode_str, dir_str) - - def _is_valid(self, value): - is_ok = False - network = ipaddress.ip_network(value) - if self._type == 'source' or self._type == 'source-or-dest': - for ip_range in self.settings['allowedSourceRanges']: - allowed_network = ipaddress.ip_network(ip_range['cidr']) - if network.subnet_of(allowed_network): - if self.mode != 'approve' or ip_range['approved']: - is_ok = True - break - if self._type == 'destination' or self._type == 'source-or-dest': - for ip_range in self.settings['allowedDestinationRanges']: - allowed_network = ipaddress.ip_network(ip_range['cidr']) - if network.subnet_of(allowed_network): - if self.mode != 'approve' or ip_range['approved']: - is_ok = True - break - - return is_ok - - -class NetworkTag(Validator): - """ Custom network tag validator """ - tag = 'networktag' - settings = {} - mode = None - _type = None - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def fail(self, value): - mode_str = 'allowed' - if self.mode == 'approve': - mode_str = 'automatically approved' - return '\'%s\' is not an %s network tag.' % (value, mode_str) - - def _is_valid(self, value): - is_ok = False - for tag in self.settings['allowedNetworkTags']: - if fnmatch(value, tag['tag']): - if self.mode != 'approve' or tag['approved']: - is_ok = True - break - return is_ok - - -class ServiceAccount(Validator): - """ Custom service account validator """ - tag = 'serviceaccount' - settings = {} - mode = None - _type = None - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def fail(self, value): - mode_str = 'allowed' - if self.mode == 'approve': - mode_str = 'automatically approved' - return '\'%s\' is not an %s service account.' % (value, mode_str) - - def _is_valid(self, value): - is_ok = False - for sa in self.settings['allowedServiceAccounts']: - if fnmatch(value, sa['serviceAccount']): - if self.mode != 'approve' or sa['approved']: - is_ok = True - break - return is_ok - - -class NetworkPorts(Validator): - """ Custom ports validator """ - tag = 'networkports' - settings = {} - mode = None - _type = None - allowed_port_map = [] - approved_port_map = [] - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - for port in self.settings['allowedPorts']: - ports = self._process_port_definition(port['ports']) - self.allowed_port_map.extend(ports) - if port['approved']: - self.approved_port_map.extend(ports) - - def _process_port_definition(self, port_definition): - ports = [] - if not isinstance(port_definition, int) and '-' in port_definition: - start, end = port_definition.split('-', 2) - for port in range(int(start), int(end) + 1): - ports.append(int(port)) - else: - ports.append(int(port_definition)) - return ports - - def fail(self, value): - mode_str = 'allowed' - if self.mode == 'approve': - mode_str = 'automatically approved' - return '\'%s\' is not an %s IP port.' % (value, mode_str) - - def _is_valid(self, value): - ports = self._process_port_definition(value) - is_ok = True - for port in ports: - if self.mode == 'approve' and port not in self.approved_port_map: - is_ok = False - break - elif port not in self.allowed_port_map: - is_ok = False - break - - return is_ok - - -class FirewallValidator: - schema = None - settings = None - validators = None - - def __init__(self, settings, mode): - self.settings = settings - - self.validators = DefaultValidators.copy() - Netmask.settings = self.settings - Netmask.mode = mode - self.validators[Netmask.tag] = Netmask - - NetworkTag.settings = self.settings - NetworkTag.mode = mode - self.validators[NetworkTag.tag] = NetworkTag - - ServiceAccount.settings = self.settings - ServiceAccount.mode = mode - self.validators[ServiceAccount.tag] = ServiceAccount - - NetworkPorts.settings = self.settings - NetworkPorts.mode = mode - self.validators[NetworkPorts.tag] = NetworkPorts - - def set_schema_from_file(self, schema): - self.schema = yamale.make_schema(path=schema, validators=self.validators) - - def set_schema_from_string(self, schema): - self.schema = yamale.make_schema(content=schema, validators=self.validators) - - def validate_file(self, file): - print('Validating %s...' % (file), file=sys.stderr) - data = yamale.make_data(file) - yamale.validate(self.schema, data) - - -@click.command() -@click.argument('files') -@click.option('--schema', default='/schemas/firewallSchema.yaml', - help='YAML schema file') -@click.option('--settings', default='/schemas/firewallSchemaSettings.yaml', - help='schema configuration file') -@click.option('--mode', default='validate', - help='select mode (validate or approve)') -@click.option('--github', is_flag=True, default=False, - help='output GitHub action compatible variables') -def main(**kwargs): - args = SimpleNamespace(**kwargs) - files = [args.files] - if '*' in args.files: - files = glob.glob(args.files, recursive=True) - - print('Arguments: %s' % (str(sys.argv)), file=sys.stderr) - - f = open(args.settings) - settings = yaml.load(f, Loader=yaml.SafeLoader) - - firewall_validator = FirewallValidator(settings, args.mode) - firewall_validator.set_schema_from_file(args.schema) - output = {'ok': True, 'errors': {}} - for file in files: - try: - firewall_validator.validate_file(file) - except yamale.yamale_error.YamaleError as e: - if file not in output['errors']: - output['errors'][file] = [] - output['ok'] = False - for result in e.results: - for err in result.errors: - output['errors'][file].append(err) - - if args.github: - print('::set-output name=ok::%s' % ('true' if output['ok'] else 'false')) - print('::set-output name=errors::%s' % (json.dumps(output['errors']))) - print(json.dumps(output), file=sys.stderr) - else: - print(json.dumps(output)) - if not output['ok'] and not args.github: - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/blueprints/networking/decentralized-firewall/variables.tf b/blueprints/networking/decentralized-firewall/variables.tf deleted file mode 100644 index d3b5b4b04f..0000000000 --- a/blueprints/networking/decentralized-firewall/variables.tf +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -variable "billing_account_id" { - description = "Billing account id used as default for new projects." - type = string -} - -variable "ip_ranges" { - description = "Subnet IP CIDR ranges." - type = map(string) - default = { - prod = "10.0.16.0/24" - dev = "10.0.32.0/24" - } -} - -variable "prefix" { - description = "Prefix used for resource names." - type = string - validation { - condition = var.prefix != "" - error_message = "Prefix cannot be empty." - } -} - -variable "project_services" { - description = "Service APIs enabled by default in new projects." - type = list(string) - default = [ - "container.googleapis.com", - "dns.googleapis.com", - "stackdriver.googleapis.com", - ] -} - -variable "region" { - description = "Region used." - type = string - default = "europe-west1" -} - -variable "root_node" { - description = "Hierarchy node where projects will be created, 'organizations/org_id' or 'folders/folder_id'." - type = string -} diff --git a/default-versions.tf b/default-versions.tf index 3db0e2076e..f43fef270d 100644 --- a/default-versions.tf +++ b/default-versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/fast/stages/2-networking-a-peering/dns-landing.tf b/fast/stages/2-networking-a-peering/dns-landing.tf index 2eefbc86d8..2c627122e4 100644 --- a/fast/stages/2-networking-a-peering/dns-landing.tf +++ b/fast/stages/2-networking-a-peering/dns-landing.tf @@ -82,8 +82,10 @@ module "landing-dns-policy-googleapis" { source = "../../../modules/dns-response-policy" project_id = module.landing-project.project_id name = "googleapis" + factories_config = { + rules = var.factories_config.dns_policy_rules_file + } networks = { landing = module.landing-vpc.self_link } - rules_file = var.factories_config.dns_policy_rules_file } diff --git a/fast/stages/2-networking-a-peering/main.tf b/fast/stages/2-networking-a-peering/main.tf index 9637c492b4..3db74dad58 100644 --- a/fast/stages/2-networking-a-peering/main.tf +++ b/fast/stages/2-networking-a-peering/main.tf @@ -60,7 +60,7 @@ module "firewall-policy-default" { source = "../../../modules/net-firewall-policy" name = var.factories_config.firewall_policy_name parent_id = module.folder.id - rules_factory_config = { + factories_config = { cidr_file_path = "${var.factories_config.data_dir}/cidrs.yaml" ingress_rules_file_path = "${var.factories_config.data_dir}/hierarchical-ingress-rules.yaml" } diff --git a/fast/stages/2-networking-b-vpn/dns-landing.tf b/fast/stages/2-networking-b-vpn/dns-landing.tf index 2eefbc86d8..2c627122e4 100644 --- a/fast/stages/2-networking-b-vpn/dns-landing.tf +++ b/fast/stages/2-networking-b-vpn/dns-landing.tf @@ -82,8 +82,10 @@ module "landing-dns-policy-googleapis" { source = "../../../modules/dns-response-policy" project_id = module.landing-project.project_id name = "googleapis" + factories_config = { + rules = var.factories_config.dns_policy_rules_file + } networks = { landing = module.landing-vpc.self_link } - rules_file = var.factories_config.dns_policy_rules_file } diff --git a/fast/stages/2-networking-b-vpn/main.tf b/fast/stages/2-networking-b-vpn/main.tf index 9637c492b4..3db74dad58 100644 --- a/fast/stages/2-networking-b-vpn/main.tf +++ b/fast/stages/2-networking-b-vpn/main.tf @@ -60,7 +60,7 @@ module "firewall-policy-default" { source = "../../../modules/net-firewall-policy" name = var.factories_config.firewall_policy_name parent_id = module.folder.id - rules_factory_config = { + factories_config = { cidr_file_path = "${var.factories_config.data_dir}/cidrs.yaml" ingress_rules_file_path = "${var.factories_config.data_dir}/hierarchical-ingress-rules.yaml" } diff --git a/fast/stages/2-networking-c-nva/dns-landing.tf b/fast/stages/2-networking-c-nva/dns-landing.tf index e18114fa7f..4b252dbd5e 100644 --- a/fast/stages/2-networking-c-nva/dns-landing.tf +++ b/fast/stages/2-networking-c-nva/dns-landing.tf @@ -91,9 +91,11 @@ module "landing-dns-policy-googleapis" { source = "../../../modules/dns-response-policy" project_id = module.landing-project.project_id name = "googleapis" + factories_config = { + rules = var.factories_config.dns_policy_rules_file + } networks = { landing-trusted = module.landing-trusted-vpc.self_link landing-untrusted = module.landing-untrusted-vpc.self_link } - rules_file = var.factories_config.dns_policy_rules_file } diff --git a/fast/stages/2-networking-c-nva/main.tf b/fast/stages/2-networking-c-nva/main.tf index 2f5fead609..ee2d58d6fd 100644 --- a/fast/stages/2-networking-c-nva/main.tf +++ b/fast/stages/2-networking-c-nva/main.tf @@ -61,7 +61,7 @@ module "firewall-policy-default" { source = "../../../modules/net-firewall-policy" name = var.factories_config.firewall_policy_name parent_id = module.folder.id - rules_factory_config = { + factories_config = { cidr_file_path = "${var.factories_config.data_dir}/cidrs.yaml" ingress_rules_file_path = "${var.factories_config.data_dir}/hierarchical-ingress-rules.yaml" } diff --git a/fast/stages/2-networking-d-separate-envs/dns-dev.tf b/fast/stages/2-networking-d-separate-envs/dns-dev.tf index 018b2391b6..46d41316c3 100644 --- a/fast/stages/2-networking-d-separate-envs/dns-dev.tf +++ b/fast/stages/2-networking-d-separate-envs/dns-dev.tf @@ -77,8 +77,10 @@ module "dev-dns-policy-googleapis" { source = "../../../modules/dns-response-policy" project_id = module.dev-spoke-project.project_id name = "googleapis" + factories_config = { + rules = var.factories_config.dns_policy_rules_file + } networks = { dev = module.dev-spoke-vpc.self_link } - rules_file = var.factories_config.dns_policy_rules_file } diff --git a/fast/stages/2-networking-d-separate-envs/dns-prod.tf b/fast/stages/2-networking-d-separate-envs/dns-prod.tf index 0c86e476e8..cfef28dd75 100644 --- a/fast/stages/2-networking-d-separate-envs/dns-prod.tf +++ b/fast/stages/2-networking-d-separate-envs/dns-prod.tf @@ -77,8 +77,10 @@ module "prod-dns-policy-googleapis" { source = "../../../modules/dns-response-policy" project_id = module.prod-spoke-project.project_id name = "googleapis" + factories_config = { + rules = var.factories_config.dns_policy_rules_file + } networks = { prod = module.prod-spoke-vpc.self_link } - rules_file = var.factories_config.dns_policy_rules_file } diff --git a/fast/stages/2-networking-d-separate-envs/main.tf b/fast/stages/2-networking-d-separate-envs/main.tf index e9f632c98e..928969abde 100644 --- a/fast/stages/2-networking-d-separate-envs/main.tf +++ b/fast/stages/2-networking-d-separate-envs/main.tf @@ -56,7 +56,7 @@ module "firewall-policy-default" { source = "../../../modules/net-firewall-policy" name = var.factories_config.firewall_policy_name parent_id = module.folder.id - rules_factory_config = { + factories_config = { cidr_file_path = "${var.factories_config.data_dir}/cidrs.yaml" ingress_rules_file_path = "${var.factories_config.data_dir}/hierarchical-ingress-rules.yaml" } diff --git a/fast/stages/2-networking-e-nva-bgp/dns-landing.tf b/fast/stages/2-networking-e-nva-bgp/dns-landing.tf index e18114fa7f..4b252dbd5e 100644 --- a/fast/stages/2-networking-e-nva-bgp/dns-landing.tf +++ b/fast/stages/2-networking-e-nva-bgp/dns-landing.tf @@ -91,9 +91,11 @@ module "landing-dns-policy-googleapis" { source = "../../../modules/dns-response-policy" project_id = module.landing-project.project_id name = "googleapis" + factories_config = { + rules = var.factories_config.dns_policy_rules_file + } networks = { landing-trusted = module.landing-trusted-vpc.self_link landing-untrusted = module.landing-untrusted-vpc.self_link } - rules_file = var.factories_config.dns_policy_rules_file } diff --git a/fast/stages/2-networking-e-nva-bgp/main.tf b/fast/stages/2-networking-e-nva-bgp/main.tf index 2f5fead609..ee2d58d6fd 100644 --- a/fast/stages/2-networking-e-nva-bgp/main.tf +++ b/fast/stages/2-networking-e-nva-bgp/main.tf @@ -61,7 +61,7 @@ module "firewall-policy-default" { source = "../../../modules/net-firewall-policy" name = var.factories_config.firewall_policy_name parent_id = module.folder.id - rules_factory_config = { + factories_config = { cidr_file_path = "${var.factories_config.data_dir}/cidrs.yaml" ingress_rules_file_path = "${var.factories_config.data_dir}/hierarchical-ingress-rules.yaml" } diff --git a/fast/stages/3-project-factory/dev/README.md b/fast/stages/3-project-factory/dev/README.md index 07b748c50f..09f246e5c4 100644 --- a/fast/stages/3-project-factory/dev/README.md +++ b/fast/stages/3-project-factory/dev/README.md @@ -1,7 +1,7 @@ # Project factory The Project Factory (or PF) builds on top of your foundations to create and set up projects (and related resources) to be used for your workloads. -It is organized in folders representing environments (e.g., "dev", "prod"), each implemented by a stand-alone terraform [resource factory](https://medium.com/google-cloud/resource-factories-a-descriptive-approach-to-terraform-581b3ebb59c). +It is organized in folders representing environments (e.g., "dev", "prod"), each implemented by a stand-alone terraform [process factory](../../../../blueprints/factories/README.md). ## Design overview and choices @@ -13,7 +13,7 @@ A single factory creates projects in a well-defined context, according to your r Projects for each environment across different teams are created by dedicated service accounts, as exemplified in the diagram above. While there's no intrinsic limitation regarding where the project factory can create a projects, the IAM bindings for the service account effectively enforce boundaries (e.g., the production service account shouldn't be able to create or have any access to the development projects, and vice versa). -The project factory exposes all the features of the underlying [project module](../../../../modules/project/), including Shared VPC service project attachment, VPC SC perimeter membership, etc. +The project factory stage lightly wraps the underlying [project-factory module](../../../../modules/project-factory/), including Shared VPC service project attachment, VPC SC perimeter membership, etc. ## How to run this stage @@ -55,7 +55,7 @@ gcloud alpha storage cp gs://xxx-prod-iac-core-outputs-0/tfvars/2-security.auto. If you're not using FAST, refer to the [Variables](#variables) table at the bottom of this document for a full list of variables, their origin (e.g., a stage or specific to this one), and descriptions explaining their meaning. -Besides the values above, the project factory is driven by data files which closely follow the variables exposed by the [project module](../../../../modules/project/), with one file per project. Please refer to the underlying [project factory blueprint](../../../../blueprints/factories/project-factory/) documentation for details on the format. +Besides the values above, the project factory is driven by YAML data files, with one file per project. Please refer to the underlying [project factory module](../../../../modules/project-factory/) documentation for details on the format. Once the configuration is complete, run the project factory with: diff --git a/fast/stages/3-project-factory/dev/main.tf b/fast/stages/3-project-factory/dev/main.tf index 48867a9984..0efd6ad67d 100644 --- a/fast/stages/3-project-factory/dev/main.tf +++ b/fast/stages/3-project-factory/dev/main.tf @@ -17,7 +17,7 @@ # tfdoc:file:description Project factory. module "projects" { - source = "../../../../blueprints/factories/project-factory" + source = "../../../../modules/project-factory" data_defaults = { billing_account = var.billing_account.id # more defaults are available, check the project factory variables diff --git a/modules/README.md b/modules/README.md index fc12865ee4..f8c30b7035 100644 --- a/modules/README.md +++ b/modules/README.md @@ -39,6 +39,10 @@ These modules are used in the examples included in this repository. If you are u - [Project](./project) - [Projects (data source)](./projects-data-source) +## Process factories + +- [Project factory](./project-factory/) + ## Networking modules - [Address reservation](./net-address) diff --git a/modules/__docs/20231106-factories.md b/modules/__docs/20231106-factories.md new file mode 100644 index 0000000000..b9ebf45186 --- /dev/null +++ b/modules/__docs/20231106-factories.md @@ -0,0 +1,102 @@ +# Factories Refactor and Plan Forward + +**authors:** [Ludo](https://github.com/ludoo) +**last modified:** February 16, 2024 + +## Status + +Under discussion. + +## Context + +Factories evolved progressively in Fabric, from the original firewall factory module, to a semi-standardized approach to management of repeated resources. This progression happened piecemeal and it's now time to define a clear strategy for factories in both Fabric and FAST, so that we can remove guesswork from new developments and provide a predictive approach to users. + +The remainder of this section provides a summary of the current status. + +### Modules + +Several modules implement factories for repeated resources which are typically dependent from the main resource managed in the module: + +- `billing-account` provides a factory for billing alert rules tied to the billing account +- `dns-response-policy` provides a factory for rules in within the policy +- `net-firewall-policy` provides a factory for rules within the policy +- `net-vpc` provides a factory for subnets in the VPC +- `net-vpc-firewall` provides a factory for VPC firewall rules +- `organization` and `folder` provide a factory for hierarchical firewall rules within their policy +- `organization`, `folder` and `project` provide a factory for organization policies + +The common pattern for modules is management of *multiple resources* typically dependent from the single *main resource* managed by the module. + +### Blueprints + +The `factories` folder in blueprints contains a collection of factories with a fuzzier approach + +- `bigquery-factory` manages tables and views for 1-n datasets by wrapping the `bigquery-dataset` module via simple locals +- `cloud-identity-group-factory` manages Cloud Identity group members for 1-n groups by wrapping the `cloud-identity-group` via simple locals +- `net-vpc-firewall-yaml` is the original factory module managing VPC firewall rules, superseded by the factory in the `net-vpc-firewall` module +- `project-factory` combines the project, service account, and (planned) billing account and VPC modules to implement end-to-end project creation and configuration + +There's no clear common pattern for these factories, where some could be moved to the respective module and the project factory combines a collection of modules to implement a process. + +### FAST + +FAST currently leverages module-level factories (organization policies, subnets, firewalls, etc.), and also provides the project factory as a dedicated level 3 stage by wrapping the relevant blueprint and localizing a few variables for the environment (`prefix`, `labels`). + +## Proposal + +While the current approach is reasonably clear in regards to modules, it has never been formalized in a set of guidelines that can help authors define when and how new factories would made sense. + +On top of this, the `factories` blueprints folder contains code that that should really be moved to module-level factories, and the project factory which could/should be published directly as a FAST stage, since those are consumable as standalone modules. + +This proposal aims at addressing the above problems. + +### Module-level factory approach + +The current approach for module-level factories can be summarized in a single principle: + +> factories implemented in modules manage multiple resources which depend from one single main resource (or a small set of main resources) which are the main driver of the module. + +For example, the module managing a firewall policy exposes a factory for its rules, or the module managing a VPC exposes a factory for its subnets. But the project module would not expose a projects factory, as one project maps to a single module invocation. + +The proposal on factory modules then is to: + +- align all factory variables to the same standard, outlined below +- move the groups and bigquery factories from blueprints to the respective modules +- eventually add more factories when it makes sense to do so (e.g. for KMS keys, service accounts, etc.) + +The variable interface for module-level factories should use a single top-level `factory_configs` variable, whose type is an object with one or more attributes which are named according to the specific factory. This will allow composing multiple factory configurations into a single variable in FAST stages, by avoiding name overlaps. An example: + +```hcl +variable "factory_configs" { + description = "Path to folder containing budget alerts data files." + type = object({ + budgets_data_path = optional(string, "data/billing-budgets") + }) + nullable = false + default = {} +} +``` + +### Blueprint factories + +The `factories` folder in blueprints will be emptied, and a single README left in it pointing to all the module-level and FAST stage factories available. + +As outlined above, the existing factories will be moved to modules (bigquery and groups), FAST (project factory), or deleted (firewall rules). + +### FAST factories + +The only change for FAST factories will be moving the project factory from blueprints to the stage folder, and updating the path used for the environment-level wrapping stage. + +### File schema and filesystem organization + +Factory files schema must mimick and implement the variable interface for the module, including optionals and validation - which are implemented in code and checks. + +With notable exceptions (currently only the `cidrs.yaml` file consumed by firewall factories), the following convention for files/directory is proposed: + +- Factories should consume directories (vs single files) +- All files should contain a dictionary of resources or a single resource +- If the factory accepts one resource per file (e.g. VPC subnets), the file name should be used for the resource name and the YAML should allow defining a `name:` override +- Files in a directory should be parsed together and flattened into a single dictionary + +This allows developers to implement multiple resources in a single file or to use one file per resource, as they see fit. + diff --git a/modules/__experimental/alloydb-instance/README.md b/modules/__experimental_deprecated/alloydb-instance/README.md similarity index 100% rename from modules/__experimental/alloydb-instance/README.md rename to modules/__experimental_deprecated/alloydb-instance/README.md diff --git a/modules/__experimental/alloydb-instance/main.tf b/modules/__experimental_deprecated/alloydb-instance/main.tf similarity index 100% rename from modules/__experimental/alloydb-instance/main.tf rename to modules/__experimental_deprecated/alloydb-instance/main.tf diff --git a/modules/__experimental/alloydb-instance/outputs.tf b/modules/__experimental_deprecated/alloydb-instance/outputs.tf similarity index 100% rename from modules/__experimental/alloydb-instance/outputs.tf rename to modules/__experimental_deprecated/alloydb-instance/outputs.tf diff --git a/modules/__experimental/alloydb-instance/variables.tf b/modules/__experimental_deprecated/alloydb-instance/variables.tf similarity index 100% rename from modules/__experimental/alloydb-instance/variables.tf rename to modules/__experimental_deprecated/alloydb-instance/variables.tf diff --git a/modules/__experimental/alloydb-instance/versions.tf b/modules/__experimental_deprecated/alloydb-instance/versions.tf similarity index 85% rename from modules/__experimental/alloydb-instance/versions.tf rename to modules/__experimental_deprecated/alloydb-instance/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/__experimental/alloydb-instance/versions.tf +++ b/modules/__experimental_deprecated/alloydb-instance/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/blueprints/factories/bigquery-factory/README.md b/modules/__experimental_deprecated/bigquery-factory/README.md similarity index 94% rename from blueprints/factories/bigquery-factory/README.md rename to modules/__experimental_deprecated/bigquery-factory/README.md index baf2f6d269..78362c5e9d 100644 --- a/blueprints/factories/bigquery-factory/README.md +++ b/modules/__experimental_deprecated/bigquery-factory/README.md @@ -11,6 +11,7 @@ You can create as many files as you like, the code will loop through it and crea ### Terraform code In this section we show how to create tables and views from a file structure similar to the one shown below. + ```bash bigquery │ @@ -53,12 +54,12 @@ With this file structure, we can use the factory as follows: ```hcl module "bq" { - source = "./fabric/blueprints/factories/bigquery-factory" + source = "./fabric/modules/__experimental_deprecated/bigquery-factory" project_id = var.project_id tables_path = "bigquery/tables" views_path = "bigquery/views" } -# tftest modules=2 resources=3 files=table,view inventory=simple.yaml +# tftest modules=2 resources=3 files=table,view ``` @@ -76,4 +77,3 @@ module "bq" { - [ ] add external table support - [ ] add materialized view support - diff --git a/blueprints/factories/bigquery-factory/main.tf b/modules/__experimental_deprecated/bigquery-factory/main.tf similarity index 100% rename from blueprints/factories/bigquery-factory/main.tf rename to modules/__experimental_deprecated/bigquery-factory/main.tf diff --git a/blueprints/factories/bigquery-factory/variables.tf b/modules/__experimental_deprecated/bigquery-factory/variables.tf similarity index 100% rename from blueprints/factories/bigquery-factory/variables.tf rename to modules/__experimental_deprecated/bigquery-factory/variables.tf diff --git a/blueprints/factories/cloud-identity-group-factory/README.md b/modules/__experimental_deprecated/cloud-identity-group-factory/README.md similarity index 92% rename from blueprints/factories/cloud-identity-group-factory/README.md rename to modules/__experimental_deprecated/cloud-identity-group-factory/README.md index 318eea2578..51e611a83e 100644 --- a/blueprints/factories/cloud-identity-group-factory/README.md +++ b/modules/__experimental_deprecated/cloud-identity-group-factory/README.md @@ -10,11 +10,11 @@ Yaml abstraction for Groups can simplify groups creation and members management. ```hcl module "groups" { - source = "./fabric/blueprints/factories/cloud-identity-group-factory" + source = "./fabric/modules/__experimental_deprecated/cloud-identity-group-factory" customer_id = "customers/C0xxxxxxx" data_dir = "data" } -# tftest modules=2 resources=3 files=group1 inventory=example.yaml +# tftest modules=2 resources=3 files=group1 ``` ```yaml diff --git a/blueprints/factories/cloud-identity-group-factory/main.tf b/modules/__experimental_deprecated/cloud-identity-group-factory/main.tf similarity index 100% rename from blueprints/factories/cloud-identity-group-factory/main.tf rename to modules/__experimental_deprecated/cloud-identity-group-factory/main.tf diff --git a/blueprints/factories/cloud-identity-group-factory/outputs.tf b/modules/__experimental_deprecated/cloud-identity-group-factory/outputs.tf similarity index 100% rename from blueprints/factories/cloud-identity-group-factory/outputs.tf rename to modules/__experimental_deprecated/cloud-identity-group-factory/outputs.tf diff --git a/blueprints/factories/cloud-identity-group-factory/variables.tf b/modules/__experimental_deprecated/cloud-identity-group-factory/variables.tf similarity index 100% rename from blueprints/factories/cloud-identity-group-factory/variables.tf rename to modules/__experimental_deprecated/cloud-identity-group-factory/variables.tf diff --git a/modules/__experimental/net-dns-policy-address/README.md b/modules/__experimental_deprecated/net-dns-policy-address/README.md similarity index 100% rename from modules/__experimental/net-dns-policy-address/README.md rename to modules/__experimental_deprecated/net-dns-policy-address/README.md diff --git a/modules/__experimental/net-dns-policy-address/main.tf b/modules/__experimental_deprecated/net-dns-policy-address/main.tf similarity index 100% rename from modules/__experimental/net-dns-policy-address/main.tf rename to modules/__experimental_deprecated/net-dns-policy-address/main.tf diff --git a/modules/__experimental/net-dns-policy-address/outputs.tf b/modules/__experimental_deprecated/net-dns-policy-address/outputs.tf similarity index 100% rename from modules/__experimental/net-dns-policy-address/outputs.tf rename to modules/__experimental_deprecated/net-dns-policy-address/outputs.tf diff --git a/modules/__experimental/net-dns-policy-address/variables.tf b/modules/__experimental_deprecated/net-dns-policy-address/variables.tf similarity index 100% rename from modules/__experimental/net-dns-policy-address/variables.tf rename to modules/__experimental_deprecated/net-dns-policy-address/variables.tf diff --git a/modules/__experimental/net-neg/README.md b/modules/__experimental_deprecated/net-neg/README.md similarity index 100% rename from modules/__experimental/net-neg/README.md rename to modules/__experimental_deprecated/net-neg/README.md diff --git a/modules/__experimental/net-neg/main.tf b/modules/__experimental_deprecated/net-neg/main.tf similarity index 100% rename from modules/__experimental/net-neg/main.tf rename to modules/__experimental_deprecated/net-neg/main.tf diff --git a/modules/__experimental/net-neg/outputs.tf b/modules/__experimental_deprecated/net-neg/outputs.tf similarity index 100% rename from modules/__experimental/net-neg/outputs.tf rename to modules/__experimental_deprecated/net-neg/outputs.tf diff --git a/modules/__experimental/net-neg/variables.tf b/modules/__experimental_deprecated/net-neg/variables.tf similarity index 100% rename from modules/__experimental/net-neg/variables.tf rename to modules/__experimental_deprecated/net-neg/variables.tf diff --git a/modules/__experimental/net-neg/versions.tf b/modules/__experimental_deprecated/net-neg/versions.tf similarity index 85% rename from modules/__experimental/net-neg/versions.tf rename to modules/__experimental_deprecated/net-neg/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/__experimental/net-neg/versions.tf +++ b/modules/__experimental_deprecated/net-neg/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/__experimental/project-iam-magic/versions.tf b/modules/__experimental_deprecated/project-iam-magic/versions.tf similarity index 85% rename from modules/__experimental/project-iam-magic/versions.tf rename to modules/__experimental_deprecated/project-iam-magic/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/__experimental/project-iam-magic/versions.tf +++ b/modules/__experimental_deprecated/project-iam-magic/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/analytics-hub/main.tf b/modules/analytics-hub/main.tf index e82798c175..0480dea636 100644 --- a/modules/analytics-hub/main.tf +++ b/modules/analytics-hub/main.tf @@ -18,9 +18,10 @@ locals { prefix = var.prefix == null || var.prefix == "" ? "" : "${var.prefix}_" _factory_listings = { for f in try(fileset(var.factories_config.listings, "*.yaml"), []) : - trimsuffix(f, ".yaml") => yamldecode(file("${var.factories_config.listings}/${f}")) + trimsuffix(f, ".yaml") => yamldecode( + file("${var.factories_config.listings}/${f}") + ) } - factory_listings = merge(local._factory_listings, var.listings) } diff --git a/modules/analytics-hub/versions.tf b/modules/analytics-hub/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/analytics-hub/versions.tf +++ b/modules/analytics-hub/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/api-gateway/versions.tf b/modules/api-gateway/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/api-gateway/versions.tf +++ b/modules/api-gateway/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/apigee/versions.tf b/modules/apigee/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/apigee/versions.tf +++ b/modules/apigee/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/artifact-registry/versions.tf b/modules/artifact-registry/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/artifact-registry/versions.tf +++ b/modules/artifact-registry/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/bigquery-dataset/versions.tf b/modules/bigquery-dataset/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/bigquery-dataset/versions.tf +++ b/modules/bigquery-dataset/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/bigtable-instance/versions.tf b/modules/bigtable-instance/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/bigtable-instance/versions.tf +++ b/modules/bigtable-instance/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/billing-account/README.md b/modules/billing-account/README.md index 23a73a1336..427fa2b917 100644 --- a/modules/billing-account/README.md +++ b/modules/billing-account/README.md @@ -260,16 +260,16 @@ update_rules: | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [id](variables.tf#L131) | Billing account id. | string | ✓ | | +| [id](variables.tf#L130) | Billing account id. | string | ✓ | | | [budget_notification_channels](variables.tf#L17) | Notification channels used by budget alerts. | map(object({…})) | | {} | | [budgets](variables.tf#L47) | Billing budgets. Notification channels are either keys in corresponding variable, or external ids. | map(object({…})) | | {} | -| [factory_config](variables.tf#L121) | Path to folder containing budget alerts data files. | object({…}) | | {} | +| [factories_config](variables.tf#L121) | Path to folder containing budget alerts data files. | object({…}) | | {} | | [iam](variables-iam.tf#L17) | IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_bindings](variables-iam.tf#L24) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | | [iam_bindings_additive](variables-iam.tf#L39) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | | [iam_by_principals](variables-iam.tf#L54) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | map(list(string)) | | {} | -| [logging_sinks](variables.tf#L136) | Logging sinks to create for the organization. | map(object({…})) | | {} | -| [projects](variables.tf#L169) | Projects associated with this billing account. | list(string) | | [] | +| [logging_sinks](variables.tf#L135) | Logging sinks to create for the organization. | map(object({…})) | | {} | +| [projects](variables.tf#L168) | Projects associated with this billing account. | list(string) | | [] | ## Outputs diff --git a/modules/billing-account/factory.tf b/modules/billing-account/factory.tf index f32c7c350f..ccca93228e 100644 --- a/modules/billing-account/factory.tf +++ b/modules/billing-account/factory.tf @@ -19,7 +19,7 @@ locals { for f in fileset("${local._factory_path}", "**/*.yaml") : trimsuffix(f, ".yaml") => yamldecode(file("${local._factory_path}/${f}")) } - _factory_path = var.factory_config.budgets_data_path + _factory_path = try(pathexpand(var.factories_config.budgets_data_path), "") factory_budgets = { for k, v in local._factory_data : k => merge(v, { amount = merge( diff --git a/modules/billing-account/variables.tf b/modules/billing-account/variables.tf index 3927b05bad..e69e6963f1 100644 --- a/modules/billing-account/variables.tf +++ b/modules/billing-account/variables.tf @@ -118,8 +118,7 @@ variable "budgets" { } } -variable "factory_config" { - # TODO: align all other factory variable names +variable "factories_config" { description = "Path to folder containing budget alerts data files." type = object({ budgets_data_path = optional(string, "data/billing-budgets") diff --git a/modules/billing-account/versions.tf b/modules/billing-account/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/billing-account/versions.tf +++ b/modules/billing-account/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/binauthz/versions.tf b/modules/binauthz/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/binauthz/versions.tf +++ b/modules/binauthz/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-config-container/__need_fixing/onprem/versions.tf b/modules/cloud-config-container/__need_fixing/onprem/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-config-container/__need_fixing/onprem/versions.tf +++ b/modules/cloud-config-container/__need_fixing/onprem/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-config-container/__need_fixing/squid/versions.tf b/modules/cloud-config-container/__need_fixing/squid/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-config-container/__need_fixing/squid/versions.tf +++ b/modules/cloud-config-container/__need_fixing/squid/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-config-container/coredns/versions.tf b/modules/cloud-config-container/coredns/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-config-container/coredns/versions.tf +++ b/modules/cloud-config-container/coredns/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-config-container/cos-generic-metadata/versions.tf b/modules/cloud-config-container/cos-generic-metadata/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-config-container/cos-generic-metadata/versions.tf +++ b/modules/cloud-config-container/cos-generic-metadata/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-config-container/envoy-sni-dyn-fwd-proxy/versions.tf b/modules/cloud-config-container/envoy-sni-dyn-fwd-proxy/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-config-container/envoy-sni-dyn-fwd-proxy/versions.tf +++ b/modules/cloud-config-container/envoy-sni-dyn-fwd-proxy/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-config-container/envoy-traffic-director/versions.tf b/modules/cloud-config-container/envoy-traffic-director/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-config-container/envoy-traffic-director/versions.tf +++ b/modules/cloud-config-container/envoy-traffic-director/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-config-container/mysql/versions.tf b/modules/cloud-config-container/mysql/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-config-container/mysql/versions.tf +++ b/modules/cloud-config-container/mysql/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-config-container/nginx-tls/versions.tf b/modules/cloud-config-container/nginx-tls/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-config-container/nginx-tls/versions.tf +++ b/modules/cloud-config-container/nginx-tls/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-config-container/nginx/versions.tf b/modules/cloud-config-container/nginx/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-config-container/nginx/versions.tf +++ b/modules/cloud-config-container/nginx/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-config-container/simple-nva/versions.tf b/modules/cloud-config-container/simple-nva/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-config-container/simple-nva/versions.tf +++ b/modules/cloud-config-container/simple-nva/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-function-v1/versions.tf b/modules/cloud-function-v1/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-function-v1/versions.tf +++ b/modules/cloud-function-v1/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-function-v2/versions.tf b/modules/cloud-function-v2/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-function-v2/versions.tf +++ b/modules/cloud-function-v2/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-identity-group/versions.tf b/modules/cloud-identity-group/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-identity-group/versions.tf +++ b/modules/cloud-identity-group/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-run-v2/versions.tf b/modules/cloud-run-v2/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-run-v2/versions.tf +++ b/modules/cloud-run-v2/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloud-run/versions.tf b/modules/cloud-run/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloud-run/versions.tf +++ b/modules/cloud-run/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/cloudsql-instance/versions.tf b/modules/cloudsql-instance/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/cloudsql-instance/versions.tf +++ b/modules/cloudsql-instance/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/compute-mig/versions.tf b/modules/compute-mig/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/compute-mig/versions.tf +++ b/modules/compute-mig/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/compute-vm/versions.tf b/modules/compute-vm/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/compute-vm/versions.tf +++ b/modules/compute-vm/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/container-registry/versions.tf b/modules/container-registry/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/container-registry/versions.tf +++ b/modules/container-registry/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/data-catalog-policy-tag/versions.tf b/modules/data-catalog-policy-tag/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/data-catalog-policy-tag/versions.tf +++ b/modules/data-catalog-policy-tag/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/data-catalog-tag-template/versions.tf b/modules/data-catalog-tag-template/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/data-catalog-tag-template/versions.tf +++ b/modules/data-catalog-tag-template/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/data-catalog-tag/versions.tf b/modules/data-catalog-tag/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/data-catalog-tag/versions.tf +++ b/modules/data-catalog-tag/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/dataform-repository/versions.tf b/modules/dataform-repository/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/dataform-repository/versions.tf +++ b/modules/dataform-repository/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/datafusion/versions.tf b/modules/datafusion/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/datafusion/versions.tf +++ b/modules/datafusion/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/dataplex-datascan/README.md b/modules/dataplex-datascan/README.md index 4053b60699..4b9ecbccc3 100644 --- a/modules/dataplex-datascan/README.md +++ b/modules/dataplex-datascan/README.md @@ -161,8 +161,8 @@ module "dataplex-datascan" { resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations" } incremental_field = "modified_date" - data_quality_spec_file = { - path = "config/data_quality_spec.yaml" + factories_config = { + data_quality_spec = "config/data_quality_spec.yaml" } } # tftest modules=1 resources=1 files=data_quality_spec inventory=datascan_dq.yaml @@ -244,8 +244,8 @@ module "dataplex-datascan" { resource = "//bigquery.googleapis.com/projects/bigquery-public-data/datasets/austin_bikeshare/tables/bikeshare_stations" } incremental_field = "modified_date" - data_quality_spec_file = { - path = "config/data_quality_spec_camel_case.yaml" + factories_config = { + data_quality_spec = "config/data_quality_spec_camel_case.yaml" } } # tftest modules=1 resources=1 files=data_quality_spec_camel_case inventory=datascan_dq.yaml @@ -431,21 +431,21 @@ module "dataplex-datascan" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [data](variables.tf#L17) | The data source for DataScan. The source can be either a Dataplex `entity` or a BigQuery `resource`. | object({…}) | ✓ | | -| [name](variables.tf#L118) | Name of Dataplex Scan. | string | ✓ | | -| [project_id](variables.tf#L129) | The ID of the project where the Dataplex DataScan will be created. | string | ✓ | | -| [region](variables.tf#L134) | Region for the Dataplex DataScan. | string | ✓ | | +| [name](variables.tf#L119) | Name of Dataplex Scan. | string | ✓ | | +| [project_id](variables.tf#L130) | The ID of the project where the Dataplex DataScan will be created. | string | ✓ | | +| [region](variables.tf#L135) | Region for the Dataplex DataScan. | string | ✓ | | | [data_profile_spec](variables.tf#L29) | DataProfileScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataProfileSpec. | object({…}) | | null | | [data_quality_spec](variables.tf#L38) | DataQualityScan related setting. Variable descriptions are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | object({…}) | | null | -| [data_quality_spec_file](variables.tf#L85) | Path to a YAML file containing DataQualityScan related setting. Input content can use either camelCase or snake_case. Variables description are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec. | object({…}) | | null | -| [description](variables.tf#L93) | Custom description for DataScan. | string | | null | -| [execution_schedule](variables.tf#L99) | Schedule DataScan to run periodically based on a cron schedule expression. If not specified, the DataScan is created with `on_demand` schedule, which means it will not run until the user calls `dataScans.run` API. | string | | null | +| [description](variables.tf#L85) | Custom description for DataScan. | string | | null | +| [execution_schedule](variables.tf#L91) | Schedule DataScan to run periodically based on a cron schedule expression. If not specified, the DataScan is created with `on_demand` schedule, which means it will not run until the user calls `dataScans.run` API. | string | | null | +| [factories_config](variables.tf#L97) | Paths to data files and folders that enable factory functionality. | object({…}) | | {} | | [iam](variables-iam.tf#L24) | Dataplex DataScan IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [iam_bindings](variables-iam.tf#L31) | Authoritative IAM bindings in {KEY => {role = ROLE, members = [], condition = {}}}. Keys are arbitrary. | map(object({…})) | | {} | | [iam_bindings_additive](variables-iam.tf#L46) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | | [iam_by_principals](variables-iam.tf#L17) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | map(list(string)) | | {} | -| [incremental_field](variables.tf#L105) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | string | | null | -| [labels](variables.tf#L111) | Resource labels. | map(string) | | {} | -| [prefix](variables.tf#L123) | Optional prefix used to generate Dataplex DataScan ID. | string | | null | +| [incremental_field](variables.tf#L106) | The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table. | string | | null | +| [labels](variables.tf#L112) | Resource labels. | map(string) | | {} | +| [prefix](variables.tf#L124) | Optional prefix used to generate Dataplex DataScan ID. | string | | null | ## Outputs diff --git a/modules/dataplex-datascan/factory.tf b/modules/dataplex-datascan/factory.tf new file mode 100644 index 0000000000..964e232f12 --- /dev/null +++ b/modules/dataplex-datascan/factory.tf @@ -0,0 +1,150 @@ +/** + * Copyright 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _factory_data = ( + var.factories_config.data_quality_spec == null + ? null + : yamldecode(file(pathexpand(var.factories_config.data_quality_spec))) + ) + factory_data = { + post_scan_actions = try( + local._factory_data.postScanActions, + local._factory_data.post_scan_actions, + null + ) + row_filter = try( + local._factory_data.rowFilter, + local._factory_data.row_filter, + null + ) + rules = [ + for rule in try(local._factory_data.rules, []) : { + column = try(rule.column, null) + ignore_null = try(rule.ignoreNull, rule.ignore_null, null) + dimension = rule.dimension + threshold = try(rule.threshold, null) + non_null_expectation = try( + rule.nonNullExpectation, rule.non_null_expectation, null + ) + range_expectation = ( + can(rule.rangeExpectation) || can(rule.range_expectation) + ? { + min_value = try( + rule.rangeExpectation.minValue, + rule.range_expectation.min_value, + null + ) + max_value = try( + rule.rangeExpectation.maxValue, + rule.range_expectation.max_value, + null + ) + strict_min_enabled = try( + rule.rangeExpectation.strictMinEnabled, + rule.range_expectation.strict_min_enabled, + null + ) + strict_max_enabled = try( + rule.rangeExpectation.strictMaxEnabled, + rule.range_expectation.strict_max_enabled, + null + ) + } + : null + ) + regex_expectation = ( + can(rule.regexExpectation) || can(rule.regex_expectation) + ? { + regex = try( + rule.regexExpectation.regex, rule.regex_expectation.regex, null + ) + } + : null + ) + set_expectation = ( + can(rule.setExpectation) || can(rule.set_expectation) + ? { + values = try( + rule.setExpectation.values, rule.set_expectation.values, null + ) + } + : null + ) + uniqueness_expectation = try( + rule.uniquenessExpectation, rule.uniqueness_expectation, null + ) + statistic_range_expectation = ( + can(rule.statisticRangeExpectation) || can(rule.statistic_range_expectation) + ? { + statistic = try( + rule.statisticRangeExpectation.statistic, + rule.statistic_range_expectation.statistic + ) + min_value = try( + rule.statisticRangeExpectation.minValue, + rule.statistic_range_expectation.min_value, + null + ) + max_value = try( + rule.statisticRangeExpectation.maxValue, + rule.statistic_range_expectation.max_value, + null + ) + strict_min_enabled = try( + rule.statisticRangeExpectation.strictMinEnabled, + rule.statistic_range_expectation.strict_min_enabled, + null + ) + strict_max_enabled = try( + rule.statisticRangeExpectation.strictMaxEnabled, + rule.statistic_range_expectation.strict_max_enabled, + null + ) + } + : null + ) + row_condition_expectation = ( + can(rule.rowConditionExpectation) || can(rule.row_condition_expectation) + ? { + sql_expression = try( + rule.rowConditionExpectation.sqlExpression, + rule.row_condition_expectation.sql_expression, + null + ) + } + : null + ) + table_condition_expectation = ( + can(rule.tableConditionExpectation) || can(rule.table_condition_expectation) + ? { + sql_expression = try( + rule.tableConditionExpectation.sqlExpression, + rule.table_condition_expectation.sql_expression, + null + ) + } + : null + ) + } + ] + sampling_percent = try( + local._factory_data.samplingPercent, + local._factory_data.sampling_percent, + null + ) + } +} diff --git a/modules/dataplex-datascan/main.tf b/modules/dataplex-datascan/main.tf index e1b6634d5d..0d9ad82ed6 100644 --- a/modules/dataplex-datascan/main.tf +++ b/modules/dataplex-datascan/main.tf @@ -15,17 +15,31 @@ */ locals { - prefix = var.prefix == null || var.prefix == "" ? "" : "${var.prefix}-" - _file_data_quality_spec = var.data_quality_spec_file == null ? null : { - sampling_percent = try(local._file_data_quality_spec_raw.samplingPercent, local._file_data_quality_spec_raw.sampling_percent, null) - row_filter = try(local._file_data_quality_spec_raw.rowFilter, local._file_data_quality_spec_raw.row_filter, null) - rules = local._parsed_rules - post_scan_actions = try(local._file_data_quality_spec_raw.postScanActions, local._file_data_quality_spec_raw.post_scan_actions, null) + data_quality_spec = { + post_scan_actions = try( + var.data_quality_spec.post_scan_actions, + local.factory_data.post_scan_actions, + null + ) + row_filter = try( + var.data_quality_spec.row_filter, + local.factory_data.row_filter, + null + ) + rules = concat( + try(var.data_quality_spec.rules, []), + try(local.factory_data.rules, []) + ) + sampling_percent = try( + var.data_quality_spec.sampling_percent, + local.factory_data.sampling_percent, + null + ) } - data_quality_spec = ( - var.data_quality_spec != null || var.data_quality_spec_file != null ? - merge(var.data_quality_spec, local._file_data_quality_spec) : - null + prefix = var.prefix == null || var.prefix == "" ? "" : "${var.prefix}-" + use_data_quality = ( + var.data_quality_spec != null || + var.factories_config.data_quality_spec != null ) } @@ -68,7 +82,7 @@ resource "google_dataplex_datascan" "datascan" { } dynamic "data_quality_spec" { - for_each = local.data_quality_spec != null ? [""] : [] + for_each = local.use_data_quality ? [""] : [] content { sampling_percent = try(local.data_quality_spec.sampling_percent, null) row_filter = try(local.data_quality_spec.row_filter, null) @@ -76,9 +90,16 @@ resource "google_dataplex_datascan" "datascan" { for_each = local.data_quality_spec.post_scan_actions != null ? [""] : [] content { dynamic "bigquery_export" { - for_each = local.data_quality_spec.post_scan_actions.bigquery_export != null ? [""] : [] + for_each = ( + local.data_quality_spec.post_scan_actions.bigquery_export != null + ? [""] + : [] + ) content { - results_table = try(local.data_quality_spec.post_scan_actions.bigquery_export.results_table, null) + results_table = try( + local.data_quality_spec.post_scan_actions.bigquery_export.results_table, + null + ) } } } @@ -98,55 +119,85 @@ resource "google_dataplex_datascan" "datascan" { } dynamic "range_expectation" { - for_each = try(rules.value.range_expectation, null) != null ? [""] : [] + for_each = ( + try(rules.value.range_expectation, null) != null ? [""] : [] + ) content { - min_value = try(rules.value.range_expectation.min_value, null) - max_value = try(rules.value.range_expectation.max_value, null) - strict_min_enabled = try(rules.value.range_expectation.strict_min_enabled, null) - strict_max_enabled = try(rules.value.range_expectation.strict_max_enabled, null) + min_value = try( + rules.value.range_expectation.min_value, null + ) + max_value = try( + rules.value.range_expectation.max_value, null + ) + strict_min_enabled = try( + rules.value.range_expectation.strict_min_enabled, null + ) + strict_max_enabled = try( + rules.value.range_expectation.strict_max_enabled, null + ) } } dynamic "set_expectation" { - for_each = try(rules.value.set_expectation, null) != null ? [""] : [] + for_each = ( + try(rules.value.set_expectation, null) != null ? [""] : [] + ) content { values = rules.value.set_expectation.values } } dynamic "uniqueness_expectation" { - for_each = try(rules.value.uniqueness_expectation, null) != null ? [""] : [] + for_each = ( + try(rules.value.uniqueness_expectation, null) != null ? [""] : [] + ) content { } } dynamic "regex_expectation" { - for_each = try(rules.value.regex_expectation, null) != null ? [""] : [] + for_each = ( + try(rules.value.regex_expectation, null) != null ? [""] : [] + ) content { regex = rules.value.regex_expectation.regex } } dynamic "statistic_range_expectation" { - for_each = try(rules.value.statistic_range_expectation, null) != null ? [""] : [] + for_each = ( + try(rules.value.statistic_range_expectation, null) != null ? [""] : [] + ) content { - min_value = try(rules.value.statistic_range_expectation.min_value, null) - max_value = try(rules.value.statistic_range_expectation.max_value, null) - strict_min_enabled = try(rules.value.statistic_range_expectation.strict_min_enabled, null) - strict_max_enabled = try(rules.value.statistic_range_expectation.strict_max_enabled, null) - statistic = rules.value.statistic_range_expectation.statistic + min_value = try( + rules.value.statistic_range_expectation.min_value, null + ) + max_value = try( + rules.value.statistic_range_expectation.max_value, null + ) + strict_min_enabled = try( + rules.value.statistic_range_expectation.strict_min_enabled, null + ) + strict_max_enabled = try( + rules.value.statistic_range_expectation.strict_max_enabled, null + ) + statistic = rules.value.statistic_range_expectation.statistic } } dynamic "row_condition_expectation" { - for_each = try(rules.value.row_condition_expectation, null) != null ? [""] : [] + for_each = ( + try(rules.value.row_condition_expectation, null) != null ? [""] : [] + ) content { sql_expression = rules.value.row_condition_expectation.sql_expression } } dynamic "table_condition_expectation" { - for_each = try(rules.value.table_condition_expectation, null) != null ? [""] : [] + for_each = ( + try(rules.value.table_condition_expectation, null) != null ? [""] : [] + ) content { sql_expression = rules.value.table_condition_expectation.sql_expression } @@ -159,8 +210,16 @@ resource "google_dataplex_datascan" "datascan" { lifecycle { precondition { - condition = length([for spec in [var.data_profile_spec, var.data_quality_spec, var.data_quality_spec_file] : spec if spec != null]) == 1 - error_message = "DataScan can only contain one of 'data_profile_spec', 'data_quality_spec', 'data_quality_spec_file'." + condition = ( + length([ + for spec in [ + var.data_profile_spec, + var.data_quality_spec, + var.factories_config.data_quality_spec + ] : spec if spec != null + ]) == 1 + ) + error_message = "DataScan can only contain one of 'data_profile_spec', 'data_quality_spec', 'factories_config.data_quality_spec'." } precondition { condition = alltrue([ diff --git a/modules/dataplex-datascan/rules_parsing.tf b/modules/dataplex-datascan/rules_parsing.tf deleted file mode 100644 index bbdc82206b..0000000000 --- a/modules/dataplex-datascan/rules_parsing.tf +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - _file_data_quality_spec_raw = var.data_quality_spec_file != null ? yamldecode(file(var.data_quality_spec_file.path)) : tomap({}) - _parsed_rules = [ - for rule in try(local._file_data_quality_spec_raw.rules, []) : { - column = try(rule.column, null) - ignore_null = try(rule.ignoreNull, rule.ignore_null, null) - dimension = rule.dimension - threshold = try(rule.threshold, null) - non_null_expectation = try(rule.nonNullExpectation, rule.non_null_expectation, null) - range_expectation = can(rule.rangeExpectation) || can(rule.range_expectation) ? { - min_value = try(rule.rangeExpectation.minValue, rule.range_expectation.min_value, null) - max_value = try(rule.rangeExpectation.maxValue, rule.range_expectation.max_value, null) - strict_min_enabled = try(rule.rangeExpectation.strictMinEnabled, rule.range_expectation.strict_min_enabled, null) - strict_max_enabled = try(rule.rangeExpectation.strictMaxEnabled, rule.range_expectation.strict_max_enabled, null) - } : null - regex_expectation = can(rule.regexExpectation) || can(rule.regex_expectation) ? { - regex = try(rule.regexExpectation.regex, rule.regex_expectation.regex, null) - } : null - set_expectation = can(rule.setExpectation) || can(rule.set_expectation) ? { - values = try(rule.setExpectation.values, rule.set_expectation.values, null) - } : null - uniqueness_expectation = try(rule.uniquenessExpectation, rule.uniqueness_expectation, null) - statistic_range_expectation = can(rule.statisticRangeExpectation) || can(rule.statistic_range_expectation) ? { - statistic = try(rule.statisticRangeExpectation.statistic, rule.statistic_range_expectation.statistic) - min_value = try(rule.statisticRangeExpectation.minValue, rule.statistic_range_expectation.min_value, null) - max_value = try(rule.statisticRangeExpectation.maxValue, rule.statistic_range_expectation.max_value, null) - strict_min_enabled = try(rule.statisticRangeExpectation.strictMinEnabled, rule.statistic_range_expectation.strict_min_enabled, null) - strict_max_enabled = try(rule.statisticRangeExpectation.strictMaxEnabled, rule.statistic_range_expectation.strict_max_enabled, null) - } : null - row_condition_expectation = can(rule.rowConditionExpectation) || can(rule.row_condition_expectation) ? { - sql_expression = try(rule.rowConditionExpectation.sqlExpression, rule.row_condition_expectation.sql_expression, null) - } : null - table_condition_expectation = can(rule.tableConditionExpectation) || can(rule.table_condition_expectation) ? { - sql_expression = try(rule.tableConditionExpectation.sqlExpression, rule.table_condition_expectation.sql_expression, null) - } : null - } - ] -} \ No newline at end of file diff --git a/modules/dataplex-datascan/variables.tf b/modules/dataplex-datascan/variables.tf index cab105bfed..c01774f75d 100644 --- a/modules/dataplex-datascan/variables.tf +++ b/modules/dataplex-datascan/variables.tf @@ -82,14 +82,6 @@ variable "data_quality_spec" { }) } -variable "data_quality_spec_file" { - description = "Path to a YAML file containing DataQualityScan related setting. Input content can use either camelCase or snake_case. Variables description are provided in https://cloud.google.com/dataplex/docs/reference/rest/v1/DataQualitySpec." - default = null - type = object({ - path = string - }) -} - variable "description" { description = "Custom description for DataScan." default = null @@ -102,6 +94,15 @@ variable "execution_schedule" { default = null } +variable "factories_config" { + description = "Paths to data files and folders that enable factory functionality." + type = object({ + data_quality_spec = optional(string) + }) + nullable = false + default = {} +} + variable "incremental_field" { description = "The unnested field (of type Date or Timestamp) that contains values which monotonically increase over time. If not specified, a data scan will run for all data in the table." type = string diff --git a/modules/dataplex-datascan/versions.tf b/modules/dataplex-datascan/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/dataplex-datascan/versions.tf +++ b/modules/dataplex-datascan/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/dataplex/versions.tf b/modules/dataplex/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/dataplex/versions.tf +++ b/modules/dataplex/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/dataproc/versions.tf b/modules/dataproc/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/dataproc/versions.tf +++ b/modules/dataproc/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/dns-response-policy/README.md b/modules/dns-response-policy/README.md index 9f90e8d74a..010c09c5a7 100644 --- a/modules/dns-response-policy/README.md +++ b/modules/dns-response-policy/README.md @@ -4,6 +4,16 @@ This module allows management of a [Google Cloud DNS policy and its rules](https The module also allows setting rules via a factory. An example is given below. + +- [Examples](#examples) + - [Manage policy and override resolution for specific names](#manage-policy-and-override-resolution-for-specific-names) + - [Use existing policy and override resolution via wildcard with exceptions](#use-existing-policy-and-override-resolution-via-wildcard-with-exceptions) + - [Define policy rules via a factory file](#define-policy-rules-via-a-factory-file) +- [Variables](#variables) +- [Outputs](#outputs) +- [Fixtures](#fixtures) + + ## Examples ### Manage policy and override resolution for specific names @@ -96,7 +106,9 @@ module "dns-policy" { networks = { landing = var.vpc.self_link } - rules_file = "config/rules.yaml" + factories_config = { + rules = "config/rules.yaml" + } } # tftest modules=2 resources=5 files=rules-file fixtures=fixtures/dns-response-policy.tf inventory=complex.yaml e2e ``` @@ -133,14 +145,14 @@ restricted: | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L30) | Policy name. | string | ✓ | | -| [project_id](variables.tf#L49) | Project id for the zone. | string | ✓ | | +| [name](variables.tf#L39) | Policy name. | string | ✓ | | +| [project_id](variables.tf#L58) | Project id for the zone. | string | ✓ | | | [clusters](variables.tf#L17) | Map of GKE clusters to which this policy is applied in name => id format. | map(string) | | {} | | [description](variables.tf#L24) | Policy description. | string | | "Terraform managed." | -| [networks](variables.tf#L35) | Map of VPC self links to which this policy is applied in name => self link format. | map(string) | | {} | -| [policy_create](variables.tf#L42) | Set to false to use the existing policy matching name and only manage rules. | bool | | true | -| [rules](variables.tf#L54) | Map of policy rules in name => rule format. Local data takes precedence over behavior and is in the form record type => attributes. | map(object({…})) | | {} | -| [rules_file](variables.tf#L68) | Optional data file in YAML format listing rules that will be combined with those passed in via the `rules` variable. | string | | null | +| [factories_config](variables.tf#L30) | Path to folder containing rules data files for the optional factory. | object({…}) | | {} | +| [networks](variables.tf#L44) | Map of VPC self links to which this policy is applied in name => self link format. | map(string) | | {} | +| [policy_create](variables.tf#L51) | Set to false to use the existing policy matching name and only manage rules. | bool | | true | +| [rules](variables.tf#L63) | Map of policy rules in name => rule format. Local data takes precedence over behavior and is in the form record type => attributes. | map(object({…})) | | {} | ## Outputs diff --git a/modules/dns-response-policy/main.tf b/modules/dns-response-policy/main.tf index 5d16849777..66ca9cdf72 100644 --- a/modules/dns-response-policy/main.tf +++ b/modules/dns-response-policy/main.tf @@ -15,9 +15,12 @@ */ locals { - _factory_data = var.rules_file != null ? file(var.rules_file) : "{}" + _factory_data = ( + var.factories_config.rules != null + ? file(pathexpand(var.factories_config.rules)) + : "{}" + ) _factory_rules = yamldecode(local._factory_data) - factory_rules = { for k, v in local._factory_rules : k => { dns_name = v.dns_name diff --git a/modules/dns-response-policy/variables.tf b/modules/dns-response-policy/variables.tf index fa26c3bb47..35a113c548 100644 --- a/modules/dns-response-policy/variables.tf +++ b/modules/dns-response-policy/variables.tf @@ -27,6 +27,15 @@ variable "description" { default = "Terraform managed." } +variable "factories_config" { + description = "Path to folder containing rules data files for the optional factory." + type = object({ + rules = optional(string) + }) + nullable = false + default = {} +} + variable "name" { description = "Policy name." type = string @@ -64,9 +73,3 @@ variable "rules" { default = {} nullable = false } - -variable "rules_file" { - description = "Optional data file in YAML format listing rules that will be combined with those passed in via the `rules` variable." - type = string - default = null -} diff --git a/modules/dns-response-policy/versions.tf b/modules/dns-response-policy/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/dns-response-policy/versions.tf +++ b/modules/dns-response-policy/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/dns/versions.tf b/modules/dns/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/dns/versions.tf +++ b/modules/dns/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/endpoints/versions.tf b/modules/endpoints/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/endpoints/versions.tf +++ b/modules/endpoints/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/folder/versions.tf b/modules/folder/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/folder/versions.tf +++ b/modules/folder/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/gcs/versions.tf b/modules/gcs/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/gcs/versions.tf +++ b/modules/gcs/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/gcve-private-cloud/versions.tf b/modules/gcve-private-cloud/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/gcve-private-cloud/versions.tf +++ b/modules/gcve-private-cloud/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/gke-cluster-autopilot/versions.tf b/modules/gke-cluster-autopilot/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/gke-cluster-autopilot/versions.tf +++ b/modules/gke-cluster-autopilot/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/gke-cluster-standard/versions.tf b/modules/gke-cluster-standard/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/gke-cluster-standard/versions.tf +++ b/modules/gke-cluster-standard/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/gke-hub/versions.tf b/modules/gke-hub/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/gke-hub/versions.tf +++ b/modules/gke-hub/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/gke-nodepool/versions.tf b/modules/gke-nodepool/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/gke-nodepool/versions.tf +++ b/modules/gke-nodepool/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/iam-service-account/versions.tf b/modules/iam-service-account/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/iam-service-account/versions.tf +++ b/modules/iam-service-account/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/kms/versions.tf b/modules/kms/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/kms/versions.tf +++ b/modules/kms/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/logging-bucket/versions.tf b/modules/logging-bucket/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/logging-bucket/versions.tf +++ b/modules/logging-bucket/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/ncc-spoke-ra/versions.tf b/modules/ncc-spoke-ra/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/ncc-spoke-ra/versions.tf +++ b/modules/ncc-spoke-ra/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-address/versions.tf b/modules/net-address/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-address/versions.tf +++ b/modules/net-address/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-cloudnat/versions.tf b/modules/net-cloudnat/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-cloudnat/versions.tf +++ b/modules/net-cloudnat/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-firewall-policy/README.md b/modules/net-firewall-policy/README.md index e4f7cfb796..17381abef6 100644 --- a/modules/net-firewall-policy/README.md +++ b/modules/net-firewall-policy/README.md @@ -194,7 +194,7 @@ module "firewall-policy" { } } } - rules_factory_config = { + factories_config = { cidr_file_path = "configs/cidrs.yaml" egress_rules_file_path = "configs/egress.yaml" ingress_rules_file_path = "configs/ingress.yaml" @@ -258,14 +258,14 @@ issue-1995: | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L102) | Policy name. | string | ✓ | | -| [parent_id](variables.tf#L108) | Parent node where the policy will be created, `folders/nnn` or `organizations/nnn` for hierarchical policy, project id for a network policy. | string | ✓ | | +| [name](variables.tf#L113) | Policy name. | string | ✓ | | +| [parent_id](variables.tf#L119) | Parent node where the policy will be created, `folders/nnn` or `organizations/nnn` for hierarchical policy, project id for a network policy. | string | ✓ | | | [attachments](variables.tf#L17) | Ids of the resources to which this policy will be attached, in descriptive name => self link format. Specify folders or organization for hierarchical policy, VPCs for network policy. | map(string) | | {} | | [description](variables.tf#L24) | Policy description. | string | | null | | [egress_rules](variables.tf#L30) | List of egress rule definitions, action can be 'allow', 'deny', 'goto_next'. The match.layer4configs map is in protocol => optional [ports] format. | map(object({…})) | | {} | -| [ingress_rules](variables.tf#L66) | List of ingress rule definitions, action can be 'allow', 'deny', 'goto_next'. | map(object({…})) | | {} | -| [region](variables.tf#L114) | Policy region. Leave null for hierarchical policy, set to 'global' for a global network policy. | string | | null | -| [rules_factory_config](variables.tf#L120) | Configuration for the optional rules factory. | object({…}) | | {} | +| [factories_config](variables.tf#L66) | Paths to folders for the optional factories. | object({…}) | | {} | +| [ingress_rules](variables.tf#L77) | List of ingress rule definitions, action can be 'allow', 'deny', 'goto_next'. | map(object({…})) | | {} | +| [region](variables.tf#L125) | Policy region. Leave null for hierarchical policy, set to 'global' for a global network policy. | string | | null | ## Outputs diff --git a/modules/net-firewall-policy/factory.tf b/modules/net-firewall-policy/factory.tf index be065b9b68..1b678d05e9 100644 --- a/modules/net-firewall-policy/factory.tf +++ b/modules/net-firewall-policy/factory.tf @@ -16,13 +16,15 @@ locals { _factory_egress_rules = try( - yamldecode(file(var.rules_factory_config.egress_rules_file_path)), {} + yamldecode(pathexpand(file(var.factories_config.egress_rules_file_path))), + {} ) _factory_ingress_rules = try( - yamldecode(file(var.rules_factory_config.ingress_rules_file_path)), {} + yamldecode(pathexpand(file(var.factories_config.ingress_rules_file_path))), + {} ) factory_cidrs = try( - yamldecode(file(var.rules_factory_config.cidr_file_path)), {} + yamldecode(pathexpand(file(var.factories_config.cidr_file_path))), {} ) factory_egress_rules = { for k, v in local._factory_egress_rules : "egress/${k}" => { diff --git a/modules/net-firewall-policy/variables.tf b/modules/net-firewall-policy/variables.tf index c419d7c027..3a8d16b70f 100644 --- a/modules/net-firewall-policy/variables.tf +++ b/modules/net-firewall-policy/variables.tf @@ -63,6 +63,17 @@ variable "egress_rules" { } } +variable "factories_config" { + description = "Paths to folders for the optional factories." + type = object({ + cidr_file_path = optional(string) + egress_rules_file_path = optional(string) + ingress_rules_file_path = optional(string) + }) + nullable = false + default = {} +} + variable "ingress_rules" { description = "List of ingress rule definitions, action can be 'allow', 'deny', 'goto_next'." type = map(object({ @@ -116,14 +127,3 @@ variable "region" { type = string default = null } - -variable "rules_factory_config" { - description = "Configuration for the optional rules factory." - type = object({ - cidr_file_path = optional(string) - egress_rules_file_path = optional(string) - ingress_rules_file_path = optional(string) - }) - nullable = false - default = {} -} diff --git a/modules/net-firewall-policy/versions.tf b/modules/net-firewall-policy/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-firewall-policy/versions.tf +++ b/modules/net-firewall-policy/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-ipsec-over-interconnect/versions.tf b/modules/net-ipsec-over-interconnect/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-ipsec-over-interconnect/versions.tf +++ b/modules/net-ipsec-over-interconnect/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-lb-app-ext-regional/versions.tf b/modules/net-lb-app-ext-regional/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-lb-app-ext-regional/versions.tf +++ b/modules/net-lb-app-ext-regional/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-lb-app-ext/versions.tf b/modules/net-lb-app-ext/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-lb-app-ext/versions.tf +++ b/modules/net-lb-app-ext/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-lb-app-int-cross-region/versions.tf b/modules/net-lb-app-int-cross-region/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-lb-app-int-cross-region/versions.tf +++ b/modules/net-lb-app-int-cross-region/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-lb-app-int/versions.tf b/modules/net-lb-app-int/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-lb-app-int/versions.tf +++ b/modules/net-lb-app-int/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-lb-ext/versions.tf b/modules/net-lb-ext/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-lb-ext/versions.tf +++ b/modules/net-lb-ext/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-lb-int/versions.tf b/modules/net-lb-int/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-lb-int/versions.tf +++ b/modules/net-lb-int/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-lb-proxy-int/versions.tf b/modules/net-lb-proxy-int/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-lb-proxy-int/versions.tf +++ b/modules/net-lb-proxy-int/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-swp/versions.tf b/modules/net-swp/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-swp/versions.tf +++ b/modules/net-swp/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-vlan-attachment/versions.tf b/modules/net-vlan-attachment/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-vlan-attachment/versions.tf +++ b/modules/net-vlan-attachment/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-vpc-firewall/README.md b/modules/net-vpc-firewall/README.md index a50570dadf..0ba7e1d3ce 100644 --- a/modules/net-vpc-firewall/README.md +++ b/modules/net-vpc-firewall/README.md @@ -272,20 +272,18 @@ module "firewall" { } # tftest modules=1 resources=3 files=lbs inventory=factory.yaml ``` - - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [network](variables.tf#L110) | Name of the network this set of firewall rules applies to. | string | ✓ | | -| [project_id](variables.tf#L115) | Project id of the project that holds the network. | string | ✓ | | +| [network](variables.tf#L111) | Name of the network this set of firewall rules applies to. | string | ✓ | | +| [project_id](variables.tf#L116) | Project id of the project that holds the network. | string | ✓ | | | [default_rules_config](variables.tf#L17) | Optionally created convenience rules. Set the 'disabled' attribute to true, or individual rule attributes to empty lists to disable. | object({…}) | | {} | | [egress_rules](variables.tf#L37) | List of egress rule definitions, default to deny action. Null destination ranges will be replaced with 0/0. | map(object({…})) | | {} | -| [factories_config](variables.tf#L60) | Paths to data files and folders that enable factory functionality. | object({…}) | | null | -| [ingress_rules](variables.tf#L69) | List of ingress rule definitions, default to allow action. Null source ranges will be replaced with 0/0. | map(object({…})) | | {} | -| [named_ranges](variables.tf#L93) | Define mapping of names to ranges that can be used in custom rules. | map(list(string)) | | {…} | +| [factories_config](variables.tf#L60) | Paths to data files and folders that enable factory functionality. | object({…}) | | {} | +| [ingress_rules](variables.tf#L70) | List of ingress rule definitions, default to allow action. Null source ranges will be replaced with 0/0. | map(object({…})) | | {} | +| [named_ranges](variables.tf#L94) | Define mapping of names to ranges that can be used in custom rules. | map(list(string)) | | {…} | ## Outputs @@ -293,5 +291,4 @@ module "firewall" { |---|---|:---:| | [default_rules](outputs.tf#L17) | Default rule resources. | | | [rules](outputs.tf#L27) | Custom rule resources. | | - diff --git a/modules/net-vpc-firewall/main.tf b/modules/net-vpc-firewall/main.tf index f2b9f0b791..57e7607bda 100644 --- a/modules/net-vpc-firewall/main.tf +++ b/modules/net-vpc-firewall/main.tf @@ -15,10 +15,11 @@ */ locals { + _factory_rules_folder = try(pathexpand(var.factories_config.rules_folder), null) # define list of rule files - _factory_rule_files = [ - for f in try(fileset(var.factories_config.rules_folder, "**/*.yaml"), []) : - "${var.factories_config.rules_folder}/${f}" + _factory_rule_files = local._factory_rules_folder == null ? [] : [ + for f in try(fileset(local._factory_rules_folder, "**/*.yaml"), []) : + "${local._factory_rules_folder}/${f}" ] # decode rule files and account for optional attributes _factory_rule_list = flatten([ @@ -47,7 +48,11 @@ locals { if contains(["EGRESS", "INGRESS"], r.direction) } _named_ranges = merge( - can(var.factories_config.cidr_tpl_file) ? var.factories_config.cidr_tpl_file != null ? yamldecode(file(var.factories_config.cidr_tpl_file)) : {} : {}, + ( + var.factories_config.cidr_tpl_file != null + ? yamldecode(pathexpand(file(var.factories_config.cidr_tpl_file))) + : {} + ), var.named_ranges ) _rules = merge( diff --git a/modules/net-vpc-firewall/variables.tf b/modules/net-vpc-firewall/variables.tf index 132f00ed9e..104f87d5b5 100644 --- a/modules/net-vpc-firewall/variables.tf +++ b/modules/net-vpc-firewall/variables.tf @@ -61,9 +61,10 @@ variable "factories_config" { description = "Paths to data files and folders that enable factory functionality." type = object({ cidr_tpl_file = optional(string) - rules_folder = string + rules_folder = optional(string) }) - default = null + nullable = false + default = {} } variable "ingress_rules" { diff --git a/modules/net-vpc-firewall/versions.tf b/modules/net-vpc-firewall/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-vpc-firewall/versions.tf +++ b/modules/net-vpc-firewall/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-vpc-peering/versions.tf b/modules/net-vpc-peering/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-vpc-peering/versions.tf +++ b/modules/net-vpc-peering/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-vpc/subnets.tf b/modules/net-vpc/subnets.tf index 1604df9b4e..93581123bc 100644 --- a/modules/net-vpc/subnets.tf +++ b/modules/net-vpc/subnets.tf @@ -18,9 +18,10 @@ locals { _factory_data = { - for f in try(fileset(var.factories_config.subnets_folder, "**/*.yaml"), []) : - trimsuffix(basename(f), ".yaml") => yamldecode(file("${var.factories_config.subnets_folder}/${f}")) + for f in try(fileset(local._factory_path, "**/*.yaml"), []) : + trimsuffix(basename(f), ".yaml") => yamldecode(file("${local._factory_path}/${f}")) } + _factory_path = try(pathexpand(var.factories_config.subnets_folder), null) _factory_subnets = { for k, v in local._factory_data : "${v.region}/${try(v.name, k)}" => { diff --git a/modules/net-vpc/versions.tf b/modules/net-vpc/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-vpc/versions.tf +++ b/modules/net-vpc/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-vpn-dynamic/versions.tf b/modules/net-vpn-dynamic/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-vpn-dynamic/versions.tf +++ b/modules/net-vpn-dynamic/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-vpn-ha/versions.tf b/modules/net-vpn-ha/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-vpn-ha/versions.tf +++ b/modules/net-vpn-ha/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/net-vpn-static/versions.tf b/modules/net-vpn-static/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/net-vpn-static/versions.tf +++ b/modules/net-vpn-static/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/organization/versions.tf b/modules/organization/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/organization/versions.tf +++ b/modules/organization/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/blueprints/factories/project-factory/README.md b/modules/project-factory/README.md similarity index 88% rename from blueprints/factories/project-factory/README.md rename to modules/project-factory/README.md index 67081df2e2..37c6244659 100644 --- a/blueprints/factories/project-factory/README.md +++ b/modules/project-factory/README.md @@ -1,31 +1,30 @@ # Project Factory -This is a working example of how to manage project creation at scale, by wrapping the [project module](../../../modules/project/) and driving it via external data, either directly provided or parsed via YAML files. +This module implements in code the end-to-end project creation process for multiple projects via YAML data configurations. -The wrapping layer around the project module is intentionally thin, so that +It supports -- all the features of the project module are available -- no "magic" or hidden side effects are implemented in code -- debugging and integration of new features is simple +- all project-level attributes exposed by the [project module](../project/), including Shared VPC host/service configuration +- optional service account creation in the project, including basic IAM grants +- KMS key encrypt/decrypt permissions for service identities in the project +- membership in VPC SC standard or bridge perimeters +- billing budgets (TODO) +- per-project IaC configuration (TODO) + +The factory is implemented as a thin wrapping layer, so that no "magic" or hidden side effects are implemented in code, and debugging or integration of new features are simple. The code is meant to be executed by a high level service accounts with powerful permissions: - Shared VPC connection if service project attachment is desired - project creation on the nodes (folder or org) where projects will be defined -The module also supports optional creation of specific resources that are usually part of the project creation flow: - -- service accounts used for VM instances, and associated basic roles -- KMS key encrypt/decrypt permissions for service identities in the project -- membership in VPC SC standard or bridge perimeters - ## Leveraging data defaults, merges, optionals -In addition to the yaml files describing projects, the project factory accepts three additional sets of inputs: +In addition to the YAML-based project configurations, the factory accepts three additional sets of inputs via Terraform variables: -- the `data_defaults` variable allows specifying defaults for specific project attributes, which are only used if the attributes are not present in a project yaml -- the `data_overrides` variable works similarly to defaults, but the values specified here take precedence over those in yaml files -- the `data_merges` variable allows specifying additional values that are merged to sets of maps present in the yaml file, which are preserved +- the `data_defaults` variable allows defining defaults for specific project attributes, which are only used if the attributes are not passed in via YAML +- the `data_overrides` variable works similarly to defaults, but the values specified here take precedence over those in YAML files +- the `data_merges` variable allows specifying additional values for map or set based variables, which are merged with the data coming from YAML Some examples on where to use each of the three sets are provided below. @@ -33,7 +32,7 @@ Some examples on where to use each of the three sets are provided below. ```hcl module "project-factory" { - source = "./fabric/blueprints/factories/project-factory" + source = "./fabric/modules/project-factory" # use a default billing account if none is specified via yaml data_defaults = { billing_account = "012345-67890A-ABCDEF" @@ -152,7 +151,7 @@ These tests validate fixes to the project factory. ```hcl module "project-factory" { - source = "./fabric/blueprints/factories/project-factory" + source = "./fabric/modules/project-factory" data_defaults = { billing_account = "012345-67890A-ABCDEF" } diff --git a/blueprints/factories/project-factory/factory.tf b/modules/project-factory/factory.tf similarity index 100% rename from blueprints/factories/project-factory/factory.tf rename to modules/project-factory/factory.tf diff --git a/blueprints/factories/project-factory/main.tf b/modules/project-factory/main.tf similarity index 96% rename from blueprints/factories/project-factory/main.tf rename to modules/project-factory/main.tf index 1cb17be850..80cc80df4d 100644 --- a/blueprints/factories/project-factory/main.tf +++ b/modules/project-factory/main.tf @@ -15,7 +15,7 @@ */ module "projects" { - source = "../../../modules/project" + source = "../project" for_each = local.projects billing_account = each.value.billing_account name = each.key @@ -65,7 +65,7 @@ module "projects" { } module "service-accounts" { - source = "../../../modules/iam-service-account" + source = "../iam-service-account" for_each = { for k in local.service_accounts : "${k.project}-${k.name}" => k } diff --git a/blueprints/factories/project-factory/outputs.tf b/modules/project-factory/outputs.tf similarity index 100% rename from blueprints/factories/project-factory/outputs.tf rename to modules/project-factory/outputs.tf diff --git a/blueprints/factories/project-factory/variables.tf b/modules/project-factory/variables.tf similarity index 100% rename from blueprints/factories/project-factory/variables.tf rename to modules/project-factory/variables.tf diff --git a/modules/project/versions.tf b/modules/project/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/project/versions.tf +++ b/modules/project/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/projects-data-source/versions.tf b/modules/projects-data-source/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/projects-data-source/versions.tf +++ b/modules/projects-data-source/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/pubsub/versions.tf b/modules/pubsub/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/pubsub/versions.tf +++ b/modules/pubsub/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/secret-manager/versions.tf b/modules/secret-manager/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/secret-manager/versions.tf +++ b/modules/secret-manager/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/service-directory/versions.tf b/modules/service-directory/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/service-directory/versions.tf +++ b/modules/service-directory/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/source-repository/versions.tf b/modules/source-repository/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/source-repository/versions.tf +++ b/modules/source-repository/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/vpc-sc/versions.tf b/modules/vpc-sc/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/vpc-sc/versions.tf +++ b/modules/vpc-sc/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/modules/workstation-cluster/versions.tf b/modules/workstation-cluster/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/modules/workstation-cluster/versions.tf +++ b/modules/workstation-cluster/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/tests/blueprints/factories/bigquery_factory/examples/simple.yaml b/tests/blueprints/factories/bigquery_factory/examples/simple.yaml deleted file mode 100644 index d32492d6c5..0000000000 --- a/tests/blueprints/factories/bigquery_factory/examples/simple.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -values: - module.bq.module.bq["my_dataset"].google_bigquery_dataset.default: - dataset_id: my_dataset - project: project-id - module.bq.module.bq["my_dataset"].google_bigquery_table.default["countries"]: - dataset_id: my_dataset - friendly_name: countries - labels: - env: prod - project: project-id - schema: '[{"name":"country","type":"STRING"},{"name":"population","type":"INT64"}]' - table_id: countries - module.bq.module.bq["my_dataset"].google_bigquery_table.views["department"]: - dataset_id: my_dataset - friendly_name: department - labels: - env: prod - project: project-id - table_id: department - view: - - query: SELECT SUM(population) from my_dataset.countries - use_legacy_sql: false - -counts: - google_bigquery_dataset: 1 - google_bigquery_table: 2 diff --git a/tests/blueprints/factories/cloud_identity_group_factory/examples/example.yaml b/tests/blueprints/factories/cloud_identity_group_factory/examples/example.yaml deleted file mode 100644 index 1a8db1b593..0000000000 --- a/tests/blueprints/factories/cloud_identity_group_factory/examples/example.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -values: - module.groups.module.group["group1@example.com"].google_cloud_identity_group.group: - description: Group 1 - display_name: Group 1 - group_key: - - id: group1@example.com - namespace: null - initial_group_config: EMPTY - labels: - cloudidentity.googleapis.com/groups.discussion_forum: '' - parent: customers/C0xxxxxxx - module.groups.module.group["group1@example.com"].google_cloud_identity_group_membership.managers["user2@example.com"]: - preferred_member_key: - - id: user2@example.com - namespace: null - roles: - - name: MANAGER - - name: MEMBER - module.groups.module.group["group1@example.com"].google_cloud_identity_group_membership.members["user1@example.com"]: - preferred_member_key: - - id: user1@example.com - namespace: null - roles: - - name: MEMBER - -counts: - google_cloud_identity_group: 1 - google_cloud_identity_group_membership: 2 diff --git a/tests/blueprints/factories/net_vpc_firewall_yaml/examples/example.yaml b/tests/blueprints/factories/net_vpc_firewall_yaml/examples/example.yaml deleted file mode 100644 index c2375ae574..0000000000 --- a/tests/blueprints/factories/net_vpc_firewall_yaml/examples/example.yaml +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright 2023 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -values: - module.dev-firewall.google_compute_firewall.rules["deny-all"]: - allow: [] - deny: - - ports: [] - protocol: all - destination_ranges: - - 0.0.0.0/0 - direction: EGRESS - disabled: null - log_config: [] - name: fwr-my-dev-network-all-e-deny-all - network: my-dev-network - priority: 65535 - project: my-dev-project - source_ranges: null - source_service_accounts: null - source_tags: null - target_service_accounts: null - target_tags: null - timeouts: null - module.dev-firewall.google_compute_firewall.rules["lb-health-checks"]: - allow: - - ports: [] - protocol: tcp - deny: [] - direction: INGRESS - disabled: null - log_config: [] - name: fwr-my-dev-network-all-i-lb-health-checks - network: my-dev-network - priority: 1001 - project: my-dev-project - source_ranges: - - 130.211.0.0/22 - - 35.191.0.0/16 - source_service_accounts: null - source_tags: null - target_service_accounts: null - target_tags: null - timeouts: null - module.dev-firewall.google_compute_firewall.rules["web-app-dev-egress"]: - allow: - - ports: - - '443' - protocol: tcp - deny: [] - destination_ranges: - - 192.168.0.0/24 - direction: EGRESS - disabled: null - log_config: [] - name: fwr-my-dev-network-sac-e-web-app-dev-egress - network: my-dev-network - priority: 1000 - project: my-dev-project - source_ranges: null - source_service_accounts: null - source_tags: null - target_service_accounts: - - myapp@myproject-dev.iam.gserviceaccount.com - target_tags: null - timeouts: null - module.dev-firewall.google_compute_firewall.rules["web-app-dev-ingress"]: - allow: - - ports: - - '1234' - protocol: tcp - deny: [] - direction: INGRESS - disabled: null - log_config: [] - name: fwr-my-dev-network-sac-i-web-app-dev-ingress - network: my-dev-network - priority: 1000 - project: my-dev-project - source_ranges: null - source_service_accounts: - - frontend-sa@myproject-dev.iam.gserviceaccount.com - source_tags: null - target_service_accounts: - - web-app-a@myproject-dev.iam.gserviceaccount.com - target_tags: null - timeouts: null - module.prod-firewall.google_compute_firewall.rules["deny-all"]: - allow: [] - deny: - - ports: [] - protocol: all - destination_ranges: - - 0.0.0.0/0 - direction: EGRESS - disabled: null - log_config: - - metadata: INCLUDE_ALL_METADATA - name: fwr-my-prod-network-all-e-deny-all - network: my-prod-network - priority: 65535 - project: my-prod-project - source_ranges: null - source_service_accounts: null - source_tags: null - target_service_accounts: null - target_tags: null - timeouts: null - module.prod-firewall.google_compute_firewall.rules["lb-health-checks"]: - allow: - - ports: [] - protocol: tcp - deny: [] - direction: INGRESS - disabled: null - log_config: - - metadata: INCLUDE_ALL_METADATA - name: fwr-my-prod-network-all-i-lb-health-checks - network: my-prod-network - priority: 1001 - project: my-prod-project - source_ranges: - - 130.211.0.0/22 - - 35.191.0.0/16 - source_service_accounts: null - source_tags: null - target_service_accounts: null - target_tags: null - timeouts: null - module.prod-firewall.google_compute_firewall.rules["web-app-prod-egress"]: - allow: - - ports: - - '443' - protocol: tcp - deny: [] - destination_ranges: - - 192.168.10.0/24 - direction: EGRESS - disabled: null - log_config: - - metadata: INCLUDE_ALL_METADATA - name: fwr-my-prod-network-sac-e-web-app-prod-egress - network: my-prod-network - priority: 1000 - project: my-prod-project - source_ranges: null - source_service_accounts: null - source_tags: null - target_service_accounts: - - myapp@myproject-prod.iam.gserviceaccount.com - target_tags: null - timeouts: null - module.prod-firewall.google_compute_firewall.rules["web-app-prod-ingress"]: - allow: - - ports: - - '1234' - protocol: tcp - deny: [] - direction: INGRESS - disabled: null - log_config: - - metadata: INCLUDE_ALL_METADATA - name: fwr-my-prod-network-sac-i-web-app-prod-ingress - network: my-prod-network - priority: 1000 - project: my-prod-project - source_ranges: null - source_service_accounts: - - frontend-sa@myproject-prod.iam.gserviceaccount.com - source_tags: null - target_service_accounts: - - web-app-a@myproject-prod.iam.gserviceaccount.com - target_tags: null - timeouts: null - -counts: - google_compute_firewall: 8 diff --git a/tests/examples_e2e/setup_module/versions.tf b/tests/examples_e2e/setup_module/versions.tf index 3db0e2076e..f43fef270d 100644 --- a/tests/examples_e2e/setup_module/versions.tf +++ b/tests/examples_e2e/setup_module/versions.tf @@ -13,15 +13,15 @@ # limitations under the License. terraform { - required_version = ">= 1.7.0" + required_version = ">= 1.7.4" required_providers { google = { source = "hashicorp/google" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.11.0, < 6.0.0" # tftest + version = ">= 5.12.0, < 6.0.0" # tftest } } } diff --git a/tests/modules/cloud_run/examples/audit-logs.yaml b/tests/modules/cloud_run/examples/audit-logs.yaml index 3b1d964dcf..3f8635cf39 100644 --- a/tests/modules/cloud_run/examples/audit-logs.yaml +++ b/tests/modules/cloud_run/examples/audit-logs.yaml @@ -14,31 +14,45 @@ values: module.cloud_run.google_cloud_run_service.service: + autogenerate_revision_name: false + location: europe-west8 + metadata: + - annotations: null + generation: 0 + labels: null name: hello project: project-id template: - - spec: - - containers: - - image: us-docker.pkg.dev/cloudrun/container/hello - + - metadata: + - {} + spec: + - containers: + - args: null + command: null + env: [] + env_from: [] + image: us-docker.pkg.dev/cloudrun/container/hello + liveness_probe: [] + volume_mounts: [] + working_dir: null + volumes: [] + timeouts: null module.cloud_run.google_cloud_run_service_iam_binding.binding["roles/run.invoker"]: condition: [] location: europe-west8 members: - - serviceAccount:eventarc-trigger@project-id.iam.gserviceaccount.com + - serviceAccount:eventarc-trigger@project-id.iam.gserviceaccount.com project: project-id role: roles/run.invoker service: hello - module.cloud_run.google_eventarc_trigger.audit_log_triggers["setiampolicy"]: + channel: null destination: - - cloud_function: null - cloud_run_service: + - cloud_run_service: - path: null region: europe-west8 service: hello - gke: [] - workflow: null + labels: null location: europe-west8 matching_criteria: - attribute: methodName @@ -52,15 +66,20 @@ values: value: google.cloud.audit.log.v1.written name: audit-log-setiampolicy project: project-id - + service_account: eventarc-trigger@project-id.iam.gserviceaccount.com + timeouts: null module.sa.google_project_iam_member.project-roles["project-id-roles/eventarc.eventReceiver"]: condition: [] project: project-id role: roles/eventarc.eventReceiver - module.sa.google_service_account.service_account[0]: account_id: eventarc-trigger + create_ignore_already_exists: null + description: null + disabled: false + display_name: Terraform-managed. project: project-id + timeouts: null counts: google_cloud_run_service: 1 @@ -68,3 +87,7 @@ counts: google_eventarc_trigger: 1 google_project_iam_member: 1 google_service_account: 1 + modules: 2 + resources: 5 + +outputs: {} diff --git a/tests/modules/cloud_run/examples/eventarc.yaml b/tests/modules/cloud_run/examples/eventarc.yaml index 961add60e2..d7c8ef9e21 100644 --- a/tests/modules/cloud_run/examples/eventarc.yaml +++ b/tests/modules/cloud_run/examples/eventarc.yaml @@ -14,21 +14,37 @@ values: module.cloud_run.google_cloud_run_service.service: + autogenerate_revision_name: false + location: europe-west8 + metadata: + - annotations: null + generation: 0 + labels: null name: hello project: project-id template: - - spec: - - containers: - - image: us-docker.pkg.dev/cloudrun/container/hello + - metadata: + - {} + spec: + - containers: + - args: null + command: null + env: [] + env_from: [] + image: us-docker.pkg.dev/cloudrun/container/hello + liveness_probe: [] + volume_mounts: [] + working_dir: null + volumes: [] + timeouts: null module.cloud_run.google_eventarc_trigger.pubsub_triggers["topic-1"]: + channel: null destination: - - cloud_function: null - cloud_run_service: + - cloud_run_service: - path: null region: europe-west8 service: hello - gke: [] - workflow: null + labels: null location: europe-west8 matching_criteria: - attribute: type @@ -36,16 +52,24 @@ values: value: google.cloud.pubsub.topic.v1.messagePublished name: pubsub-topic-1 project: project-id + service_account: null + timeouts: null transport: - pubsub: - topic: projects/project-id/topics/pubsub_sink - module.pubsub.google_pubsub_topic.default: + kms_key_name: null + labels: null + message_retention_duration: null name: pubsub_sink project: project-id - + timeouts: null counts: google_cloud_run_service: 1 google_eventarc_trigger: 1 google_pubsub_topic: 1 + modules: 2 + resources: 3 + +outputs: {} diff --git a/tests/modules/cloud_run/examples/trigger-service-account.yaml b/tests/modules/cloud_run/examples/trigger-service-account.yaml index 3877a71e0e..ca15d9fbe4 100644 --- a/tests/modules/cloud_run/examples/trigger-service-account.yaml +++ b/tests/modules/cloud_run/examples/trigger-service-account.yaml @@ -17,7 +17,9 @@ values: autogenerate_revision_name: false location: europe-west8 metadata: - - {} + - annotations: null + generation: 0 + labels: null name: hello project: project-id template: @@ -35,24 +37,19 @@ values: working_dir: null volumes: [] timeouts: null - module.cloud_run.google_cloud_run_service_iam_member.default[0]: condition: [] location: europe-west8 project: project-id role: roles/run.invoker service: hello - module.cloud_run.google_eventarc_trigger.pubsub_triggers["topic-1"]: channel: null destination: - - cloud_function: null - cloud_run_service: + - cloud_run_service: - path: null region: europe-west8 service: hello - gke: [] - workflow: null labels: null location: europe-west8 matching_criteria: @@ -65,17 +62,21 @@ values: transport: - pubsub: - topic: projects/project-id/topics/pubsub_sink - module.cloud_run.google_service_account.trigger_service_account[0]: account_id: tf-cr-trigger-hello + create_ignore_already_exists: null + description: null + disabled: false + display_name: Terraform trigger for Cloud Run hello. project: project-id - + timeouts: null module.pubsub.google_pubsub_topic.default: kms_key_name: null labels: null message_retention_duration: null name: pubsub_sink project: project-id + timeouts: null counts: google_cloud_run_service: 1 @@ -83,6 +84,7 @@ counts: google_eventarc_trigger: 1 google_pubsub_topic: 1 google_service_account: 1 + modules: 2 + resources: 5 outputs: {} - diff --git a/tests/modules/gcve_private_cloud/examples/additional-clusters.yaml b/tests/modules/gcve_private_cloud/examples/additional-clusters.yaml index 1df21f49a0..841b4095d5 100644 --- a/tests/modules/gcve_private_cloud/examples/additional-clusters.yaml +++ b/tests/modules/gcve_private_cloud/examples/additional-clusters.yaml @@ -54,7 +54,6 @@ values: network_config: - management_cidr: 192.168.0.0/24 project: gcve-test-project - type: STANDARD counts: google_vmwareengine_cluster: 2 diff --git a/tests/modules/gcve_private_cloud/examples/basic.yaml b/tests/modules/gcve_private_cloud/examples/basic.yaml index 08d7d7d377..40803f0209 100644 --- a/tests/modules/gcve_private_cloud/examples/basic.yaml +++ b/tests/modules/gcve_private_cloud/examples/basic.yaml @@ -42,7 +42,6 @@ values: network_config: - management_cidr: 192.168.0.0/24 project: gcve-test-project - type: STANDARD counts: google_vmwareengine_network: 1 diff --git a/tests/modules/gcve_private_cloud/examples/custom-management.yaml b/tests/modules/gcve_private_cloud/examples/custom-management.yaml index 444474ce7f..6c7d7268a0 100644 --- a/tests/modules/gcve_private_cloud/examples/custom-management.yaml +++ b/tests/modules/gcve_private_cloud/examples/custom-management.yaml @@ -42,7 +42,6 @@ values: network_config: - management_cidr: 192.168.0.0/24 project: gcve-test-project - type: STANDARD counts: google_vmwareengine_network: 1 diff --git a/tests/modules/gcve_private_cloud/examples/network-policy.yaml b/tests/modules/gcve_private_cloud/examples/network-policy.yaml index a35a753af3..bfd3133de1 100644 --- a/tests/modules/gcve_private_cloud/examples/network-policy.yaml +++ b/tests/modules/gcve_private_cloud/examples/network-policy.yaml @@ -42,7 +42,6 @@ values: network_config: - management_cidr: 192.168.0.0/24 project: gcve-test-project - type: STANDARD counts: google_vmwareengine_network: 1 diff --git a/tests/blueprints/factories/project_factory/examples/example.yaml b/tests/modules/project_factory/examples/example.yaml similarity index 100% rename from tests/blueprints/factories/project_factory/examples/example.yaml rename to tests/modules/project_factory/examples/example.yaml