diff --git a/modules/projects-data-source/README.md b/modules/projects-data-source/README.md index e48ac9770b..5d35f1ab0c 100644 --- a/modules/projects-data-source/README.md +++ b/modules/projects-data-source/README.md @@ -1,9 +1,14 @@ # Projects Data Source Module -This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects and folders under a specific `parent` recursively. +This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects under a specific `parent` recursively with only one API call against [Cloud Asset Inventory](https://cloud.google.com/asset-inventory) service. A good usage pattern would be when we want all the projects under a specific folder (including nested subfolders) to be included into [VPC Service Controls](../vpc-sc/). Instead of manually maintaining the list of project numbers as an input to the `vpc-sc` module we can use that module to retrieve all the project numbers dynamically. +### IAM Permissions required + +- `roles/cloudasset.viewer` on the `parent` level or above + + ## Examples ### All projects in my org @@ -14,12 +19,8 @@ module "my-org" { parent = "organizations/123456789" } -output "projects" { - value = module.my-org.projects -} - -output "folders" { - value = module.my-org.folders +output "project_numbers" { + value = module.my-org.project_numbers } # tftest skip (uses data sources) @@ -31,18 +32,46 @@ output "folders" { module "my-dev" { source = "./fabric/modules/projects-data-source" parent = "folders/123456789" - filter = "labels.env:DEV lifecycleState:ACTIVE" + query = "labels.env:DEV state:ACTIVE" } output "dev-projects" { value = module.my-dev.projects } -output "dev-folders" { - value = module.my-dev.folders +# tftest skip (uses data sources) +``` + +### Projects under org with folder/project exclusions +```hcl +module "my-filtered" { + source = "./fabric/modules/projects-data-source" + parent = "organizations/123456789" + ignore_projects = [ + "sandbox-*", # wildcard ignore + "project-full-id", # specific project id + "0123456789" # specific project number + ] + + include_projects = [ + "sandbox-114", # include specific project which was excluded by wildcard + "415216609246" # include specific project which was excluded by wildcard (by project number) + ] + + ignore_folders = [ # subfolders are ingoner as well + "343991594985", + "437102807785", + "345245235245" + ] + query = "state:ACTIVE" +} + +output "filtered-projects" { + value = module.my-filtered.projects } # tftest skip (uses data sources) + ``` @@ -50,15 +79,17 @@ output "dev-folders" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [parent](variables.tf#L23) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | ✓ | | -| [filter](variables.tf#L17) | A string filter as defined in the [REST API](https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#query-parameters). | string | | "lifecycleState:ACTIVE" | +| [parent](variables.tf#L55) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | ✓ | | +| [ignore_folders](variables.tf#L17) | A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable. | list(string) | | [] | +| [ignore_projects](variables.tf#L28) | A list of project IDs, numbers or prefixes to exclude matching projects from the module output. | list(string) | | [] | +| [include_projects](variables.tf#L41) | A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries. | list(string) | | [] | +| [query](variables.tf#L64) | A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax). | string | | "state:ACTIVE" | ## Outputs | name | description | sensitive | |---|---|:---:| -| [folders](outputs.tf#L17) | Map of folders attributes keyed by folder id. | | -| [project_numbers](outputs.tf#L22) | List of project numbers. | | -| [projects](outputs.tf#L27) | Map of projects attributes keyed by projects id. | | +| [project_numbers](outputs.tf#L17) | List of project numbers. | | +| [projects](outputs.tf#L22) | List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format. | | diff --git a/modules/projects-data-source/main.tf b/modules/projects-data-source/main.tf index 76df425e5c..6bd5631ccf 100644 --- a/modules/projects-data-source/main.tf +++ b/modules/projects-data-source/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,129 +15,27 @@ */ locals { - folders_l1_map = { for item in data.google_folders.folders_l1.folders : item.name => item } - - folders_l2_map = merge([ - for _, v in data.google_folders.folders_l2 : - { for item in v.folders : item.name => item } - ]...) - - folders_l3_map = merge([ - for _, v in data.google_folders.folders_l3 : - { for item in v.folders : item.name => item } - ]...) - - folders_l4_map = merge([ - for _, v in data.google_folders.folders_l4 : - { for item in v.folders : item.name => item } - ]...) - - folders_l5_map = merge([ - for _, v in data.google_folders.folders_l5 : - { for item in v.folders : item.name => item } - ]...) - - folders_l6_map = merge([ - for _, v in data.google_folders.folders_l6 : - { for item in v.folders : item.name => item } - ]...) - - folders_l7_map = merge([ - for _, v in data.google_folders.folders_l7 : - { for item in v.folders : item.name => item } - ]...) - - folders_l8_map = merge([ - for _, v in data.google_folders.folders_l8 : - { for item in v.folders : item.name => item } - ]...) - - folders_l9_map = merge([ - for _, v in data.google_folders.folders_l9 : - { for item in v.folders : item.name => item } - ]...) - - folders_l10_map = merge([ - for _, v in data.google_folders.folders_l10 : - { for item in v.folders : item.name => item } - ]...) - - all_folders = merge( - local.folders_l1_map, - local.folders_l2_map, - local.folders_l3_map, - local.folders_l4_map, - local.folders_l5_map, - local.folders_l6_map, - local.folders_l7_map, - local.folders_l8_map, - local.folders_l9_map, - local.folders_l10_map + _ignore_folder_numbers = [for folder_id in var.ignore_folders : trimprefix(folder_id, "folders/")] + _ignore_folders_query = join(" AND NOT folders:", concat([""], local._ignore_folder_numbers)) + query = var.query != "" ? ( + format("%s%s", var.query, local._ignore_folders_query) + ) : ( + format("%s%s", var.query, trimprefix(local._ignore_folders_query, " AND ")) ) - parent_ids = toset(concat( - [split("/", var.parent)[1]], - [for k, _ in local.all_folders : split("/", k)[1]] - )) - - projects = merge([ - for _, v in data.google_projects.projects : - { for item in v.projects : item.project_id => item } - ]...) -} - -# 10 datasources are used to cover 10 possible nested layers in GCP organization hirerarcy. -data "google_folders" "folders_l1" { - parent_id = var.parent -} - -data "google_folders" "folders_l2" { - for_each = local.folders_l1_map - parent_id = each.value.name -} - -data "google_folders" "folders_l3" { - for_each = local.folders_l2_map - parent_id = each.value.name -} - -data "google_folders" "folders_l4" { - for_each = local.folders_l3_map - parent_id = each.value.name -} - -data "google_folders" "folders_l5" { - for_each = local.folders_l4_map - parent_id = each.value.name -} - -data "google_folders" "folders_l6" { - for_each = local.folders_l5_map - parent_id = each.value.name -} - -data "google_folders" "folders_l7" { - for_each = local.folders_l6_map - parent_id = each.value.name -} - -data "google_folders" "folders_l8" { - for_each = local.folders_l7_map - parent_id = each.value.name -} - -data "google_folders" "folders_l9" { - for_each = local.folders_l8_map - parent_id = each.value.name -} - -data "google_folders" "folders_l10" { - for_each = local.folders_l9_map - parent_id = each.value.name -} - -# Getting all projects parented by any of the folders in the tree including root prg/folder provided by `parent` variable. -data "google_projects" "projects" { - for_each = local.parent_ids - filter = "parent.id:${each.value} ${var.filter}" + ignore_patterns = [for item in var.ignore_projects : "^${replace(item, "*", ".*")}$"] + ignore_regexp = length(local.ignore_patterns) > 0 ? join("|", local.ignore_patterns) : "^NO_PROJECTS_TO_IGNORE$" + projects_after_ignore = [for item in data.google_cloud_asset_resources_search_all.projects.results : item if( + length(concat(try(regexall(local.ignore_regexp, trimprefix(item.project, "projects/")), []), try(regexall(local.ignore_regexp, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")), []))) == 0 + ) || contains(var.include_projects, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")) || contains(var.include_projects, trimprefix(item.project, "projects/")) + ] +} + +data "google_cloud_asset_resources_search_all" "projects" { + provider = google-beta + scope = var.parent + asset_types = [ + "cloudresourcemanager.googleapis.com/Project" + ] + query = local.query } diff --git a/modules/projects-data-source/outputs.tf b/modules/projects-data-source/outputs.tf index b7e38ae2cf..b1710fa20b 100644 --- a/modules/projects-data-source/outputs.tf +++ b/modules/projects-data-source/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,12 @@ * limitations under the License. */ -output "folders" { - description = "Map of folders attributes keyed by folder id." - value = local.all_folders -} - output "project_numbers" { description = "List of project numbers." - value = [for _, v in local.projects : v.number] + value = [for item in local.projects_after_ignore : trimprefix(item.project, "projects/")] } output "projects" { - description = "Map of projects attributes keyed by projects id." - value = local.projects + description = "List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format." + value = local.projects_after_ignore } diff --git a/modules/projects-data-source/variables.tf b/modules/projects-data-source/variables.tf index a7f393d335..9fef35ab6c 100644 --- a/modules/projects-data-source/variables.tf +++ b/modules/projects-data-source/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,10 +14,42 @@ * limitations under the License. */ -variable "filter" { - description = "A string filter as defined in the [REST API](https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#query-parameters)." - type = string - default = "lifecycleState:ACTIVE" +variable "ignore_folders" { + description = "A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable." + type = list(string) + default = [] + # example exlusing a folder + # ignore_folders = [ + # "folders/0123456789", + # "2345678901" + # ] +} + +variable "ignore_projects" { + description = "A list of project IDs, numbers or prefixes to exclude matching projects from the module output." + type = list(string) + default = [] + # example + #ignore_projects = [ + # "dev-proj-1", + # "uat-proj-2", + # "0123456789", + # "prd-proj-*" + #] +} + +variable "include_projects" { + description = "A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries." + type = list(string) + default = [] + # example excluding all the projects starting with "prf-" except "prd-123457" + #ignore_projects = [ + # "prd-*" + #] + #include_projects = [ + # "prd-123457", + # "0123456789" + #] } variable "parent" { @@ -28,3 +60,9 @@ variable "parent" { error_message = "Parent must be of the form folders/folder_id or organizations/organization_id." } } + +variable "query" { + description = "A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax)." + type = string + default = "state:ACTIVE" +} diff --git a/modules/projects-data-source/versions.tf b/modules/projects-data-source/versions.tf index 08492c6f95..d9c1d37c73 100644 --- a/modules/projects-data-source/versions.tf +++ b/modules/projects-data-source/versions.tf @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License.