From a497aef707ee6d044f1aebca1af65728c9de141b Mon Sep 17 00:00:00 2001 From: Aleksandr Averbukh Date: Sat, 18 Feb 2023 21:36:51 +0100 Subject: [PATCH 1/5] feat: new version of projects-data-source based on AssetInventory ds --- modules/projects-data-source/README.md | 60 ++++++--- modules/projects-data-source/main.tf | 146 ++++------------------ modules/projects-data-source/outputs.tf | 13 +- modules/projects-data-source/variables.tf | 52 ++++++-- modules/projects-data-source/versions.tf | 2 +- 5 files changed, 118 insertions(+), 155 deletions(-) diff --git a/modules/projects-data-source/README.md b/modules/projects-data-source/README.md index 6fd7dd8abf..d447692315 100644 --- a/modules/projects-data-source/README.md +++ b/modules/projects-data-source/README.md @@ -1,9 +1,14 @@ # Projects Data Source Module -This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects and folders under a specific `parent` recursively. +This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects under a specific `parent` recursively with only one API call against [Cloud Asset Inventory](https://cloud.google.com/asset-inventory) service. A good usage pattern would be when we want all the projects under a specific folder (including nested subfolders) to be included into [VPC Service Controls](../vpc-sc/). Instead of manually maintaining the list of project numbers as an input to the `vpc-sc` module we can use that module to retrieve all the project numbers dynamically. +### IAM Permissions required + +- `roles/cloudasset.viewer` on the `parent` level or above + + ## Examples ### All projects in my org @@ -15,11 +20,7 @@ module "my-org" { } output "projects" { - value = module.my-org.projects -} - -output "folders" { - value = module.my-org.folders + value = module.my-org.projects_numbers } # tftest skip (uses data sources) @@ -31,34 +32,65 @@ output "folders" { module "my-dev" { source = "./fabric/modules/projects-data-source" parent = "folders/123456789" - filter = "labels.env:DEV lifecycleState:ACTIVE" + query = "labels.env:DEV state:ACTIVE" } output "dev-projects" { value = module.my-dev.projects } -output "dev-folders" { - value = module.my-dev.folders +# tftest skip (uses data sources) +``` + +### Projects under org with folder/project exclusions +```hcl +module "my-filtered" { + source = "./fabric/modules/projects-data-source" + parent = "organizations/123456789" + ignore_projects = [ + "sandbox-*", # wildcard ignore + "project-full-id", # specific project id + "0123456789" # specific project number + ] + + include_projects = [ + "sandbox-114", # include specific project which was excluded by wildcard + "415216609246" # include specific project which was excluded by wildcard (by project number) + ] + + ignore_folders = [ # subfolders are ingoner as well + "343991594985", + "437102807785", + "345245235245" + ] + query = "state:ACTIVE" +} + +output "filtered-projects" { + value = module.my-filtered.projects } # tftest skip (uses data sources) + ``` + ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [parent](variables.tf#L23) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | ✓ | | -| [filter](variables.tf#L17) | A string filter as defined in the [REST API](https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#query-parameters). | string | | "lifecycleState:ACTIVE" | +| [parent](variables.tf#L17) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | ✓ | | +| [ignore_folders](variables.tf#L58) | A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable. | list(string) | | [] | +| [ignore_projects](variables.tf#L32) | A list of project IDs, numbers or prefixes to exclude matching projects from the module output. | list(string) | | [] | +| [include_projects](variables.tf#L44) | A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries. | list(string) | | [] | +| [query](variables.tf#L26) | A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax). | string | | "state:ACTIVE" | ## Outputs | name | description | sensitive | |---|---|:---:| -| [folders](outputs.tf#L17) | Map of folders attributes keyed by folder id. | | -| [project_numbers](outputs.tf#L22) | List of project numbers. | | -| [projects](outputs.tf#L27) | Map of projects attributes keyed by projects id. | | +| [project_numbers](outputs.tf#L17) | List of project numbers. | | +| [projects](outputs.tf#L22) | List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format. | | diff --git a/modules/projects-data-source/main.tf b/modules/projects-data-source/main.tf index 76df425e5c..5bb16b9dd0 100644 --- a/modules/projects-data-source/main.tf +++ b/modules/projects-data-source/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,129 +15,27 @@ */ locals { - folders_l1_map = { for item in data.google_folders.folders_l1.folders : item.name => item } - - folders_l2_map = merge([ - for _, v in data.google_folders.folders_l2 : - { for item in v.folders : item.name => item } - ]...) - - folders_l3_map = merge([ - for _, v in data.google_folders.folders_l3 : - { for item in v.folders : item.name => item } - ]...) - - folders_l4_map = merge([ - for _, v in data.google_folders.folders_l4 : - { for item in v.folders : item.name => item } - ]...) - - folders_l5_map = merge([ - for _, v in data.google_folders.folders_l5 : - { for item in v.folders : item.name => item } - ]...) - - folders_l6_map = merge([ - for _, v in data.google_folders.folders_l6 : - { for item in v.folders : item.name => item } - ]...) - - folders_l7_map = merge([ - for _, v in data.google_folders.folders_l7 : - { for item in v.folders : item.name => item } - ]...) - - folders_l8_map = merge([ - for _, v in data.google_folders.folders_l8 : - { for item in v.folders : item.name => item } - ]...) - - folders_l9_map = merge([ - for _, v in data.google_folders.folders_l9 : - { for item in v.folders : item.name => item } - ]...) - - folders_l10_map = merge([ - for _, v in data.google_folders.folders_l10 : - { for item in v.folders : item.name => item } - ]...) - - all_folders = merge( - local.folders_l1_map, - local.folders_l2_map, - local.folders_l3_map, - local.folders_l4_map, - local.folders_l5_map, - local.folders_l6_map, - local.folders_l7_map, - local.folders_l8_map, - local.folders_l9_map, - local.folders_l10_map + _ignore_folder_numbers = [for folder_id in var.ignore_folders: trimprefix(folder_id, "folders/")] + _ignore_folders_query = join(" AND NOT folders:", concat([""], local._ignore_folder_numbers)) + query = var.query != "" ? ( + format("%s%s", var.query, local._ignore_folders_query) + ) : ( + format("%s%s", var.query, trimprefix(local._ignore_folders_query, " AND ")) ) - parent_ids = toset(concat( - [split("/", var.parent)[1]], - [for k, _ in local.all_folders : split("/", k)[1]] - )) - - projects = merge([ - for _, v in data.google_projects.projects : - { for item in v.projects : item.project_id => item } - ]...) -} - -# 10 datasources are used to cover 10 possible nested layers in GCP organization hirerarcy. -data "google_folders" "folders_l1" { - parent_id = var.parent -} - -data "google_folders" "folders_l2" { - for_each = local.folders_l1_map - parent_id = each.value.name -} - -data "google_folders" "folders_l3" { - for_each = local.folders_l2_map - parent_id = each.value.name -} - -data "google_folders" "folders_l4" { - for_each = local.folders_l3_map - parent_id = each.value.name -} - -data "google_folders" "folders_l5" { - for_each = local.folders_l4_map - parent_id = each.value.name -} - -data "google_folders" "folders_l6" { - for_each = local.folders_l5_map - parent_id = each.value.name -} - -data "google_folders" "folders_l7" { - for_each = local.folders_l6_map - parent_id = each.value.name -} - -data "google_folders" "folders_l8" { - for_each = local.folders_l7_map - parent_id = each.value.name -} - -data "google_folders" "folders_l9" { - for_each = local.folders_l8_map - parent_id = each.value.name -} - -data "google_folders" "folders_l10" { - for_each = local.folders_l9_map - parent_id = each.value.name -} - -# Getting all projects parented by any of the folders in the tree including root prg/folder provided by `parent` variable. -data "google_projects" "projects" { - for_each = local.parent_ids - filter = "parent.id:${each.value} ${var.filter}" + ignore_patterns = [for item in var.ignore_projects: "^${replace(item, "*", ".*")}$"] + ignore_regexp = length(local.ignore_patterns) > 0 ? join("|", local.ignore_patterns) : "^NO_PROJECTS_TO_IGNORE$" + projects_after_ignore = [ for item in data.google_cloud_asset_resources_search_all.projects.results : item if ( + length(concat(try(regexall(local.ignore_regexp, trimprefix(item.project, "projects/")), []), try(regexall(local.ignore_regexp, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")), []))) == 0 + ) || contains(var.include_projects, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")) || contains(var.include_projects, trimprefix(item.project, "projects/")) + ] +} + +data google_cloud_asset_resources_search_all projects { + provider = google-beta + scope = var.parent + asset_types = [ + "cloudresourcemanager.googleapis.com/Project" + ] + query = local.query } diff --git a/modules/projects-data-source/outputs.tf b/modules/projects-data-source/outputs.tf index b7e38ae2cf..b1710fa20b 100644 --- a/modules/projects-data-source/outputs.tf +++ b/modules/projects-data-source/outputs.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,17 +14,12 @@ * limitations under the License. */ -output "folders" { - description = "Map of folders attributes keyed by folder id." - value = local.all_folders -} - output "project_numbers" { description = "List of project numbers." - value = [for _, v in local.projects : v.number] + value = [for item in local.projects_after_ignore : trimprefix(item.project, "projects/")] } output "projects" { - description = "Map of projects attributes keyed by projects id." - value = local.projects + description = "List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format." + value = local.projects_after_ignore } diff --git a/modules/projects-data-source/variables.tf b/modules/projects-data-source/variables.tf index a7f393d335..3273959ad8 100644 --- a/modules/projects-data-source/variables.tf +++ b/modules/projects-data-source/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2023 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,12 +14,6 @@ * limitations under the License. */ -variable "filter" { - description = "A string filter as defined in the [REST API](https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#query-parameters)." - type = string - default = "lifecycleState:ACTIVE" -} - variable "parent" { description = "Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format." type = string @@ -28,3 +22,47 @@ variable "parent" { error_message = "Parent must be of the form folders/folder_id or organizations/organization_id." } } + +variable "query" { + description = "A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax)." + type = string + default = "state:ACTIVE" +} + +variable "ignore_projects" { + description = "A list of project IDs, numbers or prefixes to exclude matching projects from the module output." + type = list(string) + default = [] + # example + #ignore_projects = [ + # "dev-proj-1", + # "uat-proj-2", + # "0123456789", + # "prd-proj-*" + #] +} +variable "include_projects" { + description = "A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries." + type = list(string) + default = [] + # example excluding all the projects starting with "prf-" except "prd-123457" + #ignore_projects = [ + # "prd-*" + #] + #include_projects = [ + # "prd-123457", + # "0123456789" + #] +} + +variable "ignore_folders" { + description = "A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable." + type = list(string) + default = [] + # example exlusing a folder + # ignore_folders = [ + # "folders/0123456789", + # "2345678901" + # ] +} + diff --git a/modules/projects-data-source/versions.tf b/modules/projects-data-source/versions.tf index 286536a65e..23f38edbce 100644 --- a/modules/projects-data-source/versions.tf +++ b/modules/projects-data-source/versions.tf @@ -1,4 +1,4 @@ -# Copyright 2022 Google LLC +# Copyright 2023 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 1c302c7ab331afbe32b78655aea4a1b02d6e4efb Mon Sep 17 00:00:00 2001 From: Aleksandr Averbukh Date: Sat, 18 Feb 2023 21:48:05 +0100 Subject: [PATCH 2/5] TF formatting --- modules/projects-data-source/main.tf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modules/projects-data-source/main.tf b/modules/projects-data-source/main.tf index 5bb16b9dd0..6bd5631ccf 100644 --- a/modules/projects-data-source/main.tf +++ b/modules/projects-data-source/main.tf @@ -15,25 +15,25 @@ */ locals { - _ignore_folder_numbers = [for folder_id in var.ignore_folders: trimprefix(folder_id, "folders/")] - _ignore_folders_query = join(" AND NOT folders:", concat([""], local._ignore_folder_numbers)) + _ignore_folder_numbers = [for folder_id in var.ignore_folders : trimprefix(folder_id, "folders/")] + _ignore_folders_query = join(" AND NOT folders:", concat([""], local._ignore_folder_numbers)) query = var.query != "" ? ( format("%s%s", var.query, local._ignore_folders_query) - ) : ( + ) : ( format("%s%s", var.query, trimprefix(local._ignore_folders_query, " AND ")) ) - ignore_patterns = [for item in var.ignore_projects: "^${replace(item, "*", ".*")}$"] + ignore_patterns = [for item in var.ignore_projects : "^${replace(item, "*", ".*")}$"] ignore_regexp = length(local.ignore_patterns) > 0 ? join("|", local.ignore_patterns) : "^NO_PROJECTS_TO_IGNORE$" - projects_after_ignore = [ for item in data.google_cloud_asset_resources_search_all.projects.results : item if ( - length(concat(try(regexall(local.ignore_regexp, trimprefix(item.project, "projects/")), []), try(regexall(local.ignore_regexp, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")), []))) == 0 + projects_after_ignore = [for item in data.google_cloud_asset_resources_search_all.projects.results : item if( + length(concat(try(regexall(local.ignore_regexp, trimprefix(item.project, "projects/")), []), try(regexall(local.ignore_regexp, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")), []))) == 0 ) || contains(var.include_projects, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")) || contains(var.include_projects, trimprefix(item.project, "projects/")) ] } -data google_cloud_asset_resources_search_all projects { +data "google_cloud_asset_resources_search_all" "projects" { provider = google-beta - scope = var.parent + scope = var.parent asset_types = [ "cloudresourcemanager.googleapis.com/Project" ] From 8174890331d3204adc770d0cbdf001aad59ba354 Mon Sep 17 00:00:00 2001 From: Aleksandr Averbukh Date: Sat, 18 Feb 2023 22:04:40 +0100 Subject: [PATCH 3/5] Sort variables --- modules/projects-data-source/variables.tf | 44 +++++++++++------------ 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/modules/projects-data-source/variables.tf b/modules/projects-data-source/variables.tf index 3273959ad8..9fef35ab6c 100644 --- a/modules/projects-data-source/variables.tf +++ b/modules/projects-data-source/variables.tf @@ -14,19 +14,15 @@ * limitations under the License. */ -variable "parent" { - description = "Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format." - type = string - validation { - condition = can(regex("(organizations|folders)/[0-9]+", var.parent)) - error_message = "Parent must be of the form folders/folder_id or organizations/organization_id." - } -} - -variable "query" { - description = "A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax)." - type = string - default = "state:ACTIVE" +variable "ignore_folders" { + description = "A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable." + type = list(string) + default = [] + # example exlusing a folder + # ignore_folders = [ + # "folders/0123456789", + # "2345678901" + # ] } variable "ignore_projects" { @@ -41,6 +37,7 @@ variable "ignore_projects" { # "prd-proj-*" #] } + variable "include_projects" { description = "A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries." type = list(string) @@ -55,14 +52,17 @@ variable "include_projects" { #] } -variable "ignore_folders" { - description = "A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable." - type = list(string) - default = [] - # example exlusing a folder - # ignore_folders = [ - # "folders/0123456789", - # "2345678901" - # ] +variable "parent" { + description = "Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format." + type = string + validation { + condition = can(regex("(organizations|folders)/[0-9]+", var.parent)) + error_message = "Parent must be of the form folders/folder_id or organizations/organization_id." + } } +variable "query" { + description = "A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax)." + type = string + default = "state:ACTIVE" +} From ff4b2fffe274d7b58dbc2f75477a03ccbc5b3198 Mon Sep 17 00:00:00 2001 From: Aleksandr Averbukh Date: Sat, 18 Feb 2023 22:11:44 +0100 Subject: [PATCH 4/5] Regenerate docs --- modules/projects-data-source/README.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/modules/projects-data-source/README.md b/modules/projects-data-source/README.md index d447692315..8fcfad968f 100644 --- a/modules/projects-data-source/README.md +++ b/modules/projects-data-source/README.md @@ -73,18 +73,17 @@ output "filtered-projects" { # tftest skip (uses data sources) ``` - ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [parent](variables.tf#L17) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | ✓ | | -| [ignore_folders](variables.tf#L58) | A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable. | list(string) | | [] | -| [ignore_projects](variables.tf#L32) | A list of project IDs, numbers or prefixes to exclude matching projects from the module output. | list(string) | | [] | -| [include_projects](variables.tf#L44) | A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries. | list(string) | | [] | -| [query](variables.tf#L26) | A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax). | string | | "state:ACTIVE" | +| [parent](variables.tf#L55) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | string | ✓ | | +| [ignore_folders](variables.tf#L17) | A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable. | list(string) | | [] | +| [ignore_projects](variables.tf#L28) | A list of project IDs, numbers or prefixes to exclude matching projects from the module output. | list(string) | | [] | +| [include_projects](variables.tf#L41) | A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries. | list(string) | | [] | +| [query](variables.tf#L64) | A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax). | string | | "state:ACTIVE" | ## Outputs From f16511b8da56c60c19b0c4439f906d3d1950c241 Mon Sep 17 00:00:00 2001 From: Aleksandr Averbukh Date: Sun, 19 Feb 2023 15:22:48 +0100 Subject: [PATCH 5/5] Fix typo in readme --- modules/projects-data-source/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/projects-data-source/README.md b/modules/projects-data-source/README.md index 8fcfad968f..5d35f1ab0c 100644 --- a/modules/projects-data-source/README.md +++ b/modules/projects-data-source/README.md @@ -19,8 +19,8 @@ module "my-org" { parent = "organizations/123456789" } -output "projects" { - value = module.my-org.projects_numbers +output "project_numbers" { + value = module.my-org.project_numbers } # tftest skip (uses data sources)