From 801efbe5e802f706f1e462e6e549420c3b9cfe41 Mon Sep 17 00:00:00 2001 From: Dinesh Kumar Baskar Date: Wed, 1 Nov 2023 19:12:22 +0100 Subject: [PATCH 1/5] Folder module test changes --- .gitignore | 5 +- modules/folder/README.md | 110 +++++++++++++++++++++++++++++---------- 2 files changed, 87 insertions(+), 28 deletions(-) diff --git a/.gitignore b/.gitignore index a4e6b621fd..b0abda50fa 100644 --- a/.gitignore +++ b/.gitignore @@ -56,4 +56,7 @@ blueprints/gke/autopilot/ansible/gssh.sh blueprints/gke/autopilot/ansible/vars/vars.yaml blueprints/gke/autopilot/bundle/monitoring/kustomization.yaml blueprints/gke/autopilot/bundle/locust/kustomization.yaml -blueprints/gke/autopilot/bundle.tar.gz \ No newline at end of file +blueprints/gke/autopilot/bundle.tar.gz +**/tmplv* +venv/ +**/e2e_tests.tfvars \ No newline at end of file diff --git a/modules/folder/README.md b/modules/folder/README.md index 8c75cadd5e..d56f997e6b 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -21,10 +21,10 @@ This module allows the creation and management of folders, including support for ```hcl module "folder" { source = "./fabric/modules/folder" - parent = "organizations/1234567890" + parent = var.folder_id name = "Folder name" group_iam = { - "cloud-owners@example.org" = [ + "${var.group_email}" = [ "roles/owner", "roles/resourcemanager.folderAdmin", "roles/resourcemanager.projectCreator" @@ -35,12 +35,12 @@ module "folder" { } iam_bindings_additive = { am1-storage-admin = { - member = "user:am1@example.org" + member = "user:one@example.org" role = "roles/storage.admin" } } } -# tftest modules=1 resources=5 inventory=iam.yaml +# tftest modules=1 resources=5 inventory=iam.yaml e2e ``` ## IAM @@ -62,7 +62,7 @@ To manage organization policies, the `orgpolicy.googleapis.com` service should b ```hcl module "folder" { source = "./fabric/modules/folder" - parent = "organizations/1234567890" + parent = var.folder_id name = "Folder name" org_policies = { "compute.disableGuestAttributesAccess" = { @@ -109,13 +109,69 @@ module "folder" { } } } -# tftest modules=1 resources=8 inventory=org-policies.yaml +# tftest modules=1 resources=8 inventory=org-policies.yaml e2e ``` ### Organization Policy Factory -See the [organization policy factory in the project module](../project#organization-policy-factory). +Organization policies can be loaded from a directory containing YAML files where each file defines one or more constraints. The structure of the YAML files is exactly the same as the org_policies variable. +Note that constraints defined via org_policies take precedence over those in org_policies_data_path. In other words, if you specify the same constraint in a YAML file and in the org_policies variable, the latter will take priority. + +```hcl +module "folder" { + source = "./fabric/modules/folder" + parent = var.folder_id + name = "Folder name" + org_policies_data_path = "configs/org-policies/" +} + +# tftest modules=1 resources=8 files=boolean,list inventory=org-policies.yaml e2e +``` + +```yaml +# tftest-file id=boolean path=configs/org-policies/boolean.yaml +compute.disableGuestAttributesAccess: + rules: + - enforce: true +compute.skipDefaultNetworkCreation: + rules: + - enforce: true +iam.disableServiceAccountKeyCreation: + rules: + - enforce: true +iam.disableServiceAccountKeyUpload: + rules: + - condition: + description: test condition + expression: resource.matchTagId('tagKeys/1234', 'tagValues/1234') + location: somewhere + title: condition + enforce: true + - enforce: false +``` + +```yaml +# tftest-file id=list path=configs/org-policies/list.yaml +compute.trustedImageProjects: + rules: + - allow: + values: + - projects/my-project +compute.vmExternalIpAccess: + rules: + - deny: + all: true +iam.allowedPolicyMemberDomains: + rules: + - allow: + values: + - C0xxxxxxx + - C0yyyyyyy +``` + + +The example below deploys a few organization policies split between two YAML files. ## Hierarchical Firewall Policy Attachments Hierarchical firewall policies can be managed via the [`net-firewall-policy`](../net-firewall-policy/) module, including support for factories. Once a policy is available, attaching it to the organization can be done either in the firewall policy module itself, or here: @@ -133,7 +189,7 @@ module "firewall-policy" { module "folder" { source = "./fabric/modules/folder" - parent = "organizations/1234567890" + parent = var.folder_id name = "Folder name" # attachment via the organization module firewall_policy = { @@ -141,42 +197,42 @@ module "folder" { policy = module.firewall-policy.id } } -# tftest modules=2 resources=3 +# tftest modules=2 resources=3 e2e ``` - ## Log Sinks ```hcl module "gcs" { source = "./fabric/modules/gcs" - project_id = "my-project" + project_id = var.project_id name = "gcs_sink" + prefix = "test" force_destroy = true } module "dataset" { source = "./fabric/modules/bigquery-dataset" - project_id = "my-project" + project_id = var.project_id id = "bq_sink" } module "pubsub" { source = "./fabric/modules/pubsub" - project_id = "my-project" + project_id = var.project_id name = "pubsub_sink" } module "bucket" { source = "./fabric/modules/logging-bucket" parent_type = "project" - parent = "my-project" + parent = var.project_id id = "bucket" } module "folder-sink" { source = "./fabric/modules/folder" - parent = "folders/657104291943" - name = "my-folder" + folder_create = "false" + id = var.folder_id logging_sinks = { warnings = { destination = module.gcs.id @@ -206,7 +262,7 @@ module "folder-sink" { no-gce-instances = "resource.type=gce_instance" } } -# tftest modules=5 resources=14 inventory=logging.yaml +# tftest modules=5 resources=14 inventory=logging.yaml e2e ``` ## Data Access Logs @@ -218,12 +274,12 @@ This example shows how to set a non-authoritative access log configuration: ```hcl module "folder" { source = "./fabric/modules/folder" - parent = "folders/657104291943" - name = "my-folder" + parent = var.folder_id + name = "Folder name" logging_data_access = { allServices = { # logs for principals listed here will be excluded - ADMIN_READ = ["group:organization-admins@example.org"] + ADMIN_READ = ["group:${var.group_email}"] } "storage.googleapis.com" = { DATA_READ = [] @@ -231,7 +287,7 @@ module "folder" { } } } -# tftest modules=1 resources=3 inventory=logging-data-access.yaml +# tftest modules=1 resources=3 inventory=logging-data-access.yaml e2e ``` ## Tags @@ -243,7 +299,7 @@ module "org" { source = "./fabric/modules/organization" organization_id = var.organization_id tags = { - environment = { + environments = { description = "Environment specification." iam = null values = { @@ -256,14 +312,14 @@ module "org" { module "folder" { source = "./fabric/modules/folder" - name = "Test" - parent = module.org.organization_id + name = "Folder name" + parent = var.folder_id tag_bindings = { - env-prod = module.org.tag_values["environment/prod"].id - foo = "tagValues/12345678" + env-prod = module.org.tag_values["environments/prod"].id + # foo = "tagValues/12345678" } } -# tftest modules=2 resources=6 inventory=tags.yaml +# tftest modules=2 resources=6 inventory=tags.yaml e2e ``` From 74de43c97ef5fd55a340aed51d8e40a28ef585b5 Mon Sep 17 00:00:00 2001 From: Dinesh Kumar Baskar Date: Wed, 1 Nov 2023 19:18:14 +0100 Subject: [PATCH 2/5] Folder module test format changes --- modules/folder/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/folder/README.md b/modules/folder/README.md index d56f997e6b..8b2dd4dee8 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -118,6 +118,8 @@ Organization policies can be loaded from a directory containing YAML files where Note that constraints defined via org_policies take precedence over those in org_policies_data_path. In other words, if you specify the same constraint in a YAML file and in the org_policies variable, the latter will take priority. +The example below deploys a few organization policies split between two YAML files. + ```hcl module "folder" { source = "./fabric/modules/folder" @@ -170,8 +172,6 @@ iam.allowedPolicyMemberDomains: - C0yyyyyyy ``` - -The example below deploys a few organization policies split between two YAML files. ## Hierarchical Firewall Policy Attachments Hierarchical firewall policies can be managed via the [`net-firewall-policy`](../net-firewall-policy/) module, including support for factories. Once a policy is available, attaching it to the organization can be done either in the firewall policy module itself, or here: From 31a566dbcf0ad095e9e9e05503da1f46216d8040 Mon Sep 17 00:00:00 2001 From: Dinesh Kumar Baskar Date: Tue, 7 Nov 2023 17:20:42 +0100 Subject: [PATCH 3/5] updated yaml for example folder module tests --- modules/folder/README.md | 22 +++++++++---------- tests/examples/variables.tf | 7 +++++- tests/modules/folder/examples/iam.yaml | 12 +++++----- .../folder/examples/logging-data-access.yaml | 6 ++--- tests/modules/folder/examples/logging.yaml | 8 +++---- .../modules/folder/examples/org-policies.yaml | 2 +- tests/modules/folder/examples/tags.yaml | 6 ++--- 7 files changed, 33 insertions(+), 30 deletions(-) diff --git a/modules/folder/README.md b/modules/folder/README.md index 8b2dd4dee8..c9da986401 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -31,11 +31,11 @@ module "folder" { ] } iam = { - "roles/owner" = ["user:one@example.org"] + "roles/owner" = ["serviceAccount:${var.service_account.email}"] } iam_bindings_additive = { am1-storage-admin = { - member = "user:one@example.org" + member = "serviceAccount:${var.service_account.email}" role = "roles/storage.admin" } } @@ -122,12 +122,11 @@ The example below deploys a few organization policies split between two YAML fil ```hcl module "folder" { - source = "./fabric/modules/folder" - parent = var.folder_id - name = "Folder name" + source = "./fabric/modules/folder" + parent = var.folder_id + name = "Folder name" org_policies_data_path = "configs/org-policies/" } - # tftest modules=1 resources=8 files=boolean,list inventory=org-policies.yaml e2e ``` @@ -231,8 +230,8 @@ module "bucket" { module "folder-sink" { source = "./fabric/modules/folder" - folder_create = "false" - id = var.folder_id + name = "Folder name" + parent = var.folder_id logging_sinks = { warnings = { destination = module.gcs.id @@ -299,7 +298,7 @@ module "org" { source = "./fabric/modules/organization" organization_id = var.organization_id tags = { - environments = { + environment = { description = "Environment specification." iam = null values = { @@ -315,11 +314,10 @@ module "folder" { name = "Folder name" parent = var.folder_id tag_bindings = { - env-prod = module.org.tag_values["environments/prod"].id - # foo = "tagValues/12345678" + env-prod = module.org.tag_values["environment/prod"].id } } -# tftest modules=2 resources=6 inventory=tags.yaml e2e +# tftest modules=2 resources=5 inventory=tags.yaml e2e ``` diff --git a/tests/examples/variables.tf b/tests/examples/variables.tf index ebdbb155cb..bba4132c17 100644 --- a/tests/examples/variables.tf +++ b/tests/examples/variables.tf @@ -52,6 +52,10 @@ variable "region" { default = "region" } +variable "group_email" { + default = "group_email" +} + variable "service_account" { default = { id = "service_account_id" @@ -93,4 +97,5 @@ variable "vpc2" { variable "zone" { default = "zone" -} +} + diff --git a/tests/modules/folder/examples/iam.yaml b/tests/modules/folder/examples/iam.yaml index a1c06e4688..7b849dfc39 100644 --- a/tests/modules/folder/examples/iam.yaml +++ b/tests/modules/folder/examples/iam.yaml @@ -15,27 +15,27 @@ values: module.folder.google_folder.folder[0]: display_name: Folder name - parent: organizations/1234567890 + parent: folders/1122334455 timeouts: null module.folder.google_folder_iam_binding.authoritative["roles/owner"]: condition: [] members: - - group:cloud-owners@example.org - - user:one@example.org + - group:group_email + - serviceAccount:service_account_email role: roles/owner module.folder.google_folder_iam_binding.authoritative["roles/resourcemanager.folderAdmin"]: condition: [] members: - - group:cloud-owners@example.org + - group:group_email role: roles/resourcemanager.folderAdmin module.folder.google_folder_iam_binding.authoritative["roles/resourcemanager.projectCreator"]: condition: [] members: - - group:cloud-owners@example.org + - group:group_email role: roles/resourcemanager.projectCreator module.folder.google_folder_iam_member.bindings["am1-storage-admin"]: condition: [] - member: user:am1@example.org + member: serviceAccount:service_account_email role: roles/storage.admin counts: diff --git a/tests/modules/folder/examples/logging-data-access.yaml b/tests/modules/folder/examples/logging-data-access.yaml index 60b387258e..2c6d464191 100644 --- a/tests/modules/folder/examples/logging-data-access.yaml +++ b/tests/modules/folder/examples/logging-data-access.yaml @@ -14,13 +14,13 @@ values: module.folder.google_folder.folder[0]: - display_name: my-folder - parent: folders/657104291943 + display_name: Folder name + parent: folders/1122334455 timeouts: null module.folder.google_folder_iam_audit_config.default["allServices"]: audit_log_config: - exempted_members: - - group:organization-admins@example.org + - group:group_email log_type: ADMIN_READ service: allServices module.folder.google_folder_iam_audit_config.default["storage.googleapis.com"]: diff --git a/tests/modules/folder/examples/logging.yaml b/tests/modules/folder/examples/logging.yaml index b55b4bbb63..263ca74e10 100644 --- a/tests/modules/folder/examples/logging.yaml +++ b/tests/modules/folder/examples/logging.yaml @@ -16,8 +16,8 @@ values: module.folder-sink.google_bigquery_dataset_iam_member.bq-sinks-binding["info"]: role: roles/bigquery.dataEditor module.folder-sink.google_folder.folder[0]: - display_name: my-folder - parent: folders/657104291943 + display_name: Folder name + parent: folders/1122334455 module.folder-sink.google_logging_folder_exclusion.logging-exclusion["no-gce-instances"]: description: no-gce-instances (Terraform-managed). filter: resource.type=gce_instance @@ -46,7 +46,7 @@ values: name: notice module.folder-sink.google_logging_folder_sink.sink["warnings"]: description: warnings (Terraform-managed). - destination: storage.googleapis.com/gcs_sink + destination: storage.googleapis.com/test-gcs_sink disabled: false exclusions: [] filter: severity=WARNING @@ -60,7 +60,7 @@ values: condition: [] role: roles/pubsub.publisher module.folder-sink.google_storage_bucket_iam_member.gcs-sinks-binding["warnings"]: - bucket: gcs_sink + bucket: test-gcs_sink condition: [] role: roles/storage.objectCreator diff --git a/tests/modules/folder/examples/org-policies.yaml b/tests/modules/folder/examples/org-policies.yaml index c7bee1239e..6bfe6fcdb1 100644 --- a/tests/modules/folder/examples/org-policies.yaml +++ b/tests/modules/folder/examples/org-policies.yaml @@ -15,7 +15,7 @@ values: module.folder.google_folder.folder[0]: display_name: Folder name - parent: organizations/1234567890 + parent: folders/1122334455 module.folder.google_org_policy_policy.default["compute.disableGuestAttributesAccess"]: spec: - inherit_from_parent: null diff --git a/tests/modules/folder/examples/tags.yaml b/tests/modules/folder/examples/tags.yaml index 1451b6d1c7..11c390eb78 100644 --- a/tests/modules/folder/examples/tags.yaml +++ b/tests/modules/folder/examples/tags.yaml @@ -16,13 +16,13 @@ tests/examples/test_plan.py::test_example[modules/folder:Tags] values: module.folder.google_folder.folder[0]: display_name: Test - parent: organizations/1122334455 + parent: folders/1122334455 module.folder.google_tags_tag_binding.binding["env-prod"]: {} module.folder.google_tags_tag_binding.binding["foo"]: tag_value: tagValues/12345678 module.org.google_tags_tag_key.default["environment"]: description: Environment specification. - parent: organizations/1122334455 + parent: folders/1122334455 purpose: null purpose_data: null short_name: environment @@ -36,6 +36,6 @@ tests/examples/test_plan.py::test_example[modules/folder:Tags] values: counts: google_folder: 1 - google_tags_tag_binding: 2 + google_tags_tag_binding: 1 google_tags_tag_key: 1 google_tags_tag_value: 2 From b9f2b09e13c2af0548011fbb9437e318901d08fe Mon Sep 17 00:00:00 2001 From: Dinesh Kumar Baskar Date: Tue, 21 Nov 2023 17:54:45 +0100 Subject: [PATCH 4/5] fixed example testing --- .gitignore | 5 +---- tests/examples/variables.tf | 7 +------ tests/modules/folder/examples/iam.yaml | 6 +++--- tests/modules/folder/examples/logging-data-access.yaml | 2 +- 4 files changed, 6 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index b0abda50fa..a4e6b621fd 100644 --- a/.gitignore +++ b/.gitignore @@ -56,7 +56,4 @@ blueprints/gke/autopilot/ansible/gssh.sh blueprints/gke/autopilot/ansible/vars/vars.yaml blueprints/gke/autopilot/bundle/monitoring/kustomization.yaml blueprints/gke/autopilot/bundle/locust/kustomization.yaml -blueprints/gke/autopilot/bundle.tar.gz -**/tmplv* -venv/ -**/e2e_tests.tfvars \ No newline at end of file +blueprints/gke/autopilot/bundle.tar.gz \ No newline at end of file diff --git a/tests/examples/variables.tf b/tests/examples/variables.tf index bba4132c17..ebdbb155cb 100644 --- a/tests/examples/variables.tf +++ b/tests/examples/variables.tf @@ -52,10 +52,6 @@ variable "region" { default = "region" } -variable "group_email" { - default = "group_email" -} - variable "service_account" { default = { id = "service_account_id" @@ -97,5 +93,4 @@ variable "vpc2" { variable "zone" { default = "zone" -} - +} diff --git a/tests/modules/folder/examples/iam.yaml b/tests/modules/folder/examples/iam.yaml index 7b849dfc39..09d0e65b02 100644 --- a/tests/modules/folder/examples/iam.yaml +++ b/tests/modules/folder/examples/iam.yaml @@ -20,18 +20,18 @@ values: module.folder.google_folder_iam_binding.authoritative["roles/owner"]: condition: [] members: - - group:group_email + - group:organization-admins@example.org - serviceAccount:service_account_email role: roles/owner module.folder.google_folder_iam_binding.authoritative["roles/resourcemanager.folderAdmin"]: condition: [] members: - - group:group_email + - group:organization-admins@example.org role: roles/resourcemanager.folderAdmin module.folder.google_folder_iam_binding.authoritative["roles/resourcemanager.projectCreator"]: condition: [] members: - - group:group_email + - group:organization-admins@example.org role: roles/resourcemanager.projectCreator module.folder.google_folder_iam_member.bindings["am1-storage-admin"]: condition: [] diff --git a/tests/modules/folder/examples/logging-data-access.yaml b/tests/modules/folder/examples/logging-data-access.yaml index 2c6d464191..69b66d0648 100644 --- a/tests/modules/folder/examples/logging-data-access.yaml +++ b/tests/modules/folder/examples/logging-data-access.yaml @@ -20,7 +20,7 @@ values: module.folder.google_folder_iam_audit_config.default["allServices"]: audit_log_config: - exempted_members: - - group:group_email + - group:organization-admins@example.org log_type: ADMIN_READ service: allServices module.folder.google_folder_iam_audit_config.default["storage.googleapis.com"]: From 59178422ab0f6e1fa23974358a26b50adc557210 Mon Sep 17 00:00:00 2001 From: Dinesh Kumar Baskar Date: Tue, 21 Nov 2023 18:11:46 +0100 Subject: [PATCH 5/5] fixed gcs_sink name typo --- modules/folder/README.md | 1 - tests/modules/folder/examples/logging.yaml | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/modules/folder/README.md b/modules/folder/README.md index c9da986401..81f21b01d1 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -205,7 +205,6 @@ module "gcs" { source = "./fabric/modules/gcs" project_id = var.project_id name = "gcs_sink" - prefix = "test" force_destroy = true } diff --git a/tests/modules/folder/examples/logging.yaml b/tests/modules/folder/examples/logging.yaml index 263ca74e10..6b6d8570ea 100644 --- a/tests/modules/folder/examples/logging.yaml +++ b/tests/modules/folder/examples/logging.yaml @@ -46,7 +46,7 @@ values: name: notice module.folder-sink.google_logging_folder_sink.sink["warnings"]: description: warnings (Terraform-managed). - destination: storage.googleapis.com/test-gcs_sink + destination: storage.googleapis.com/gcs_sink disabled: false exclusions: [] filter: severity=WARNING @@ -60,7 +60,7 @@ values: condition: [] role: roles/pubsub.publisher module.folder-sink.google_storage_bucket_iam_member.gcs-sinks-binding["warnings"]: - bucket: test-gcs_sink + bucket: gcs_sink condition: [] role: roles/storage.objectCreator