From 8c51a2a3862c8fed0595c6a61dde70d7fe3ea1f2 Mon Sep 17 00:00:00 2001 From: Ludovico Magnocavallo Date: Tue, 1 Nov 2022 14:25:07 +0100 Subject: [PATCH] Enable org policy service and add README notice to modules (#936) * enable org policy service and add README notice to modules * fix tests --- .../data-platform-foundations/03-orchestration.tf | 1 + .../data-platform-foundations/README.md | 11 ++++++----- blueprints/data-solutions/data-playground/main.tf | 1 + blueprints/factories/project-factory/README.md | 2 +- blueprints/factories/project-factory/main.tf | 3 ++- modules/folder/README.md | 2 ++ modules/organization/README.md | 2 ++ modules/project/README.md | 2 ++ .../data_platform_foundations/test_plan.py | 4 +--- .../data_solutions/data_playground/test_plan.py | 5 ++--- 10 files changed, 20 insertions(+), 13 deletions(-) diff --git a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf index 4ced84f230..2990a2c57d 100644 --- a/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf +++ b/blueprints/data-solutions/data-platform-foundations/03-orchestration.tf @@ -84,6 +84,7 @@ module "orch-project" { "container.googleapis.com", "containerregistry.googleapis.com", "dataflow.googleapis.com", + "orgpolicy.googleapis.com", "pubsub.googleapis.com", "servicenetworking.googleapis.com", "storage.googleapis.com", diff --git a/blueprints/data-solutions/data-platform-foundations/README.md b/blueprints/data-solutions/data-platform-foundations/README.md index 034bb32a72..51545d58e5 100644 --- a/blueprints/data-solutions/data-platform-foundations/README.md +++ b/blueprints/data-solutions/data-platform-foundations/README.md @@ -160,9 +160,10 @@ You can find more details and best practices on using DLP to De-identification a [Data Catalog](https://cloud.google.com/data-catalog) helps you to document your data entry at scale. Data Catalog relies on [tags](https://cloud.google.com/data-catalog/docs/tags-and-tag-templates#tags) and [tag template](https://cloud.google.com/data-catalog/docs/tags-and-tag-templates#tag-templates) to manage metadata for all data entries in a unified and centralized service. To implement [column-level security](https://cloud.google.com/bigquery/docs/column-level-security-intro) on BigQuery, we suggest to use `Tags` and `Tag templates`. The default configuration will implement 3 tags: - - `3_Confidential`: policy tag for columns that include very sensitive information, such as credit card numbers. - - `2_Private`: policy tag for columns that include sensitive personal identifiable information (PII) information, such as a person's first name. - - `1_Sensitive`: policy tag for columns that include data that cannot be made public, such as the credit limit. + +- `3_Confidential`: policy tag for columns that include very sensitive information, such as credit card numbers. +- `2_Private`: policy tag for columns that include sensitive personal identifiable information (PII) information, such as a person's first name. +- `1_Sensitive`: policy tag for columns that include data that cannot be made public, such as the credit limit. Anything that is not tagged is available to all users who have access to the data warehouse. @@ -222,7 +223,7 @@ module "data-platform" { prefix = "myprefix" } -# tftest modules=42 resources=315 +# tftest modules=42 resources=316 ``` ## Customizations @@ -238,7 +239,7 @@ To do this, you need to remove IAM binging at project-level for the `data-analys ## Demo pipeline -The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features. +The application layer is out of scope of this script. As a demo purpuse only, several Cloud Composer DAGs are provided. Demos will import data from the `drop off` area to the `Data Warehouse Confidential` dataset suing different features. You can find examples in the `[demo](./demo)` folder. diff --git a/blueprints/data-solutions/data-playground/main.tf b/blueprints/data-solutions/data-playground/main.tf index 3fb6999a60..bcdea5dfa5 100644 --- a/blueprints/data-solutions/data-playground/main.tf +++ b/blueprints/data-solutions/data-playground/main.tf @@ -35,6 +35,7 @@ module "project" { "dataflow.googleapis.com", "ml.googleapis.com", "notebooks.googleapis.com", + "orgpolicy.googleapis.com", "servicenetworking.googleapis.com", "stackdriver.googleapis.com", "storage.googleapis.com", diff --git a/blueprints/factories/project-factory/README.md b/blueprints/factories/project-factory/README.md index 595467c050..cee829ff97 100644 --- a/blueprints/factories/project-factory/README.md +++ b/blueprints/factories/project-factory/README.md @@ -74,7 +74,7 @@ module "projects" { service_identities_iam = try(each.value.service_identities_iam, {}) vpc = try(each.value.vpc, null) } -# tftest modules=7 resources=28 +# tftest modules=7 resources=29 ``` ### Projects configuration diff --git a/blueprints/factories/project-factory/main.tf b/blueprints/factories/project-factory/main.tf index effcfd73e1..1fe5e1e494 100644 --- a/blueprints/factories/project-factory/main.tf +++ b/blueprints/factories/project-factory/main.tf @@ -56,7 +56,8 @@ locals { )) _services = concat([ "billingbudgets.googleapis.com", - "essentialcontacts.googleapis.com" + "essentialcontacts.googleapis.com", + "orgpolicy.googleapis.com", ], length(var.dns_zones) > 0 ? ["dns.googleapis.com"] : [], try(var.vpc.gke_setup, null) != null ? ["container.googleapis.com"] : [], diff --git a/modules/folder/README.md b/modules/folder/README.md index 0014e7616a..d543004a25 100644 --- a/modules/folder/README.md +++ b/modules/folder/README.md @@ -26,6 +26,8 @@ module "folder" { ### Organization policies +To manage organization policies, the `orgpolicy.googleapis.com` service should be enabled in the quota project. + ```hcl module "folder" { source = "./fabric/modules/folder" diff --git a/modules/organization/README.md b/modules/organization/README.md index fb9197c318..3c57b74349 100644 --- a/modules/organization/README.md +++ b/modules/organization/README.md @@ -7,6 +7,8 @@ This module allows managing several organization properties: - audit logging configuration for services - organization policies +To manage organization policies, the `orgpolicy.googleapis.com` service should be enabled in the quota project. + ## Example ```hcl diff --git a/modules/project/README.md b/modules/project/README.md index 8e6c64d4fc..dbb66fcd1f 100644 --- a/modules/project/README.md +++ b/modules/project/README.md @@ -156,6 +156,8 @@ module "project" { ## Organization policies +To manage organization policies, the `orgpolicy.googleapis.com` service should be enabled in the quota project. + ```hcl module "project" { source = "./fabric/modules/project" diff --git a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py index 0e4b77f538..1b51472cdc 100644 --- a/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py +++ b/tests/blueprints/data_solutions/data_platform_foundations/test_plan.py @@ -12,11 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. - import os import pytest - FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') @@ -24,4 +22,4 @@ def test_resources(e2e_plan_runner): "Test that plan works and the numbers of resources is as expected." modules, resources = e2e_plan_runner(FIXTURES_DIR) assert len(modules) == 41 - assert len(resources) == 314 + assert len(resources) == 315 diff --git a/tests/blueprints/data_solutions/data_playground/test_plan.py b/tests/blueprints/data_solutions/data_playground/test_plan.py index 05bda08cde..2653c7eabe 100644 --- a/tests/blueprints/data_solutions/data_playground/test_plan.py +++ b/tests/blueprints/data_solutions/data_playground/test_plan.py @@ -12,15 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. - import os import pytest - FIXTURES_DIR = os.path.join(os.path.dirname(__file__), 'fixture') + def test_resources(e2e_plan_runner): "Test that plan works and the numbers of resources is as expected." modules, resources = e2e_plan_runner(FIXTURES_DIR) assert len(modules) == 7 - assert len(resources) == 34 + assert len(resources) == 35