diff --git a/README.md b/README.md index a40a47d0d8..043e3aeeff 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Currently available modules: - **foundational** - [billing account](./modules/billing-account), [Cloud Identity group](./modules/cloud-identity-group/), [folder](./modules/folder), [service accounts](./modules/iam-service-account), [logging bucket](./modules/logging-bucket), [organization](./modules/organization), [project](./modules/project), [projects-data-source](./modules/projects-data-source) - **networking** - [DNS](./modules/dns), [DNS Response Policy](./modules/dns-response-policy/), [Cloud Endpoints](./modules/endpoints), [address reservation](./modules/net-address), [NAT](./modules/net-cloudnat), [VLAN Attachment](./modules/net-vlan-attachment/), [External Application LB](./modules/net-lb-app-ext/), [External Passthrough Network LB](./modules/net-lb-ext), [External Regional Application Load Balancer](./modules/net-lb-app-ext-regional/), [Firewall policy](./modules/net-firewall-policy), [Internal Application LB](./modules/net-lb-app-int), [Cross-region Internal Application LB](./modules/net-lb-app-int-cross-region), [Internal Passthrough Network LB](./modules/net-lb-int), [Internal Proxy Network LB](./modules/net-lb-proxy-int), [IPSec over Interconnect](./modules/net-ipsec-over-interconnect), [VPC](./modules/net-vpc), [VPC firewall](./modules/net-vpc-firewall), [VPC peering](./modules/net-vpc-peering), [VPN dynamic](./modules/net-vpn-dynamic), [HA VPN](./modules/net-vpn-ha), [VPN static](./modules/net-vpn-static), [Service Directory](./modules/service-directory), [Secure Web Proxy](./modules/net-swp) - **compute** - [VM/VM group](./modules/compute-vm), [MIG](./modules/compute-mig), [COS container](./modules/cloud-config-container/cos-generic-metadata/) (coredns, mysql, onprem, squid), [GKE cluster](./modules/gke-cluster-standard), [GKE hub](./modules/gke-hub), [GKE nodepool](./modules/gke-nodepool), [GCVE private cloud](./modules/gcve-private-cloud) -- **data** - [BigQuery dataset](./modules/bigquery-dataset), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex DataScan](./modules/dataplex-datascan/), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Data Catalog Tag Template](./modules/data-catalog-tag-template), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub), [Dataform Repository](./modules/dataform-repository/) +- **data** - [BigQuery dataset](./modules/bigquery-dataset), [Bigtable instance](./modules/bigtable-instance), [Dataplex](./modules/dataplex), [Dataplex DataScan](./modules/dataplex-datascan/), [Cloud SQL instance](./modules/cloudsql-instance), [Data Catalog Policy Tag](./modules/data-catalog-policy-tag), [Data Catalog Tag](./modules/data-catalog-tag), [Data Catalog Tag Template](./modules/data-catalog-tag-template), [Datafusion](./modules/datafusion), [Dataproc](./modules/dataproc), [GCS](./modules/gcs), [Pub/Sub](./modules/pubsub), [Dataform Repository](./modules/dataform-repository/) - **development** - [API Gateway](./modules/api-gateway), [Apigee](./modules/apigee), [Artifact Registry](./modules/artifact-registry), [Container Registry](./modules/container-registry), [Cloud Source Repository](./modules/source-repository), [Workstation cluster](./modules/workstation-cluster) - **security** - [Binauthz](./modules/binauthz/), [KMS](./modules/kms), [SecretManager](./modules/secret-manager), [VPC Service Control](./modules/vpc-sc) - **serverless** - [Cloud Function v1](./modules/cloud-function-v1), [Cloud Function v2](./modules/cloud-function-v2), [Cloud Run](./modules/cloud-run), [Cloud Run v2](./modules/cloud-run-v2) diff --git a/modules/README.md b/modules/README.md index 2fee211353..366920bd37 100644 --- a/modules/README.md +++ b/modules/README.md @@ -79,6 +79,7 @@ These modules are used in the examples included in this repository. If you are u - [Bigtable instance](./bigtable-instance) - [Cloud SQL instance](./cloudsql-instance) - [Data Catalog Policy Tag](./data-catalog-policy-tag) +- [Data Catalog Tag](./data-catalog-tag) - [Data Catalog Tag Template](./data-catalog-tag-template) - [Dataform Repository](./dataform-repository/) - [Datafusion](./datafusion) diff --git a/modules/data-catalog-tag/README.md b/modules/data-catalog-tag/README.md new file mode 100644 index 0000000000..88c353828d --- /dev/null +++ b/modules/data-catalog-tag/README.md @@ -0,0 +1,126 @@ +# Google Cloud Data Catalog Tag Module + +This module allows managing [Data Catalog Tag](https://cloud.google.com/data-catalog/docs/tags-and-tag-templates) on GCP resources such as BigQuery Datasets, Tables or columns. + +## TODO + +- Add support for entries different than Bigquery resources. + +## Examples + +### Dataset Tag + +```hcl +module "data-catalog-tag" { + source = "./fabric/modules/data-catalog-tag" + tags = { + "landing/countries" = { + project_id = "project-data-product" + parent = "projects/project-data-product/datasets/landing" + location = "europe-west-1" + template = "projects/project-datagov/locations/europe-west1/tagTemplates/demo" + fields = { + source = "DB-1" + } + } + } +} +# tftest modules=1 resources=1 +``` + +### Table Tag + +```hcl +module "data-catalog-tag" { + source = "./fabric/modules/data-catalog-tag" + tags = { + "landing/countries" = { + project_id = "project-data-product" + parent = "projects/project-data-product/datasets/landing/tables/countries" + location = "europe-west-1" + template = "projects/project-datagov/locations/europe-west1/tagTemplates/demo" + fields = { + source = "DB-1 Table-A" + } + } + } +} +# tftest modules=1 resources=1 +``` + +### Column Tag + +```hcl +module "data-catalog-tag" { + source = "./fabric/modules/data-catalog-tag" + tags = { + "landing/countries" = { + project_id = "project-data-product" + parent = "projects/project-data-product/datasets/landing/tables/countries" + column = "country" + location = "europe-west-1" + template = "projects/project-datagov/locations/europe-west1/tagTemplates/demo" + fields = { + source = "DB-1 Table-A Column-B" + } + } + } +} +# tftest modules=1 resources=1 +``` + +### Factory + +Similarly to other modules, a rules factory (see [Resource Factories](../../blueprints/factories/)) is also included here to allow tags management via descriptive configuration files. + +Factory configuration is via one optional attributes in the `factory_config_path` variable specifying the path where tags files are stored. + +Factory tags are merged with rules declared in code, with the latter taking precedence where both use the same key. + +This is an example of a simple factory: + +```hcl +module "data-catalog-tag" { + source = "./fabric/modules/data-catalog-tag" + tags = { + "landing/countries" = { + project_id = "project-data-product" + parent = "projects/project-data-product/datasets/landing/tables/countries" + column = "country" + location = "europe-west-1" + template = "projects/project-datagov/locations/europe-west1/tagTemplates/demo" + fields = { + source = "DB-1 Table-A Column-B" + } + } + } + factories_config = { + tags = "data" + } +} +# tftest modules=1 resources=2 files=demo_tag +``` + +```yaml +# tftest-file id=demo_tag path=data/tag_1.yaml + +project_id: project-data-product +parent: projects/project-data-product/datasets/exposure +template: projects/project-datagov/locations/europe-west1/tagTemplates/test +fields: + owner_email: example@example.com +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [factories_config](variables.tf#L17) | Paths to data files and folders that enable factory functionality. | object({…}) | | {} | +| [tags](variables.tf#L26) | Tags definitions in the form {TAG => TAG_DEFINITION}. | map(object({…})) | | {} | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [data_catalog_tag_ids](outputs.tf#L17) | Data catalog tag ids. | | + diff --git a/modules/data-catalog-tag/main.tf b/modules/data-catalog-tag/main.tf new file mode 100644 index 0000000000..760f49fa5e --- /dev/null +++ b/modules/data-catalog-tag/main.tf @@ -0,0 +1,38 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + _factory_tag_template = { + for f in try(fileset(var.factories_config.tags, "*.yaml"), []) : + trimsuffix(f, ".yaml") => yamldecode(file("${var.factories_config.tags}/${f}")) + } + + factory_tag_template = merge(local._factory_tag_template, var.tags) +} + +resource "google_data_catalog_tag" "engine" { + for_each = local.factory_tag_template + parent = "projects/${each.value.project_id}/locations/${each.value.project_id}/entryGroups/@bigquery/entries/${trim(base64encode(each.value.parent), "=")}" + column = try(each.value.column, null) + template = each.value.template + dynamic "fields" { + for_each = each.value.fields + content { + field_name = fields.key + string_value = fields.value + } + } +} diff --git a/modules/data-catalog-tag/outputs.tf b/modules/data-catalog-tag/outputs.tf new file mode 100644 index 0000000000..8ee99a901f --- /dev/null +++ b/modules/data-catalog-tag/outputs.tf @@ -0,0 +1,20 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "data_catalog_tag_ids" { + description = "Data catalog tag ids." + value = { for k, v in google_data_catalog_tag.engine : k => v.id } +} diff --git a/modules/data-catalog-tag/variables.tf b/modules/data-catalog-tag/variables.tf new file mode 100644 index 0000000000..6bcc3777c5 --- /dev/null +++ b/modules/data-catalog-tag/variables.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "factories_config" { + description = "Paths to data files and folders that enable factory functionality." + type = object({ + tags = optional(string) + }) + nullable = false + default = {} +} + +variable "tags" { + description = "Tags definitions in the form {TAG => TAG_DEFINITION}." + type = map(object({ + project_id = string + parent = string + column = optional(string) + location = string + template = string + fields = map(string) + })) + default = {} +} diff --git a/modules/data-catalog-tag/versions.tf b/modules/data-catalog-tag/versions.tf new file mode 100644 index 0000000000..3db0e2076e --- /dev/null +++ b/modules/data-catalog-tag/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.7.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.11.0, < 6.0.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 5.11.0, < 6.0.0" # tftest + } + } +}