From 9d460470eddf11711b96ffba11aeea41244fc5d8 Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Sat, 17 Feb 2024 16:01:54 +0000 Subject: [PATCH 1/6] add analytics hub module --- modules/analytics-hub/README.md | 164 +++++++++++++++++++++++++++++ modules/analytics-hub/iam.tf | 50 +++++++++ modules/analytics-hub/main.tf | 64 +++++++++++ modules/analytics-hub/outputs.tf | 33 ++++++ modules/analytics-hub/variables.tf | 103 ++++++++++++++++++ modules/analytics-hub/versions.tf | 27 +++++ 6 files changed, 441 insertions(+) create mode 100644 modules/analytics-hub/README.md create mode 100644 modules/analytics-hub/iam.tf create mode 100644 modules/analytics-hub/main.tf create mode 100644 modules/analytics-hub/outputs.tf create mode 100644 modules/analytics-hub/variables.tf create mode 100644 modules/analytics-hub/versions.tf diff --git a/modules/analytics-hub/README.md b/modules/analytics-hub/README.md new file mode 100644 index 0000000000..5c32ea0c07 --- /dev/null +++ b/modules/analytics-hub/README.md @@ -0,0 +1,164 @@ +# BigQuery Analytics Hub + +This module allows managing [Analytics Hub](https://cloud.google.com/bigquery/docs/analytics-hub-introduction) Exchange and Listing resources. + +## Examples + +### Exchange + +```hcl +module "analytics-hub" { + source = "./fabric/modules/analytics-hub" + project_id = "project-id" + region = "us-central1" + prefix = "test" + name = "exchange" + primary_contact = "exchange-owner-group@domain.com" + documentation = "documentation" +} +# tftest modules=1 resources=1 +``` + +### Listings + +```hcl +module "analytics-hub" { + source = "./fabric/modules/analytics-hub" + project_id = "project-id" + region = "us-central1" + name = "exchange" + listings = { + "listing_id" = { + bigquery_dataset = "projects/{project}/datasets/{dataset}" + }, + "listing_id_2" = { + bigquery_dataset = "projects/thinhha-pso-test-project/datasets/biglake_test" + restricted_export_config = { + enabled = true + restrict_query_result = true + } + description = "(Optional) Short description of the listing." + primary_contact = "(Optional) Email or URL of the primary point of contact of the listing." + documentation = "(Optional) Documentation describing the listing." + request_access = "(Optional) Email or URL of the request access of the listing. Subscribers can use this reference to request access." + categories = [] + data_provider = { + name = "(Required) Name of the data provider." + primary_contact = "(Optional) Email or URL of the data provider." + } + publisher = { + name = "(Required) Name of the dlisting publisher." + primary_contact = "(Optional) Email or URL of the listing publisher." + } + } + } +} +# tftest modules=1 resources=3 +``` + +### IAM + +This module supports setting IAM permissions on both the exchange and listing resources. IAM permissions on the exchange is inherited on the listings. See [this page](https://cloud.google.com/bigquery/docs/analytics-hub-grant-roles) for IAM roles that can be granted on exchange and listings. + +```hcl +module "analytics-hub" { + source = "./fabric/modules/analytics-hub" + project_id = "project-id" + region = "us-central1" + name = "exchange" + iam = { + "roles/analyticshub.viewer" = [ + "group:viewer@domain.com" + ], + } + listings = { + "listing_id" = { + bigquery_dataset = "projects/{project}/datasets/{dataset}" + iam = { + "roles/analyticshub.subscriber" = [ + "group:subscriber@domain.com" + ], + "roles/analyticshub.subscriptionOwner" = [ + "group:subscription-owner@domain.com" + ], + } + } + } +} +# tftest modules=1 resources=5 +``` + +### Factory + +Similarly to other modules, a rules factory (see [Resource Factories](../../blueprints/factories/)) is also included here to allow managing listings inside the same exchange via descriptive configuration files. + +Factory configuration is via one optional attributes in the `factory_config_path` variable specifying the path where tags files are stored. + +Factory tags are merged with rules declared in code, with the latter taking precedence where both use the same key. + +This is an example of a simple factory: + +```hcl +module "analytics-hub" { + source = "./fabric/modules/analytics-hub" + project_id = "project-id" + region = "us-central1" + name = "exchange" + listings = { + "listing_id" = { + bigquery_dataset = "projects/{project}/datasets/{dataset}" + }, + } + factories_config = { + listings = "listings" + } +} +# tftest modules=1 resources=5 files=yaml +``` + +```yaml +# tftest-file id=yaml path=listings/listing_1.yaml +bigquery_dataset: projects/{project}/datasets/{dataset} +restricted_export_config: + enabled: true +description: "(Optional) Short description of the listing." +primary_contact: "(Optional) Email or URL of the primary point of contact of the listing." +documentation: "(Optional) Documentation describing the listing." +request_access: "(Optional) Email or URL of the request access of the listing. Subscribers can use this reference to request access." +categories: [] +data_provider: + name: "(Required) Name of the data provider." + primary_contact: "(Optional) Email or URL of the data provider." +publisher: + name: "(Required) Name of the dlisting publisher." + primary_contact: "(Optional) Email or URL of the listing publisher." +iam: + roles/analyticshub.subscriber: + - group:subscriber@domain.com + roles/analyticshub.subscriptionOwner: + - group:subscription-owner@domain.com +``` + +## Variables + +| name | description | type | required | default | +|---|---|:---:|:---:|:---:| +| [name](variables.tf#L78) | The ID of the data exchange. Must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces. | string | ✓ | | +| [project_id](variables.tf#L95) | The ID of the project where the Analytics Hub Exchange will be created. | string | ✓ | | +| [region](variables.tf#L100) | Region for the data exchange | string | ✓ | | +| [description](variables.tf#L17) | Resource description for data exchange | string | | null | +| [documentation](variables.tf#L23) | Documentation describing the data exchange. | string | | null | +| [factories_config](variables.tf#L29) | Paths to data files and folders that enable factory functionality. | object({…}) | | {} | +| [iam](variables.tf#L38) | Authoritative IAM bindings for the data exchange in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [icon](variables.tf#L45) | Base64 encoded image representing the data exchange. | string | | null | +| [listings](variables.tf#L51) | Listings definitions in the form {LISTING_ID => LISTING_CONFIGS}. LISTING_ID must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces. | map(object({…})) | | {} | +| [prefix](variables.tf#L83) | Optional prefix for Analytics Hub Exchange ID. | string | | null | +| [primary_contact](variables.tf#L89) | Email or URL of the primary point of contact of the data exchange.. | string | | null | + +## Outputs + +| name | description | sensitive | +|---|---|:---:| +| [data_exchange_id](outputs.tf#L17) | Data exchange id. | | +| [data_listings](outputs.tf#L26) | Data listings and corresponding configs. | | + diff --git a/modules/analytics-hub/iam.tf b/modules/analytics-hub/iam.tf new file mode 100644 index 0000000000..63c1f119a3 --- /dev/null +++ b/modules/analytics-hub/iam.tf @@ -0,0 +1,50 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + listing_iam = flatten([ + for listing_id, listing_configs in local.factory_listings : [ + for role, members in coalesce(listing_configs.iam, tomap({})) : { + listing_id = listing_id + role = role + members = members + } + ] + ]) +} + +resource "google_bigquery_analytics_hub_data_exchange_iam_binding" "exchange_iam_bindings" { + for_each = var.iam + project = google_bigquery_analytics_hub_data_exchange.data_exchange.project + location = google_bigquery_analytics_hub_data_exchange.data_exchange.location + data_exchange_id = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id + role = each.key + members = each.value +} + +resource "google_bigquery_analytics_hub_listing_iam_binding" "listing_iam_bindings" { + for_each = { + for index, listing_iam in local.listing_iam : + "${listing_iam.listing_id}-${listing_iam.role}" => listing_iam + } + project = google_bigquery_analytics_hub_data_exchange.data_exchange.project + location = google_bigquery_analytics_hub_data_exchange.data_exchange.location + data_exchange_id = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id + listing_id = each.value.listing_id + role = each.value.role + members = each.value.members + depends_on = [google_bigquery_analytics_hub_listing.listing] +} diff --git a/modules/analytics-hub/main.tf b/modules/analytics-hub/main.tf new file mode 100644 index 0000000000..e82798c175 --- /dev/null +++ b/modules/analytics-hub/main.tf @@ -0,0 +1,64 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + prefix = var.prefix == null || var.prefix == "" ? "" : "${var.prefix}_" + _factory_listings = { + for f in try(fileset(var.factories_config.listings, "*.yaml"), []) : + trimsuffix(f, ".yaml") => yamldecode(file("${var.factories_config.listings}/${f}")) + } + + factory_listings = merge(local._factory_listings, var.listings) +} + +resource "google_bigquery_analytics_hub_data_exchange" "data_exchange" { + project = var.project_id + location = var.region + data_exchange_id = "${local.prefix}${var.name}" + display_name = "${local.prefix}${var.name}" + description = var.description + primary_contact = var.primary_contact + documentation = var.documentation + icon = var.icon +} + +resource "google_bigquery_analytics_hub_listing" "listing" { + for_each = local.factory_listings + project = var.project_id + location = var.region + data_exchange_id = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id + listing_id = each.key + display_name = each.key + description = try(each.value.description, null) + primary_contact = try(each.value.primary_contact, null) + documentation = try(each.value.documentation, null) + icon = try(each.value.icon, null) + request_access = try(each.value.request_access, null) + categories = try(each.value.categories, null) + + bigquery_dataset { + dataset = each.value.bigquery_dataset + } + + dynamic "restricted_export_config" { + for_each = each.value.restricted_export_config != null ? [""] : [] + content { + enabled = try(each.value.restricted_export_config.enabled, null) + restrict_query_result = try(each.value.restricted_export_config.restrict_query_result, null) + } + } + +} diff --git a/modules/analytics-hub/outputs.tf b/modules/analytics-hub/outputs.tf new file mode 100644 index 0000000000..7a6a4c757f --- /dev/null +++ b/modules/analytics-hub/outputs.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "data_exchange_id" { + description = "Data exchange id." + value = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id + depends_on = [ + google_bigquery_analytics_hub_data_exchange.data_exchange, + google_bigquery_analytics_hub_data_exchange_iam_binding.exchange_iam_bindings + ] +} + +output "data_listings" { + description = "Data listings and corresponding configs." + value = { for k, v in google_bigquery_analytics_hub_listing.listing : k => v.bigquery_dataset } + depends_on = [ + google_bigquery_analytics_hub_listing.listing, + google_bigquery_analytics_hub_listing_iam_binding.listing_iam_bindings + ] +} diff --git a/modules/analytics-hub/variables.tf b/modules/analytics-hub/variables.tf new file mode 100644 index 0000000000..fa819494c3 --- /dev/null +++ b/modules/analytics-hub/variables.tf @@ -0,0 +1,103 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "description" { + description = "Resource description for data exchange" + default = null + type = string +} + +variable "documentation" { + description = "Documentation describing the data exchange." + default = null + type = string +} + +variable "factories_config" { + description = "Paths to data files and folders that enable factory functionality." + type = object({ + listings = optional(string) + }) + nullable = false + default = {} +} + +variable "iam" { + description = "Authoritative IAM bindings for the data exchange in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "icon" { + description = "Base64 encoded image representing the data exchange." + default = null + type = string +} + +variable "listings" { + description = "Listings definitions in the form {LISTING_ID => LISTING_CONFIGS}. LISTING_ID must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces." + type = map(object({ + description = optional(string) + bigquery_dataset = string + iam = optional(map(list(string))) + primary_contact = optional(string) + documentation = optional(string) + icon = optional(string) + request_access = optional(string) + data_provider = optional(object({ + name = string + primary_contact = optional(string) + })) + publisher = optional(object({ + name = string + primary_contact = optional(string) + })) + categories = optional(list(string)) + restricted_export_config = optional(object({ + enabled = optional(bool) + restrict_query_result = optional(bool) + })) + })) + default = {} +} + +variable "name" { + description = "The ID of the data exchange. Must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces." + type = string +} + +variable "prefix" { + description = "Optional prefix for Analytics Hub Exchange ID." + type = string + default = null +} + +variable "primary_contact" { + description = "Email or URL of the primary point of contact of the data exchange.." + type = string + default = null +} + +variable "project_id" { + description = "The ID of the project where the Analytics Hub Exchange will be created." + type = string +} + +variable "region" { + description = "Region for the data exchange" + type = string +} diff --git a/modules/analytics-hub/versions.tf b/modules/analytics-hub/versions.tf new file mode 100644 index 0000000000..3db0e2076e --- /dev/null +++ b/modules/analytics-hub/versions.tf @@ -0,0 +1,27 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +terraform { + required_version = ">= 1.7.0" + required_providers { + google = { + source = "hashicorp/google" + version = ">= 5.11.0, < 6.0.0" # tftest + } + google-beta = { + source = "hashicorp/google-beta" + version = ">= 5.11.0, < 6.0.0" # tftest + } + } +} From c083e22edba83ae181e83c9aa451470ca42037ae Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Sat, 17 Feb 2024 16:08:02 +0000 Subject: [PATCH 2/6] fix docs --- modules/analytics-hub/README.md | 6 +++--- modules/analytics-hub/variables.tf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/analytics-hub/README.md b/modules/analytics-hub/README.md index 5c32ea0c07..0a7ace4a11 100644 --- a/modules/analytics-hub/README.md +++ b/modules/analytics-hub/README.md @@ -145,15 +145,15 @@ iam: |---|---|:---:|:---:|:---:| | [name](variables.tf#L78) | The ID of the data exchange. Must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces. | string | ✓ | | | [project_id](variables.tf#L95) | The ID of the project where the Analytics Hub Exchange will be created. | string | ✓ | | -| [region](variables.tf#L100) | Region for the data exchange | string | ✓ | | -| [description](variables.tf#L17) | Resource description for data exchange | string | | null | +| [region](variables.tf#L100) | Region for the data exchange. | string | ✓ | | +| [description](variables.tf#L17) | Resource description for data exchange. | string | | null | | [documentation](variables.tf#L23) | Documentation describing the data exchange. | string | | null | | [factories_config](variables.tf#L29) | Paths to data files and folders that enable factory functionality. | object({…}) | | {} | | [iam](variables.tf#L38) | Authoritative IAM bindings for the data exchange in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | | [icon](variables.tf#L45) | Base64 encoded image representing the data exchange. | string | | null | | [listings](variables.tf#L51) | Listings definitions in the form {LISTING_ID => LISTING_CONFIGS}. LISTING_ID must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces. | map(object({…})) | | {} | | [prefix](variables.tf#L83) | Optional prefix for Analytics Hub Exchange ID. | string | | null | -| [primary_contact](variables.tf#L89) | Email or URL of the primary point of contact of the data exchange.. | string | | null | +| [primary_contact](variables.tf#L89) | Email or URL of the primary point of contact of the data exchange. | string | | null | ## Outputs diff --git a/modules/analytics-hub/variables.tf b/modules/analytics-hub/variables.tf index fa819494c3..bafad32f3a 100644 --- a/modules/analytics-hub/variables.tf +++ b/modules/analytics-hub/variables.tf @@ -15,7 +15,7 @@ */ variable "description" { - description = "Resource description for data exchange" + description = "Resource description for data exchange." default = null type = string } @@ -87,7 +87,7 @@ variable "prefix" { } variable "primary_contact" { - description = "Email or URL of the primary point of contact of the data exchange.." + description = "Email or URL of the primary point of contact of the data exchange." type = string default = null } @@ -98,6 +98,6 @@ variable "project_id" { } variable "region" { - description = "Region for the data exchange" + description = "Region for the data exchange." type = string } From 5be27fd555bb83995f0b1510b5c617084360f17a Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Sat, 17 Feb 2024 16:22:11 +0000 Subject: [PATCH 3/6] minor edit --- modules/analytics-hub/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/analytics-hub/README.md b/modules/analytics-hub/README.md index 0a7ace4a11..ef36658ad6 100644 --- a/modules/analytics-hub/README.md +++ b/modules/analytics-hub/README.md @@ -32,7 +32,7 @@ module "analytics-hub" { bigquery_dataset = "projects/{project}/datasets/{dataset}" }, "listing_id_2" = { - bigquery_dataset = "projects/thinhha-pso-test-project/datasets/biglake_test" + bigquery_dataset = "projects/{project}/datasets/{dataset}" restricted_export_config = { enabled = true restrict_query_result = true @@ -60,6 +60,8 @@ module "analytics-hub" { This module supports setting IAM permissions on both the exchange and listing resources. IAM permissions on the exchange is inherited on the listings. See [this page](https://cloud.google.com/bigquery/docs/analytics-hub-grant-roles) for IAM roles that can be granted on exchange and listings. +IAM bindings created by this module are authoritative for each given role. + ```hcl module "analytics-hub" { source = "./fabric/modules/analytics-hub" From 33d36b08e68d83560bc382de199cd97239e5c223 Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Mon, 19 Feb 2024 13:35:23 +0000 Subject: [PATCH 4/6] add more iam interfaces for exchange --- modules/analytics-hub/README.md | 108 +++++++++++++----- modules/analytics-hub/iam.tf | 34 +++++- modules/analytics-hub/outputs.tf | 3 +- modules/analytics-hub/variables-iam.tf | 49 ++++++++ modules/analytics-hub/variables.tf | 23 ++-- .../analytics_hub/examples/iam_exchange.yaml | 50 ++++++++ .../analytics_hub/examples/iam_listing.yaml | 77 +++++++++++++ 7 files changed, 296 insertions(+), 48 deletions(-) create mode 100644 modules/analytics-hub/variables-iam.tf create mode 100644 tests/modules/analytics_hub/examples/iam_exchange.yaml create mode 100644 tests/modules/analytics_hub/examples/iam_listing.yaml diff --git a/modules/analytics-hub/README.md b/modules/analytics-hub/README.md index ef36658ad6..4fa791cce1 100644 --- a/modules/analytics-hub/README.md +++ b/modules/analytics-hub/README.md @@ -4,6 +4,8 @@ This module allows managing [Analytics Hub](https://cloud.google.com/bigquery/do ## Examples +Listing argument references can be found in: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_listing + ### Exchange ```hcl @@ -21,6 +23,8 @@ module "analytics-hub" { ### Listings +Listing definitions can be provided in the form in the form {LISTING_ID => LISTING_CONFIGS}. Listing argument references can be found in: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_listing + ```hcl module "analytics-hub" { source = "./fabric/modules/analytics-hub" @@ -33,23 +37,24 @@ module "analytics-hub" { }, "listing_id_2" = { bigquery_dataset = "projects/{project}/datasets/{dataset}" - restricted_export_config = { - enabled = true - restrict_query_result = true - } - description = "(Optional) Short description of the listing." - primary_contact = "(Optional) Email or URL of the primary point of contact of the listing." - documentation = "(Optional) Documentation describing the listing." - request_access = "(Optional) Email or URL of the request access of the listing. Subscribers can use this reference to request access." - categories = [] + description = "(Optional) Short description of the listing." + documentation = "(Optional) Documentation describing the listing." + categories = [] + primary_contact = "(Optional) Email or URL of the primary point of contact of the listing." + icon = "(Optional) Base64 encoded image representing the listing." + request_access = "(Optional) Email or URL of the request access of the listing. Subscribers can use this reference to request access." data_provider = { name = "(Required) Name of the data provider." primary_contact = "(Optional) Email or URL of the data provider." } publisher = { - name = "(Required) Name of the dlisting publisher." + name = "(Required) Name of the listing publisher." primary_contact = "(Optional) Email or URL of the listing publisher." } + restricted_export_config = { + enabled = true + restrict_query_result = true + } } } } @@ -58,9 +63,49 @@ module "analytics-hub" { ### IAM -This module supports setting IAM permissions on both the exchange and listing resources. IAM permissions on the exchange is inherited on the listings. See [this page](https://cloud.google.com/bigquery/docs/analytics-hub-grant-roles) for IAM roles that can be granted on exchange and listings. +This module supports setting IAM permissions on both the exchange and listing resources. IAM permissions on the exchange is inherited on the listings. + +See [this page](https://cloud.google.com/bigquery/docs/analytics-hub-grant-roles) to see IAM roles that can be granted on exchange and listings. + +#### Exchange +Input to variables `iam`, `iam_bindings`, and `iam_by_principals` will be merged, and are [authoritative for the given role](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_data_exchange_iam#google_bigquery_analytics_hub_data_exchange_iam_binding). Inputs to variable `iam_bindings_additive` are [additive](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_data_exchange_iam#google_bigquery_analytics_hub_data_exchange_iam_member). + +In practice, you should only need to use either `iam` or `iam_bindings`. + +```hcl +module "analytics-hub" { + source = "./fabric/modules/analytics-hub" + project_id = "project-id" + region = "us-central1" + name = "exchange" + iam = { + "roles/analyticshub.viewer" = [ + "group:viewer@domain.com" + ], + } + iam_bindings = { + "viewers" = { + role = "roles/analyticshub.viewer" + members = ["user:thinhha@google.com"] + } + } + iam_by_principals = { + "user:thinhha@google.com" = [ + "roles/analyticshub.viewer" + ] + } + iam_bindings_additive = { + "subscribers" = { + role = "roles/analyticshub.subscriber" + member = "user:thinhha@google.com" + } + } +} +# tftest modules=1 resources=3 inventory=iam_exchange.yaml +``` -IAM bindings created by this module are authoritative for each given role. +#### Listings +The listings variable block support the `iam` input which are [authoritative for the given role](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_listing_iam#google_bigquery_analytics_hub_listing_iam_binding). ```hcl module "analytics-hub" { @@ -87,7 +132,7 @@ module "analytics-hub" { } } } -# tftest modules=1 resources=5 +# tftest modules=1 resources=5 inventory=iam_listing.yaml ``` ### Factory @@ -121,46 +166,51 @@ module "analytics-hub" { ```yaml # tftest-file id=yaml path=listings/listing_1.yaml bigquery_dataset: projects/{project}/datasets/{dataset} -restricted_export_config: - enabled: true description: "(Optional) Short description of the listing." -primary_contact: "(Optional) Email or URL of the primary point of contact of the listing." documentation: "(Optional) Documentation describing the listing." -request_access: "(Optional) Email or URL of the request access of the listing. Subscribers can use this reference to request access." categories: [] +icon: "(Optional) Base64 encoded image representing the listing." +primary_contact: "(Optional) Email or URL of the primary point of contact of the listing." +request_access: "(Optional) Email or URL of the request access of the listing. Subscribers can use this reference to request access." data_provider: name: "(Required) Name of the data provider." primary_contact: "(Optional) Email or URL of the data provider." -publisher: - name: "(Required) Name of the dlisting publisher." - primary_contact: "(Optional) Email or URL of the listing publisher." iam: roles/analyticshub.subscriber: - group:subscriber@domain.com roles/analyticshub.subscriptionOwner: - group:subscription-owner@domain.com +publisher: + name: "(Required) Name of the listing publisher." + primary_contact: "(Optional) Email or URL of the listing publisher." +restricted_export_config: + enabled: true + restrict_query_result: true ``` ## Variables | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [name](variables.tf#L78) | The ID of the data exchange. Must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces. | string | ✓ | | -| [project_id](variables.tf#L95) | The ID of the project where the Analytics Hub Exchange will be created. | string | ✓ | | -| [region](variables.tf#L100) | Region for the data exchange. | string | ✓ | | +| [name](variables.tf#L71) | The ID of the data exchange. Must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping or characters outside of ASCII spaces. | string | ✓ | | +| [project_id](variables.tf#L88) | The ID of the project where the data exchange will be created. | string | ✓ | | +| [region](variables.tf#L93) | Region for the data exchange. | string | ✓ | | | [description](variables.tf#L17) | Resource description for data exchange. | string | | null | | [documentation](variables.tf#L23) | Documentation describing the data exchange. | string | | null | | [factories_config](variables.tf#L29) | Paths to data files and folders that enable factory functionality. | object({…}) | | {} | -| [iam](variables.tf#L38) | Authoritative IAM bindings for the data exchange in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | -| [icon](variables.tf#L45) | Base64 encoded image representing the data exchange. | string | | null | -| [listings](variables.tf#L51) | Listings definitions in the form {LISTING_ID => LISTING_CONFIGS}. LISTING_ID must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces. | map(object({…})) | | {} | -| [prefix](variables.tf#L83) | Optional prefix for Analytics Hub Exchange ID. | string | | null | -| [primary_contact](variables.tf#L89) | Email or URL of the primary point of contact of the data exchange. | string | | null | +| [iam](variables-iam.tf#L17) | Authoritative IAM bindings in {ROLE => [MEMBERS]} format. | map(list(string)) | | {} | +| [iam_bindings](variables-iam.tf#L24) | Authoritative IAM bindings in {KEY => {role = ROLE, members = []}}. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_bindings_additive](variables-iam.tf#L34) | Individual additive IAM bindings. Keys are arbitrary. | map(object({…})) | | {} | +| [iam_by_principals](variables-iam.tf#L44) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | map(list(string)) | | {} | +| [icon](variables.tf#L38) | Base64 encoded image representing the data exchange. | string | | null | +| [listings](variables.tf#L44) | Listings definitions in the form {LISTING_ID => LISTING_CONFIGS}. LISTING_ID must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping or characters outside of ASCII spaces. | map(object({…})) | | {} | +| [prefix](variables.tf#L76) | Optional prefix for data exchange ID. | string | | null | +| [primary_contact](variables.tf#L82) | Email or URL of the primary point of contact of the data exchange. | string | | null | ## Outputs | name | description | sensitive | |---|---|:---:| | [data_exchange_id](outputs.tf#L17) | Data exchange id. | | -| [data_listings](outputs.tf#L26) | Data listings and corresponding configs. | | +| [data_listings](outputs.tf#L27) | Data listings and corresponding configs. | | diff --git a/modules/analytics-hub/iam.tf b/modules/analytics-hub/iam.tf index 63c1f119a3..ed8b98fe19 100644 --- a/modules/analytics-hub/iam.tf +++ b/modules/analytics-hub/iam.tf @@ -15,6 +15,25 @@ */ locals { + _exchange_iam_principal_roles = distinct(flatten(values(var.iam_by_principals))) + _exchange_iam_principals = { + for r in local._exchange_iam_principal_roles : r => [ + for k, v in var.iam_by_principals : + k if try(index(v, r), null) != null + ] + } + _exchange_iam_bindings = { + for key, iam_bindings in var.iam_bindings : + iam_bindings.role => iam_bindings.members + } + exchange_iam = { + for role in distinct(concat(keys(var.iam), keys(local._exchange_iam_principals), keys(local._exchange_iam_bindings))) : + role => concat( + try(var.iam[role], []), + try(local._exchange_iam_principals[role], []), + try(local._exchange_iam_bindings[role], []), + ) + } listing_iam = flatten([ for listing_id, listing_configs in local.factory_listings : [ for role, members in coalesce(listing_configs.iam, tomap({})) : { @@ -26,8 +45,9 @@ locals { ]) } + resource "google_bigquery_analytics_hub_data_exchange_iam_binding" "exchange_iam_bindings" { - for_each = var.iam + for_each = local.exchange_iam project = google_bigquery_analytics_hub_data_exchange.data_exchange.project location = google_bigquery_analytics_hub_data_exchange.data_exchange.location data_exchange_id = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id @@ -35,6 +55,15 @@ resource "google_bigquery_analytics_hub_data_exchange_iam_binding" "exchange_iam members = each.value } +resource "google_bigquery_analytics_hub_data_exchange_iam_member" "exchange_iam_members" { + for_each = var.iam_bindings_additive + project = google_bigquery_analytics_hub_data_exchange.data_exchange.project + location = google_bigquery_analytics_hub_data_exchange.data_exchange.location + data_exchange_id = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id + role = each.value.role + member = each.value.member +} + resource "google_bigquery_analytics_hub_listing_iam_binding" "listing_iam_bindings" { for_each = { for index, listing_iam in local.listing_iam : @@ -43,8 +72,7 @@ resource "google_bigquery_analytics_hub_listing_iam_binding" "listing_iam_bindin project = google_bigquery_analytics_hub_data_exchange.data_exchange.project location = google_bigquery_analytics_hub_data_exchange.data_exchange.location data_exchange_id = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id - listing_id = each.value.listing_id + listing_id = google_bigquery_analytics_hub_listing.listing[each.value.listing_id].id role = each.value.role members = each.value.members - depends_on = [google_bigquery_analytics_hub_listing.listing] } diff --git a/modules/analytics-hub/outputs.tf b/modules/analytics-hub/outputs.tf index 7a6a4c757f..5cbf45d8a4 100644 --- a/modules/analytics-hub/outputs.tf +++ b/modules/analytics-hub/outputs.tf @@ -19,7 +19,8 @@ output "data_exchange_id" { value = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id depends_on = [ google_bigquery_analytics_hub_data_exchange.data_exchange, - google_bigquery_analytics_hub_data_exchange_iam_binding.exchange_iam_bindings + google_bigquery_analytics_hub_data_exchange_iam_binding.exchange_iam_bindings, + google_bigquery_analytics_hub_data_exchange_iam_member.exchange_iam_members ] } diff --git a/modules/analytics-hub/variables-iam.tf b/modules/analytics-hub/variables-iam.tf new file mode 100644 index 0000000000..5ad9a377d5 --- /dev/null +++ b/modules/analytics-hub/variables-iam.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "iam" { + description = "Authoritative IAM bindings in {ROLE => [MEMBERS]} format." + type = map(list(string)) + default = {} + nullable = false +} + +variable "iam_bindings" { + description = "Authoritative IAM bindings in {KEY => {role = ROLE, members = []}}. Keys are arbitrary." + type = map(object({ + members = list(string) + role = string + })) + nullable = false + default = {} +} + +variable "iam_bindings_additive" { + description = "Individual additive IAM bindings. Keys are arbitrary." + type = map(object({ + member = string + role = string + })) + nullable = false + default = {} +} + +variable "iam_by_principals" { + description = "Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable." + type = map(list(string)) + default = {} + nullable = false +} diff --git a/modules/analytics-hub/variables.tf b/modules/analytics-hub/variables.tf index bafad32f3a..6f6136389f 100644 --- a/modules/analytics-hub/variables.tf +++ b/modules/analytics-hub/variables.tf @@ -35,13 +35,6 @@ variable "factories_config" { default = {} } -variable "iam" { - description = "Authoritative IAM bindings for the data exchange in {ROLE => [MEMBERS]} format." - type = map(list(string)) - default = {} - nullable = false -} - variable "icon" { description = "Base64 encoded image representing the data exchange." default = null @@ -49,24 +42,24 @@ variable "icon" { } variable "listings" { - description = "Listings definitions in the form {LISTING_ID => LISTING_CONFIGS}. LISTING_ID must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces." + description = "Listings definitions in the form {LISTING_ID => LISTING_CONFIGS}. LISTING_ID must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping or characters outside of ASCII spaces." type = map(object({ - description = optional(string) bigquery_dataset = string - iam = optional(map(list(string))) - primary_contact = optional(string) + description = optional(string) documentation = optional(string) + categories = optional(list(string)) icon = optional(string) + primary_contact = optional(string) request_access = optional(string) data_provider = optional(object({ name = string primary_contact = optional(string) })) + iam = optional(map(list(string))) publisher = optional(object({ name = string primary_contact = optional(string) })) - categories = optional(list(string)) restricted_export_config = optional(object({ enabled = optional(bool) restrict_query_result = optional(bool) @@ -76,12 +69,12 @@ variable "listings" { } variable "name" { - description = "The ID of the data exchange. Must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping, or characters outside of ASCII, spaces." + description = "The ID of the data exchange. Must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping or characters outside of ASCII spaces." type = string } variable "prefix" { - description = "Optional prefix for Analytics Hub Exchange ID." + description = "Optional prefix for data exchange ID." type = string default = null } @@ -93,7 +86,7 @@ variable "primary_contact" { } variable "project_id" { - description = "The ID of the project where the Analytics Hub Exchange will be created." + description = "The ID of the project where the data exchange will be created." type = string } diff --git a/tests/modules/analytics_hub/examples/iam_exchange.yaml b/tests/modules/analytics_hub/examples/iam_exchange.yaml new file mode 100644 index 0000000000..c377cf85ab --- /dev/null +++ b/tests/modules/analytics_hub/examples/iam_exchange.yaml @@ -0,0 +1,50 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.analytics-hub.google_bigquery_analytics_hub_data_exchange.data_exchange: + data_exchange_id: exchange + description: null + display_name: exchange + documentation: null + icon: null + location: us-central1 + primary_contact: null + project: project-id + timeouts: null + ? module.analytics-hub.google_bigquery_analytics_hub_data_exchange_iam_binding.exchange_iam_bindings["roles/analyticshub.viewer"] + : condition: [] + data_exchange_id: exchange + location: us-central1 + members: + - group:viewer@domain.com + - user:thinhha@google.com + project: project-id + role: roles/analyticshub.viewer + module.analytics-hub.google_bigquery_analytics_hub_data_exchange_iam_member.exchange_iam_members["subscribers"]: + condition: [] + data_exchange_id: exchange + location: us-central1 + member: user:thinhha@google.com + project: project-id + role: roles/analyticshub.subscriber + +counts: + google_bigquery_analytics_hub_data_exchange: 1 + google_bigquery_analytics_hub_data_exchange_iam_binding: 1 + google_bigquery_analytics_hub_data_exchange_iam_member: 1 + modules: 1 + resources: 3 + +outputs: {} diff --git a/tests/modules/analytics_hub/examples/iam_listing.yaml b/tests/modules/analytics_hub/examples/iam_listing.yaml new file mode 100644 index 0000000000..604cb9bb02 --- /dev/null +++ b/tests/modules/analytics_hub/examples/iam_listing.yaml @@ -0,0 +1,77 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.analytics-hub.google_bigquery_analytics_hub_data_exchange.data_exchange: + data_exchange_id: exchange + description: null + display_name: exchange + documentation: null + icon: null + location: us-central1 + primary_contact: null + project: project-id + timeouts: null + ? module.analytics-hub.google_bigquery_analytics_hub_data_exchange_iam_binding.exchange_iam_bindings["roles/analyticshub.viewer"] + : condition: [] + data_exchange_id: exchange + location: us-central1 + members: + - group:viewer@domain.com + project: project-id + role: roles/analyticshub.viewer + module.analytics-hub.google_bigquery_analytics_hub_listing.listing["listing_id"]: + bigquery_dataset: + - dataset: projects/{project}/datasets/{dataset} + categories: null + data_exchange_id: exchange + data_provider: [] + description: null + display_name: listing_id + documentation: null + icon: null + listing_id: listing_id + location: us-central1 + primary_contact: null + project: project-id + publisher: [] + request_access: null + restricted_export_config: [] + timeouts: null + ? module.analytics-hub.google_bigquery_analytics_hub_listing_iam_binding.listing_iam_bindings["listing_id-roles/analyticshub.subscriber"] + : condition: [] + data_exchange_id: exchange + location: us-central1 + members: + - group:subscriber@domain.com + project: project-id + role: roles/analyticshub.subscriber + ? module.analytics-hub.google_bigquery_analytics_hub_listing_iam_binding.listing_iam_bindings["listing_id-roles/analyticshub.subscriptionOwner"] + : condition: [] + data_exchange_id: exchange + location: us-central1 + members: + - group:subscription-owner@domain.com + project: project-id + role: roles/analyticshub.subscriptionOwner + +counts: + google_bigquery_analytics_hub_data_exchange: 1 + google_bigquery_analytics_hub_data_exchange_iam_binding: 1 + google_bigquery_analytics_hub_listing: 1 + google_bigquery_analytics_hub_listing_iam_binding: 2 + modules: 1 + resources: 5 + +outputs: {} \ No newline at end of file From a3ac39bd1512d8ed2e2b65804cc17360a9b1ea03 Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Mon, 19 Feb 2024 13:40:31 +0000 Subject: [PATCH 5/6] minor docs fix --- modules/analytics-hub/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/analytics-hub/README.md b/modules/analytics-hub/README.md index 4fa791cce1..91660989c6 100644 --- a/modules/analytics-hub/README.md +++ b/modules/analytics-hub/README.md @@ -4,10 +4,10 @@ This module allows managing [Analytics Hub](https://cloud.google.com/bigquery/do ## Examples -Listing argument references can be found in: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_listing - ### Exchange +Exchange argument references can be found in: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_data_exchange + ```hcl module "analytics-hub" { source = "./fabric/modules/analytics-hub" @@ -23,7 +23,7 @@ module "analytics-hub" { ### Listings -Listing definitions can be provided in the form in the form {LISTING_ID => LISTING_CONFIGS}. Listing argument references can be found in: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_listing +Listing definitions can be provided in the form {LISTING_ID => LISTING_CONFIGS}. Listing argument references can be found in: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_listing ```hcl module "analytics-hub" { From 96d655da2ca6273530726c9c45bde0684cb14bc7 Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Mon, 19 Feb 2024 13:55:43 +0000 Subject: [PATCH 6/6] docs fix --- modules/analytics-hub/README.md | 6 +++--- tests/modules/analytics_hub/examples/iam_exchange.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/analytics-hub/README.md b/modules/analytics-hub/README.md index 91660989c6..3482d797c4 100644 --- a/modules/analytics-hub/README.md +++ b/modules/analytics-hub/README.md @@ -86,18 +86,18 @@ module "analytics-hub" { iam_bindings = { "viewers" = { role = "roles/analyticshub.viewer" - members = ["user:thinhha@google.com"] + members = ["user:user@domain.com"] } } iam_by_principals = { - "user:thinhha@google.com" = [ + "user:user@domain.com" = [ "roles/analyticshub.viewer" ] } iam_bindings_additive = { "subscribers" = { role = "roles/analyticshub.subscriber" - member = "user:thinhha@google.com" + member = "user:user@domain.com" } } } diff --git a/tests/modules/analytics_hub/examples/iam_exchange.yaml b/tests/modules/analytics_hub/examples/iam_exchange.yaml index c377cf85ab..4ba6912004 100644 --- a/tests/modules/analytics_hub/examples/iam_exchange.yaml +++ b/tests/modules/analytics_hub/examples/iam_exchange.yaml @@ -29,14 +29,14 @@ values: location: us-central1 members: - group:viewer@domain.com - - user:thinhha@google.com + - user:user@domain.com project: project-id role: roles/analyticshub.viewer module.analytics-hub.google_bigquery_analytics_hub_data_exchange_iam_member.exchange_iam_members["subscribers"]: condition: [] data_exchange_id: exchange location: us-central1 - member: user:thinhha@google.com + member: user:user@domain.com project: project-id role: roles/analyticshub.subscriber