-
Notifications
You must be signed in to change notification settings - Fork 910
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
9 changed files
with
691 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,216 @@ | ||
| # BigQuery Analytics Hub | ||
|
|
||
| This module allows managing [Analytics Hub](https://cloud.google.com/bigquery/docs/analytics-hub-introduction) Exchange and Listing resources. | ||
|
|
||
| ## Examples | ||
|
|
||
| ### Exchange | ||
|
|
||
| Exchange argument references can be found in: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_data_exchange | ||
|
|
||
| ```hcl | ||
| module "analytics-hub" { | ||
| source = "./fabric/modules/analytics-hub" | ||
| project_id = "project-id" | ||
| region = "us-central1" | ||
| prefix = "test" | ||
| name = "exchange" | ||
| primary_contact = "[email protected]" | ||
| documentation = "documentation" | ||
| } | ||
| # tftest modules=1 resources=1 | ||
| ``` | ||
|
|
||
| ### Listings | ||
|
|
||
| Listing definitions can be provided in the form {LISTING_ID => LISTING_CONFIGS}. Listing argument references can be found in: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_listing | ||
|
|
||
| ```hcl | ||
| module "analytics-hub" { | ||
| source = "./fabric/modules/analytics-hub" | ||
| project_id = "project-id" | ||
| region = "us-central1" | ||
| name = "exchange" | ||
| listings = { | ||
| "listing_id" = { | ||
| bigquery_dataset = "projects/{project}/datasets/{dataset}" | ||
| }, | ||
| "listing_id_2" = { | ||
| bigquery_dataset = "projects/{project}/datasets/{dataset}" | ||
| description = "(Optional) Short description of the listing." | ||
| documentation = "(Optional) Documentation describing the listing." | ||
| categories = [] | ||
| primary_contact = "(Optional) Email or URL of the primary point of contact of the listing." | ||
| icon = "(Optional) Base64 encoded image representing the listing." | ||
| request_access = "(Optional) Email or URL of the request access of the listing. Subscribers can use this reference to request access." | ||
| data_provider = { | ||
| name = "(Required) Name of the data provider." | ||
| primary_contact = "(Optional) Email or URL of the data provider." | ||
| } | ||
| publisher = { | ||
| name = "(Required) Name of the listing publisher." | ||
| primary_contact = "(Optional) Email or URL of the listing publisher." | ||
| } | ||
| restricted_export_config = { | ||
| enabled = true | ||
| restrict_query_result = true | ||
| } | ||
| } | ||
| } | ||
| } | ||
| # tftest modules=1 resources=3 | ||
| ``` | ||
|
|
||
| ### IAM | ||
|
|
||
| This module supports setting IAM permissions on both the exchange and listing resources. IAM permissions on the exchange is inherited on the listings. | ||
|
|
||
| See [this page](https://cloud.google.com/bigquery/docs/analytics-hub-grant-roles) to see IAM roles that can be granted on exchange and listings. | ||
|
|
||
| #### Exchange | ||
| Input to variables `iam`, `iam_bindings`, and `iam_by_principals` will be merged, and are [authoritative for the given role](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_data_exchange_iam#google_bigquery_analytics_hub_data_exchange_iam_binding). Inputs to variable `iam_bindings_additive` are [additive](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_data_exchange_iam#google_bigquery_analytics_hub_data_exchange_iam_member). | ||
|
|
||
| In practice, you should only need to use either `iam` or `iam_bindings`. | ||
|
|
||
| ```hcl | ||
| module "analytics-hub" { | ||
| source = "./fabric/modules/analytics-hub" | ||
| project_id = "project-id" | ||
| region = "us-central1" | ||
| name = "exchange" | ||
| iam = { | ||
| "roles/analyticshub.viewer" = [ | ||
| "group:[email protected]" | ||
| ], | ||
| } | ||
| iam_bindings = { | ||
| "viewers" = { | ||
| role = "roles/analyticshub.viewer" | ||
| members = ["user:[email protected]"] | ||
| } | ||
| } | ||
| iam_by_principals = { | ||
| "user:[email protected]" = [ | ||
| "roles/analyticshub.viewer" | ||
| ] | ||
| } | ||
| iam_bindings_additive = { | ||
| "subscribers" = { | ||
| role = "roles/analyticshub.subscriber" | ||
| member = "user:[email protected]" | ||
| } | ||
| } | ||
| } | ||
| # tftest modules=1 resources=3 inventory=iam_exchange.yaml | ||
| ``` | ||
|
|
||
| #### Listings | ||
| The listings variable block support the `iam` input which are [authoritative for the given role](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_analytics_hub_listing_iam#google_bigquery_analytics_hub_listing_iam_binding). | ||
|
|
||
| ```hcl | ||
| module "analytics-hub" { | ||
| source = "./fabric/modules/analytics-hub" | ||
| project_id = "project-id" | ||
| region = "us-central1" | ||
| name = "exchange" | ||
| iam = { | ||
| "roles/analyticshub.viewer" = [ | ||
| "group:[email protected]" | ||
| ], | ||
| } | ||
| listings = { | ||
| "listing_id" = { | ||
| bigquery_dataset = "projects/{project}/datasets/{dataset}" | ||
| iam = { | ||
| "roles/analyticshub.subscriber" = [ | ||
| "group:[email protected]" | ||
| ], | ||
| "roles/analyticshub.subscriptionOwner" = [ | ||
| "group:[email protected]" | ||
| ], | ||
| } | ||
| } | ||
| } | ||
| } | ||
| # tftest modules=1 resources=5 inventory=iam_listing.yaml | ||
| ``` | ||
|
|
||
| ### Factory | ||
|
|
||
| Similarly to other modules, a rules factory (see [Resource Factories](../../blueprints/factories/)) is also included here to allow managing listings inside the same exchange via descriptive configuration files. | ||
|
|
||
| Factory configuration is via one optional attributes in the `factory_config_path` variable specifying the path where tags files are stored. | ||
|
|
||
| Factory tags are merged with rules declared in code, with the latter taking precedence where both use the same key. | ||
|
|
||
| This is an example of a simple factory: | ||
|
|
||
| ```hcl | ||
| module "analytics-hub" { | ||
| source = "./fabric/modules/analytics-hub" | ||
| project_id = "project-id" | ||
| region = "us-central1" | ||
| name = "exchange" | ||
| listings = { | ||
| "listing_id" = { | ||
| bigquery_dataset = "projects/{project}/datasets/{dataset}" | ||
| }, | ||
| } | ||
| factories_config = { | ||
| listings = "listings" | ||
| } | ||
| } | ||
| # tftest modules=1 resources=5 files=yaml | ||
| ``` | ||
|
|
||
| ```yaml | ||
| # tftest-file id=yaml path=listings/listing_1.yaml | ||
| bigquery_dataset: projects/{project}/datasets/{dataset} | ||
| description: "(Optional) Short description of the listing." | ||
| documentation: "(Optional) Documentation describing the listing." | ||
| categories: [] | ||
| icon: "(Optional) Base64 encoded image representing the listing." | ||
| primary_contact: "(Optional) Email or URL of the primary point of contact of the listing." | ||
| request_access: "(Optional) Email or URL of the request access of the listing. Subscribers can use this reference to request access." | ||
| data_provider: | ||
| name: "(Required) Name of the data provider." | ||
| primary_contact: "(Optional) Email or URL of the data provider." | ||
| iam: | ||
| roles/analyticshub.subscriber: | ||
| - group:[email protected] | ||
| roles/analyticshub.subscriptionOwner: | ||
| - group:[email protected] | ||
| publisher: | ||
| name: "(Required) Name of the listing publisher." | ||
| primary_contact: "(Optional) Email or URL of the listing publisher." | ||
| restricted_export_config: | ||
| enabled: true | ||
| restrict_query_result: true | ||
| ``` | ||
| <!-- BEGIN TFDOC --> | ||
| ## Variables | ||
| | name | description | type | required | default | | ||
| |---|---|:---:|:---:|:---:| | ||
| | [name](variables.tf#L71) | The ID of the data exchange. Must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping or characters outside of ASCII spaces. | <code>string</code> | ✓ | | | ||
| | [project_id](variables.tf#L88) | The ID of the project where the data exchange will be created. | <code>string</code> | ✓ | | | ||
| | [region](variables.tf#L93) | Region for the data exchange. | <code>string</code> | ✓ | | | ||
| | [description](variables.tf#L17) | Resource description for data exchange. | <code>string</code> | | <code>null</code> | | ||
| | [documentation](variables.tf#L23) | Documentation describing the data exchange. | <code>string</code> | | <code>null</code> | | ||
| | [factories_config](variables.tf#L29) | Paths to data files and folders that enable factory functionality. | <code title="object({ listings = optional(string) })">object({…})</code> | | <code>{}</code> | | ||
| | [iam](variables-iam.tf#L17) | Authoritative IAM bindings in {ROLE => [MEMBERS]} format. | <code>map(list(string))</code> | | <code>{}</code> | | ||
| | [iam_bindings](variables-iam.tf#L24) | Authoritative IAM bindings in {KEY => {role = ROLE, members = []}}. Keys are arbitrary. | <code title="map(object({ members = list(string) role = string }))">map(object({…}))</code> | | <code>{}</code> | | ||
| | [iam_bindings_additive](variables-iam.tf#L34) | Individual additive IAM bindings. Keys are arbitrary. | <code title="map(object({ member = string role = string }))">map(object({…}))</code> | | <code>{}</code> | | ||
| | [iam_by_principals](variables-iam.tf#L44) | Authoritative IAM binding in {PRINCIPAL => [ROLES]} format. Principals need to be statically defined to avoid cycle errors. Merged internally with the `iam` variable. | <code>map(list(string))</code> | | <code>{}</code> | | ||
| | [icon](variables.tf#L38) | Base64 encoded image representing the data exchange. | <code>string</code> | | <code>null</code> | | ||
| | [listings](variables.tf#L44) | Listings definitions in the form {LISTING_ID => LISTING_CONFIGS}. LISTING_ID must contain only Unicode letters, numbers (0-9), underscores (_). Should not use characters that require URL-escaping or characters outside of ASCII spaces. | <code title="map(object({ bigquery_dataset = string description = optional(string) documentation = optional(string) categories = optional(list(string)) icon = optional(string) primary_contact = optional(string) request_access = optional(string) data_provider = optional(object({ name = string primary_contact = optional(string) })) iam = optional(map(list(string))) publisher = optional(object({ name = string primary_contact = optional(string) })) restricted_export_config = optional(object({ enabled = optional(bool) restrict_query_result = optional(bool) })) }))">map(object({…}))</code> | | <code>{}</code> | | ||
| | [prefix](variables.tf#L76) | Optional prefix for data exchange ID. | <code>string</code> | | <code>null</code> | | ||
| | [primary_contact](variables.tf#L82) | Email or URL of the primary point of contact of the data exchange. | <code>string</code> | | <code>null</code> | | ||
|
|
||
| ## Outputs | ||
|
|
||
| | name | description | sensitive | | ||
| |---|---|:---:| | ||
| | [data_exchange_id](outputs.tf#L17) | Data exchange id. | | | ||
| | [data_listings](outputs.tf#L27) | Data listings and corresponding configs. | | | ||
| <!-- END TFDOC --> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| /** | ||
| * Copyright 2024 Google LLC | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| locals { | ||
| _exchange_iam_principal_roles = distinct(flatten(values(var.iam_by_principals))) | ||
| _exchange_iam_principals = { | ||
| for r in local._exchange_iam_principal_roles : r => [ | ||
| for k, v in var.iam_by_principals : | ||
| k if try(index(v, r), null) != null | ||
| ] | ||
| } | ||
| _exchange_iam_bindings = { | ||
| for key, iam_bindings in var.iam_bindings : | ||
| iam_bindings.role => iam_bindings.members | ||
| } | ||
| exchange_iam = { | ||
| for role in distinct(concat(keys(var.iam), keys(local._exchange_iam_principals), keys(local._exchange_iam_bindings))) : | ||
| role => concat( | ||
| try(var.iam[role], []), | ||
| try(local._exchange_iam_principals[role], []), | ||
| try(local._exchange_iam_bindings[role], []), | ||
| ) | ||
| } | ||
| listing_iam = flatten([ | ||
| for listing_id, listing_configs in local.factory_listings : [ | ||
| for role, members in coalesce(listing_configs.iam, tomap({})) : { | ||
| listing_id = listing_id | ||
| role = role | ||
| members = members | ||
| } | ||
| ] | ||
| ]) | ||
| } | ||
|
|
||
|
|
||
| resource "google_bigquery_analytics_hub_data_exchange_iam_binding" "exchange_iam_bindings" { | ||
| for_each = local.exchange_iam | ||
| project = google_bigquery_analytics_hub_data_exchange.data_exchange.project | ||
| location = google_bigquery_analytics_hub_data_exchange.data_exchange.location | ||
| data_exchange_id = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id | ||
| role = each.key | ||
| members = each.value | ||
| } | ||
|
|
||
| resource "google_bigquery_analytics_hub_data_exchange_iam_member" "exchange_iam_members" { | ||
| for_each = var.iam_bindings_additive | ||
| project = google_bigquery_analytics_hub_data_exchange.data_exchange.project | ||
| location = google_bigquery_analytics_hub_data_exchange.data_exchange.location | ||
| data_exchange_id = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id | ||
| role = each.value.role | ||
| member = each.value.member | ||
| } | ||
|
|
||
| resource "google_bigquery_analytics_hub_listing_iam_binding" "listing_iam_bindings" { | ||
| for_each = { | ||
| for index, listing_iam in local.listing_iam : | ||
| "${listing_iam.listing_id}-${listing_iam.role}" => listing_iam | ||
| } | ||
| project = google_bigquery_analytics_hub_data_exchange.data_exchange.project | ||
| location = google_bigquery_analytics_hub_data_exchange.data_exchange.location | ||
| data_exchange_id = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id | ||
| listing_id = google_bigquery_analytics_hub_listing.listing[each.value.listing_id].id | ||
| role = each.value.role | ||
| members = each.value.members | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| /** | ||
| * Copyright 2024 Google LLC | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| locals { | ||
| prefix = var.prefix == null || var.prefix == "" ? "" : "${var.prefix}_" | ||
| _factory_listings = { | ||
| for f in try(fileset(var.factories_config.listings, "*.yaml"), []) : | ||
| trimsuffix(f, ".yaml") => yamldecode(file("${var.factories_config.listings}/${f}")) | ||
| } | ||
|
|
||
| factory_listings = merge(local._factory_listings, var.listings) | ||
| } | ||
|
|
||
| resource "google_bigquery_analytics_hub_data_exchange" "data_exchange" { | ||
| project = var.project_id | ||
| location = var.region | ||
| data_exchange_id = "${local.prefix}${var.name}" | ||
| display_name = "${local.prefix}${var.name}" | ||
| description = var.description | ||
| primary_contact = var.primary_contact | ||
| documentation = var.documentation | ||
| icon = var.icon | ||
| } | ||
|
|
||
| resource "google_bigquery_analytics_hub_listing" "listing" { | ||
| for_each = local.factory_listings | ||
| project = var.project_id | ||
| location = var.region | ||
| data_exchange_id = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id | ||
| listing_id = each.key | ||
| display_name = each.key | ||
| description = try(each.value.description, null) | ||
| primary_contact = try(each.value.primary_contact, null) | ||
| documentation = try(each.value.documentation, null) | ||
| icon = try(each.value.icon, null) | ||
| request_access = try(each.value.request_access, null) | ||
| categories = try(each.value.categories, null) | ||
|
|
||
| bigquery_dataset { | ||
| dataset = each.value.bigquery_dataset | ||
| } | ||
|
|
||
| dynamic "restricted_export_config" { | ||
| for_each = each.value.restricted_export_config != null ? [""] : [] | ||
| content { | ||
| enabled = try(each.value.restricted_export_config.enabled, null) | ||
| restrict_query_result = try(each.value.restricted_export_config.restrict_query_result, null) | ||
| } | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| /** | ||
| * Copyright 2024 Google LLC | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| output "data_exchange_id" { | ||
| description = "Data exchange id." | ||
| value = google_bigquery_analytics_hub_data_exchange.data_exchange.data_exchange_id | ||
| depends_on = [ | ||
| google_bigquery_analytics_hub_data_exchange.data_exchange, | ||
| google_bigquery_analytics_hub_data_exchange_iam_binding.exchange_iam_bindings, | ||
| google_bigquery_analytics_hub_data_exchange_iam_member.exchange_iam_members | ||
| ] | ||
| } | ||
|
|
||
| output "data_listings" { | ||
| description = "Data listings and corresponding configs." | ||
| value = { for k, v in google_bigquery_analytics_hub_listing.listing : k => v.bigquery_dataset } | ||
| depends_on = [ | ||
| google_bigquery_analytics_hub_listing.listing, | ||
| google_bigquery_analytics_hub_listing_iam_binding.listing_iam_bindings | ||
| ] | ||
| } |
Oops, something went wrong.