From 3ecb7b160aa229603e624246a9f158f902a446fb Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 09:30:36 +0000 Subject: [PATCH 01/16] add bq authorized views, datasets, and routines --- modules/bigquery-dataset/README.md | 35 ++++++++ modules/bigquery-dataset/main.tf | 79 ++++++++++++++++--- modules/bigquery-dataset/outputs.tf | 12 ++- modules/bigquery-dataset/variables.tf | 29 +++++++ .../examples/authorized_resources.yaml | 38 +++++++++ 5 files changed, 178 insertions(+), 15 deletions(-) create mode 100644 tests/modules/bigquery_dataset/examples/authorized_resources.yaml diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index a1b2b2dd07..accec31b13 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -54,6 +54,41 @@ module "bigquery-dataset" { # tftest modules=1 resources=2 inventory=iam.yaml ``` +### Authorized Views, Datasets, and Routines + +You can specify authorized views, datasets, and routines via the `authorized_views`, `authorized_datasets` and `authorized_routines` variables, respectively. + +```hcl +module "bigquery-dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = "my-project" + id = "my-dataset" + authorized_views = [ + { + project_id = "view_project" + dataset_id = "view_dataset" + table_id = "view_id" + } + ] + authorized_datasets = [ + { + project_id = "auth_dataset_project" + dataset_id = "auth_dataset" + } + ] + authorized_routines = [ + { + project_id = "auth_routine_project" + dataset_id = "auth_routine_dataset" + routine_id = "auth_routine" + } + ] +} +# tftest modules=1 resources=4 inventory=authorized_resources.yaml +``` + +Authorized views can be specified both using the standard `access` options and the `authorized_views` blocks. + ### Dataset options Dataset options are set via the `options` variable. all options must be specified, but a `null` value can be set to options that need to use defaults. diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf index 62d752244f..b4fbeef4f3 100644 --- a/modules/bigquery-dataset/main.tf +++ b/modules/bigquery-dataset/main.tf @@ -20,15 +20,23 @@ locals { access_special = { for k, v in var.access : k => v if v.type == "special_group" } access_user = { for k, v in var.access : k => v if v.type == "user" } access_view = { for k, v in var.access : k => v if v.type == "view" } + identities_view = { for k, v in local.access_view : k => try( zipmap( - ["project", "dataset", "table"], + ["project_id", "dataset_id", "table_id"], split("|", var.access_identities[k]) ), - { project = null, dataset = null, table = null } + { project_id = null, dataset_id = null, table_id = null } ) } + + authorized_views = merge( + { for access_key, view in local.identities_view: "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view.value }, + { for view in var.authorized_views : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view }) + authorized_datasets = { for dataset in var.authorized_datasets : "${dataset["project_id"]}_${dataset["dataset_id"]}" => dataset } + authorized_routines = { for routine in var.authorized_routines : "${routine["project_id"]}_${routine["dataset_id"]}_${routine["routine_id"]}" => routine } + } resource "google_bigquery_dataset" "default" { @@ -78,12 +86,36 @@ resource "google_bigquery_dataset" "default" { } dynamic "access" { - for_each = var.dataset_access ? local.access_view : {} + for_each = var.dataset_access ? local.authorized_views : {} content { view { - project_id = local.identities_view[access.key].project - dataset_id = local.identities_view[access.key].dataset - table_id = local.identities_view[access.key].table + project_id = each.value.project_id + dataset_id = each.value.dataset_id + table_id = each.value.table_id + } + } + } + + dynamic "access" { + for_each = var.dataset_access ? local.authorized_datasets : {} + content { + dataset { + dataset { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + } + target_types = ["VIEWS"] + } + } + } + + dynamic "access" { + for_each = var.dataset_access ? local.authorized_routines : {} + content { + routine { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + routine_id = each.value.routine_id } } } @@ -132,15 +164,38 @@ resource "google_bigquery_dataset_access" "user_by_email" { user_by_email = try(var.access_identities[each.key]) } -resource "google_bigquery_dataset_access" "views" { - for_each = var.dataset_access ? {} : local.access_view - provider = google-beta +resource "google_bigquery_dataset_access" "authorized_views" { + for_each = var.dataset_access ? {} : local.authorized_views project = var.project_id dataset_id = google_bigquery_dataset.default.dataset_id view { - project_id = local.identities_view[each.key].project - dataset_id = local.identities_view[each.key].dataset - table_id = local.identities_view[each.key].table + project_id = each.value.project_id + dataset_id = each.value.dataset_id + table_id = each.value.table_id + } +} + +resource "google_bigquery_dataset_access" "authorized_datasets" { + for_each = var.dataset_access ? {} : local.authorized_datasets + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + dataset { + dataset { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + } + target_types = ["VIEWS"] + } +} + +resource "google_bigquery_dataset_access" "authorized_routines" { + for_each = var.dataset_access ? {} : local.authorized_routines + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + routine { + project_id = each.value.project_id + dataset_id = each.value.dataset_id + routine_id = each.value.routine_id } } diff --git a/modules/bigquery-dataset/outputs.tf b/modules/bigquery-dataset/outputs.tf index dd2da22c69..a3fb06d9f5 100644 --- a/modules/bigquery-dataset/outputs.tf +++ b/modules/bigquery-dataset/outputs.tf @@ -27,7 +27,9 @@ output "dataset_id" { google_bigquery_dataset_access.group_by_email, google_bigquery_dataset_access.special_group, google_bigquery_dataset_access.user_by_email, - google_bigquery_dataset_access.views + google_bigquery_dataset_access.authorized_views, + google_bigquery_dataset_access.authorized_datasets, + google_bigquery_dataset_access.authorized_routines ] } @@ -39,7 +41,9 @@ output "id" { google_bigquery_dataset_access.group_by_email, google_bigquery_dataset_access.special_group, google_bigquery_dataset_access.user_by_email, - google_bigquery_dataset_access.views + google_bigquery_dataset_access.authorized_views, + google_bigquery_dataset_access.authorized_datasets, + google_bigquery_dataset_access.authorized_routines ] } @@ -51,7 +55,9 @@ output "self_link" { google_bigquery_dataset_access.group_by_email, google_bigquery_dataset_access.special_group, google_bigquery_dataset_access.user_by_email, - google_bigquery_dataset_access.views + google_bigquery_dataset_access.authorized_views, + google_bigquery_dataset_access.authorized_datasets, + google_bigquery_dataset_access.authorized_routines ] } diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf index c738845e51..8a8ac00a94 100644 --- a/modules/bigquery-dataset/variables.tf +++ b/modules/bigquery-dataset/variables.tf @@ -140,3 +140,32 @@ variable "views" { })) default = {} } + +variable "authorized_views" { + description = "An array of views to give authorize for the dataset" + type = list(object({ + dataset_id = string, + project_id = string, + table_id = string # this is the view id, but we keep table_id to stay consistent as the resource + })) + default = [] +} + +variable "authorized_datasets" { + description = "An array of datasets to be authorized on the dataset" + type = list(object({ + dataset_id = string, + project_id = string, + })) + default = [] +} + +variable "authorized_routines" { + description = "An array of authorized routine to be authorized on the dataset" + type = list(object({ + project_id = string, + dataset_id = string, + routine_id = string + })) + default = [] +} diff --git a/tests/modules/bigquery_dataset/examples/authorized_resources.yaml b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml new file mode 100644 index 0000000000..12b84505ca --- /dev/null +++ b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml @@ -0,0 +1,38 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.bigquery-dataset.google_bigquery_dataset.default: + dataset_id: my-dataset + project_id: my-project + module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["auth_dataset_project_auth_dataset"]: + dataset: + dataset: + dataset_id: view_project + project_id: view_dataset + target_types: ["VIEWS"] + module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["view_project_view_dataset_view_id"]: + view: + dataset_id: auth_dataset_project + project_id: auth_dataset + table_id: view_id + module.bigquery-dataset.google_bigquery_dataset_access.authorized_routines["auth_routine_project_auth_routine_dataset_auth_routine"]: + routine: + dataset_id: auth_dataset_project + project_id: auth_dataset + routine_id: auth_routine + +counts: + google_bigquery_dataset: 1 + google_bigquery_dataset_access: 3 From d5a342471204375740a99dfe17614e4df4b3feb4 Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 09:35:29 +0000 Subject: [PATCH 02/16] fmt --- modules/bigquery-dataset/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf index b4fbeef4f3..d7d40827ca 100644 --- a/modules/bigquery-dataset/main.tf +++ b/modules/bigquery-dataset/main.tf @@ -31,8 +31,8 @@ locals { ) } - authorized_views = merge( - { for access_key, view in local.identities_view: "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view.value }, + authorized_views = merge( + { for access_key, view in local.identities_view : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view.value }, { for view in var.authorized_views : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view }) authorized_datasets = { for dataset in var.authorized_datasets : "${dataset["project_id"]}_${dataset["dataset_id"]}" => dataset } authorized_routines = { for routine in var.authorized_routines : "${routine["project_id"]}_${routine["dataset_id"]}_${routine["routine_id"]}" => routine } From 7636a3ad7b8aeb8ebbfa0138d5a962296dfcdd4a Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 09:46:59 +0000 Subject: [PATCH 03/16] add backwards compatible test case --- modules/bigquery-dataset/README.md | 31 +++++++++++++++- modules/bigquery-dataset/main.tf | 2 +- .../examples/authorized_resources.yaml | 12 +++--- .../examples/authorized_resources_views.yaml | 37 +++++++++++++++++++ 4 files changed, 74 insertions(+), 8 deletions(-) create mode 100644 tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index accec31b13..e86fe50694 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -87,7 +87,36 @@ module "bigquery-dataset" { # tftest modules=1 resources=4 inventory=authorized_resources.yaml ``` -Authorized views can be specified both using the standard `access` options and the `authorized_views` blocks. +Authorized views can be specified both using the standard `access` options and the `authorized_views` blocks. The example configuration below uses both blocks will create a dataset with three authorized views `view_id_1`, `view_id_2`, and `view_id_3`. + +```hcl +module "bigquery-dataset" { + source = "./fabric/modules/bigquery-dataset" + project_id = "my-project" + id = "my-dataset" + authorized_views = [ + { + project_id = "view_project" + dataset_id = "view_dataset" + table_id = "view_id_1" + }, + { + project_id = "view_project" + dataset_id = "view_dataset" + table_id = "view_id_2" + } + ] + access = { + view_1 = { role = "READER", type = "view" } + view_2 = { role = "READER", type = "view" } + } + access_identities = { + view_1 = "view_project|view_dataset|view_id_2" + view_2 = "view_project|view_dataset|view_id_3" + } +} +# tftest modules=1 resources=4 inventory=authorized_resources_views.yaml +``` ### Dataset options diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf index d7d40827ca..0a66b8292e 100644 --- a/modules/bigquery-dataset/main.tf +++ b/modules/bigquery-dataset/main.tf @@ -32,7 +32,7 @@ locals { } authorized_views = merge( - { for access_key, view in local.identities_view : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view.value }, + { for access_key, view in local.identities_view : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view }, { for view in var.authorized_views : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view }) authorized_datasets = { for dataset in var.authorized_datasets : "${dataset["project_id"]}_${dataset["dataset_id"]}" => dataset } authorized_routines = { for routine in var.authorized_routines : "${routine["project_id"]}_${routine["dataset_id"]}_${routine["routine_id"]}" => routine } diff --git a/tests/modules/bigquery_dataset/examples/authorized_resources.yaml b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml index 12b84505ca..be241b08d4 100644 --- a/tests/modules/bigquery_dataset/examples/authorized_resources.yaml +++ b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml @@ -22,17 +22,17 @@ values: dataset_id: view_project project_id: view_dataset target_types: ["VIEWS"] - module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["view_project_view_dataset_view_id"]: - view: - dataset_id: auth_dataset_project - project_id: auth_dataset - table_id: view_id module.bigquery-dataset.google_bigquery_dataset_access.authorized_routines["auth_routine_project_auth_routine_dataset_auth_routine"]: routine: dataset_id: auth_dataset_project project_id: auth_dataset routine_id: auth_routine + module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["view_project_view_dataset_view_id"]: + view: + dataset_id: view_dataset + project_id: view_project + table_id: view_id counts: google_bigquery_dataset: 1 - google_bigquery_dataset_access: 3 + google_bigquery_dataset_access: 4 diff --git a/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml b/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml new file mode 100644 index 0000000000..297a3d424a --- /dev/null +++ b/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml @@ -0,0 +1,37 @@ +# Copyright 2023 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +values: + module.bigquery-dataset.google_bigquery_dataset.default: + dataset_id: my-dataset + project_id: my-project + module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["view_project_view_dataset_view_id_1"]: + view: + dataset_id: view_dataset + project_id: view_project + table_id: view_id_1 + module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["view_project_view_dataset_view_id_2"]: + view: + dataset_id: view_dataset + project_id: view_project + table_id: view_id_2 + module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["view_project_view_dataset_view_id_3"]: + view: + dataset_id: view_dataset + project_id: view_project + table_id: view_id_3 + +counts: + google_bigquery_dataset: 1 + google_bigquery_dataset_access: 4 From 1ff761bf7b2ce4d61ffc56f43bb47aa1be0e243f Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 09:50:12 +0000 Subject: [PATCH 04/16] minor docs clarification --- modules/bigquery-dataset/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index e86fe50694..95085cf172 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -87,7 +87,7 @@ module "bigquery-dataset" { # tftest modules=1 resources=4 inventory=authorized_resources.yaml ``` -Authorized views can be specified both using the standard `access` options and the `authorized_views` blocks. The example configuration below uses both blocks will create a dataset with three authorized views `view_id_1`, `view_id_2`, and `view_id_3`. +Authorized views can be specified both using the standard `access` options and the `authorized_views` blocks. The example configuration below uses both blocks, and will create a dataset with three authorized views `view_id_1`, `view_id_2`, and `view_id_3`. ```hcl module "bigquery-dataset" { @@ -107,12 +107,12 @@ module "bigquery-dataset" { } ] access = { - view_1 = { role = "READER", type = "view" } view_2 = { role = "READER", type = "view" } + view_3 = { role = "READER", type = "view" } } access_identities = { - view_1 = "view_project|view_dataset|view_id_2" - view_2 = "view_project|view_dataset|view_id_3" + view_2 = "view_project|view_dataset|view_id_2" + view_3 = "view_project|view_dataset|view_id_3" } } # tftest modules=1 resources=4 inventory=authorized_resources_views.yaml From e26f0a89fb4d2752b929c3dda17ee62260a08e43 Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 10:00:55 +0000 Subject: [PATCH 05/16] add docs table and fix lint tests --- modules/bigquery-dataset/README.md | 15 +++++++++------ modules/bigquery-dataset/variables.tf | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index 95085cf172..5d19b51937 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -251,6 +251,9 @@ module "bigquery-dataset" { | [encryption_key](variables.tf#L51) | Self link of the KMS key that will be used to protect destination table. | string | | null | | [friendly_name](variables.tf#L57) | Dataset friendly name. | string | | null | | [iam](variables.tf#L63) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | +| [authorized_views](variables.tf#L144) | An array of views to be authorized for the dataset. | list(object(...)) | | [] | +| [authorized_datasets](variables.tf#L154) | An array of datasets to be authorized on the dataset. | list(object(...)) | | [] | +| [authorized_routines](variables.tf#L163) | An array of authorized routine to be authorized on the dataset. | list(object(...)) | | [] | | [labels](variables.tf#L74) | Dataset labels. | map(string) | | {} | | [location](variables.tf#L80) | Dataset location. | string | | "EU" | | [options](variables.tf#L86) | Dataset options. | object({…}) | | {} | @@ -263,11 +266,11 @@ module "bigquery-dataset" { |---|---|:---:| | [dataset](outputs.tf#L17) | Dataset resource. | | | [dataset_id](outputs.tf#L22) | Dataset id. | | -| [id](outputs.tf#L34) | Fully qualified dataset id. | | -| [self_link](outputs.tf#L46) | Dataset self link. | | -| [table_ids](outputs.tf#L58) | Map of fully qualified table ids keyed by table ids. | | -| [tables](outputs.tf#L63) | Table resources. | | -| [view_ids](outputs.tf#L68) | Map of fully qualified view ids keyed by view ids. | | -| [views](outputs.tf#L73) | View resources. | | +| [id](outputs.tf#L36) | Fully qualified dataset id. | | +| [self_link](outputs.tf#L50) | Dataset self link. | | +| [table_ids](outputs.tf#L64) | Map of fully qualified table ids keyed by table ids. | | +| [tables](outputs.tf#L69) | Table resources. | | +| [view_ids](outputs.tf#L74) | Map of fully qualified view ids keyed by view ids. | | +| [views](outputs.tf#L79) | View resources. | | diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf index 8a8ac00a94..af7608dc7e 100644 --- a/modules/bigquery-dataset/variables.tf +++ b/modules/bigquery-dataset/variables.tf @@ -142,7 +142,7 @@ variable "views" { } variable "authorized_views" { - description = "An array of views to give authorize for the dataset" + description = "An array of views to be authorized for the dataset" type = list(object({ dataset_id = string, project_id = string, From 40b4fb263be8155f177c60b36a93d8811734016b Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 10:02:25 +0000 Subject: [PATCH 06/16] fix docs --- modules/bigquery-dataset/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index 5d19b51937..4d7c506dd1 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -246,14 +246,14 @@ module "bigquery-dataset" { | [project_id](variables.tf#L99) | Id of the project where datasets will be created. | string | ✓ | | | [access](variables.tf#L17) | Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`. | map(object({…})) | | {} | | [access_identities](variables.tf#L33) | Map of access identities used for basic access roles. View identities have the format 'project_id\|dataset_id\|table_id'. | map(string) | | {} | +| [authorized_datasets](variables.tf#L154) | An array of datasets to be authorized on the dataset | list(object({…})) | | [] | +| [authorized_routines](variables.tf#L163) | An array of authorized routine to be authorized on the dataset | list(object({…})) | | [] | +| [authorized_views](variables.tf#L144) | An array of views to be authorized for the dataset | list(object({…})) | | [] | | [dataset_access](variables.tf#L39) | Set access in the dataset resource instead of using separate resources. | bool | | false | | [description](variables.tf#L45) | Optional description. | string | | "Terraform managed." | | [encryption_key](variables.tf#L51) | Self link of the KMS key that will be used to protect destination table. | string | | null | | [friendly_name](variables.tf#L57) | Dataset friendly name. | string | | null | | [iam](variables.tf#L63) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | -| [authorized_views](variables.tf#L144) | An array of views to be authorized for the dataset. | list(object(...)) | | [] | -| [authorized_datasets](variables.tf#L154) | An array of datasets to be authorized on the dataset. | list(object(...)) | | [] | -| [authorized_routines](variables.tf#L163) | An array of authorized routine to be authorized on the dataset. | list(object(...)) | | [] | | [labels](variables.tf#L74) | Dataset labels. | map(string) | | {} | | [location](variables.tf#L80) | Dataset location. | string | | "EU" | | [options](variables.tf#L86) | Dataset options. | object({…}) | | {} | From 640c6e7ddebcd1252c8635892f026867132f7b53 Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 10:07:03 +0000 Subject: [PATCH 07/16] fix tests --- .../modules/bigquery_dataset/examples/authorized_resources.yaml | 2 +- .../bigquery_dataset/examples/authorized_resources_views.yaml | 2 +- tests/modules/bigquery_dataset/examples/simple.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/modules/bigquery_dataset/examples/authorized_resources.yaml b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml index be241b08d4..5a913c9981 100644 --- a/tests/modules/bigquery_dataset/examples/authorized_resources.yaml +++ b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml @@ -15,7 +15,7 @@ values: module.bigquery-dataset.google_bigquery_dataset.default: dataset_id: my-dataset - project_id: my-project + project: my-project module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["auth_dataset_project_auth_dataset"]: dataset: dataset: diff --git a/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml b/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml index 297a3d424a..2afd7a8ae7 100644 --- a/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml +++ b/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml @@ -15,7 +15,7 @@ values: module.bigquery-dataset.google_bigquery_dataset.default: dataset_id: my-dataset - project_id: my-project + project: my-project module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["view_project_view_dataset_view_id_1"]: view: dataset_id: view_dataset diff --git a/tests/modules/bigquery_dataset/examples/simple.yaml b/tests/modules/bigquery_dataset/examples/simple.yaml index acf8e819c6..171b5b88ca 100644 --- a/tests/modules/bigquery_dataset/examples/simple.yaml +++ b/tests/modules/bigquery_dataset/examples/simple.yaml @@ -33,7 +33,7 @@ values: project: my-project role: OWNER user_by_email: ludo@ludomagno.net - module.bigquery-dataset.google_bigquery_dataset_access.views["view_1"]: + module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_1"]: dataset_id: my-dataset project: my-project view: From 28b139ce8e95406e337482fdaaefbd043874ef2e Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 10:10:25 +0000 Subject: [PATCH 08/16] fix more lint issues --- modules/bigquery-dataset/README.md | 30 +++++++------- modules/bigquery-dataset/variables.tf | 58 +++++++++++++-------------- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index 4d7c506dd1..1da2b14397 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -242,23 +242,23 @@ module "bigquery-dataset" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| -| [id](variables.tf#L69) | Dataset id. | string | ✓ | | -| [project_id](variables.tf#L99) | Id of the project where datasets will be created. | string | ✓ | | +| [id](variables.tf#L98) | Dataset id. | string | ✓ | | +| [project_id](variables.tf#L128) | Id of the project where datasets will be created. | string | ✓ | | | [access](variables.tf#L17) | Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`. | map(object({…})) | | {} | | [access_identities](variables.tf#L33) | Map of access identities used for basic access roles. View identities have the format 'project_id\|dataset_id\|table_id'. | map(string) | | {} | -| [authorized_datasets](variables.tf#L154) | An array of datasets to be authorized on the dataset | list(object({…})) | | [] | -| [authorized_routines](variables.tf#L163) | An array of authorized routine to be authorized on the dataset | list(object({…})) | | [] | -| [authorized_views](variables.tf#L144) | An array of views to be authorized for the dataset | list(object({…})) | | [] | -| [dataset_access](variables.tf#L39) | Set access in the dataset resource instead of using separate resources. | bool | | false | -| [description](variables.tf#L45) | Optional description. | string | | "Terraform managed." | -| [encryption_key](variables.tf#L51) | Self link of the KMS key that will be used to protect destination table. | string | | null | -| [friendly_name](variables.tf#L57) | Dataset friendly name. | string | | null | -| [iam](variables.tf#L63) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | -| [labels](variables.tf#L74) | Dataset labels. | map(string) | | {} | -| [location](variables.tf#L80) | Dataset location. | string | | "EU" | -| [options](variables.tf#L86) | Dataset options. | object({…}) | | {} | -| [tables](variables.tf#L104) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | -| [views](variables.tf#L132) | View definitions. | map(object({…})) | | {} | +| [authorized_datasets](variables.tf#L39) | An array of datasets to be authorized on the dataset | list(object({…})) | | [] | +| [authorized_routines](variables.tf#L48) | An array of authorized routine to be authorized on the dataset | list(object({…})) | | [] | +| [authorized_views](variables.tf#L58) | An array of views to be authorized for the dataset | list(object({…})) | | [] | +| [dataset_access](variables.tf#L68) | Set access in the dataset resource instead of using separate resources. | bool | | false | +| [description](variables.tf#L74) | Optional description. | string | | "Terraform managed." | +| [encryption_key](variables.tf#L80) | Self link of the KMS key that will be used to protect destination table. | string | | null | +| [friendly_name](variables.tf#L86) | Dataset friendly name. | string | | null | +| [iam](variables.tf#L92) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | +| [labels](variables.tf#L103) | Dataset labels. | map(string) | | {} | +| [location](variables.tf#L109) | Dataset location. | string | | "EU" | +| [options](variables.tf#L115) | Dataset options. | object({…}) | | {} | +| [tables](variables.tf#L133) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | +| [views](variables.tf#L161) | View definitions. | map(object({…})) | | {} | ## Outputs diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf index af7608dc7e..62c40beff6 100644 --- a/modules/bigquery-dataset/variables.tf +++ b/modules/bigquery-dataset/variables.tf @@ -36,6 +36,35 @@ variable "access_identities" { default = {} } +variable "authorized_datasets" { + description = "An array of datasets to be authorized on the dataset" + type = list(object({ + dataset_id = string, + project_id = string, + })) + default = [] +} + +variable "authorized_routines" { + description = "An array of authorized routine to be authorized on the dataset" + type = list(object({ + project_id = string, + dataset_id = string, + routine_id = string + })) + default = [] +} + +variable "authorized_views" { + description = "An array of views to be authorized for the dataset" + type = list(object({ + dataset_id = string, + project_id = string, + table_id = string # this is the view id, but we keep table_id to stay consistent as the resource + })) + default = [] +} + variable "dataset_access" { description = "Set access in the dataset resource instead of using separate resources." type = bool @@ -140,32 +169,3 @@ variable "views" { })) default = {} } - -variable "authorized_views" { - description = "An array of views to be authorized for the dataset" - type = list(object({ - dataset_id = string, - project_id = string, - table_id = string # this is the view id, but we keep table_id to stay consistent as the resource - })) - default = [] -} - -variable "authorized_datasets" { - description = "An array of datasets to be authorized on the dataset" - type = list(object({ - dataset_id = string, - project_id = string, - })) - default = [] -} - -variable "authorized_routines" { - description = "An array of authorized routine to be authorized on the dataset" - type = list(object({ - project_id = string, - dataset_id = string, - routine_id = string - })) - default = [] -} From 08e3cee3e5f03f2ca076f33ebce01305b138efea Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 10:25:30 +0000 Subject: [PATCH 09/16] fix lint issue --- modules/bigquery-dataset/variables.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf index 62c40beff6..695435fdc0 100644 --- a/modules/bigquery-dataset/variables.tf +++ b/modules/bigquery-dataset/variables.tf @@ -37,7 +37,7 @@ variable "access_identities" { } variable "authorized_datasets" { - description = "An array of datasets to be authorized on the dataset" + description = "An array of datasets to be authorized on the dataset." type = list(object({ dataset_id = string, project_id = string, @@ -46,7 +46,7 @@ variable "authorized_datasets" { } variable "authorized_routines" { - description = "An array of authorized routine to be authorized on the dataset" + description = "An array of authorized routine to be authorized on the dataset." type = list(object({ project_id = string, dataset_id = string, @@ -56,7 +56,7 @@ variable "authorized_routines" { } variable "authorized_views" { - description = "An array of views to be authorized for the dataset" + description = "An array of views to be authorized for the dataset." type = list(object({ dataset_id = string, project_id = string, From fdd3cc3d69f79bb95d0005b01589d26de5bd181b Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 10:29:00 +0000 Subject: [PATCH 10/16] fix tests --- .../bigquery_dataset/examples/authorized_resources.yaml | 4 ++-- .../examples/authorized_resources_views.yaml | 6 +++--- tests/modules/bigquery_dataset/examples/simple.yaml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/modules/bigquery_dataset/examples/authorized_resources.yaml b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml index 5a913c9981..9fc3f426dd 100644 --- a/tests/modules/bigquery_dataset/examples/authorized_resources.yaml +++ b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml @@ -16,7 +16,7 @@ values: module.bigquery-dataset.google_bigquery_dataset.default: dataset_id: my-dataset project: my-project - module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["auth_dataset_project_auth_dataset"]: + module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["auth_dataset_project_auth_dataset"]: dataset: dataset: dataset_id: view_project @@ -27,7 +27,7 @@ values: dataset_id: auth_dataset_project project_id: auth_dataset routine_id: auth_routine - module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["view_project_view_dataset_view_id"]: + module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_project_view_dataset_view_id"]: view: dataset_id: view_dataset project_id: view_project diff --git a/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml b/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml index 2afd7a8ae7..3edf5fb767 100644 --- a/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml +++ b/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml @@ -16,17 +16,17 @@ values: module.bigquery-dataset.google_bigquery_dataset.default: dataset_id: my-dataset project: my-project - module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["view_project_view_dataset_view_id_1"]: + module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_project_view_dataset_view_id_1"]: view: dataset_id: view_dataset project_id: view_project table_id: view_id_1 - module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["view_project_view_dataset_view_id_2"]: + module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_project_view_dataset_view_id_2"]: view: dataset_id: view_dataset project_id: view_project table_id: view_id_2 - module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["view_project_view_dataset_view_id_3"]: + module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_project_view_dataset_view_id_3"]: view: dataset_id: view_dataset project_id: view_project diff --git a/tests/modules/bigquery_dataset/examples/simple.yaml b/tests/modules/bigquery_dataset/examples/simple.yaml index 171b5b88ca..490778ad18 100644 --- a/tests/modules/bigquery_dataset/examples/simple.yaml +++ b/tests/modules/bigquery_dataset/examples/simple.yaml @@ -33,7 +33,7 @@ values: project: my-project role: OWNER user_by_email: ludo@ludomagno.net - module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_1"]: + module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["my-project_my-dataset_my-table"]: dataset_id: my-dataset project: my-project view: From 0959d300d9ed96ba81637608145a05056399ffa3 Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 10:32:36 +0000 Subject: [PATCH 11/16] fix lint --- modules/bigquery-dataset/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index 1da2b14397..5686d8a84a 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -246,9 +246,9 @@ module "bigquery-dataset" { | [project_id](variables.tf#L128) | Id of the project where datasets will be created. | string | ✓ | | | [access](variables.tf#L17) | Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`. | map(object({…})) | | {} | | [access_identities](variables.tf#L33) | Map of access identities used for basic access roles. View identities have the format 'project_id\|dataset_id\|table_id'. | map(string) | | {} | -| [authorized_datasets](variables.tf#L39) | An array of datasets to be authorized on the dataset | list(object({…})) | | [] | -| [authorized_routines](variables.tf#L48) | An array of authorized routine to be authorized on the dataset | list(object({…})) | | [] | -| [authorized_views](variables.tf#L58) | An array of views to be authorized for the dataset | list(object({…})) | | [] | +| [authorized_datasets](variables.tf#L39) | An array of datasets to be authorized on the dataset. | list(object({…})) | | [] | +| [authorized_routines](variables.tf#L48) | An array of authorized routine to be authorized on the dataset. | list(object({…})) | | [] | +| [authorized_views](variables.tf#L58) | An array of views to be authorized for the dataset. | list(object({…})) | | [] | | [dataset_access](variables.tf#L68) | Set access in the dataset resource instead of using separate resources. | bool | | false | | [description](variables.tf#L74) | Optional description. | string | | "Terraform managed." | | [encryption_key](variables.tf#L80) | Self link of the KMS key that will be used to protect destination table. | string | | null | From 11f32dbfd77507c53785e486792303f94802216e Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 15:50:03 +0000 Subject: [PATCH 12/16] fix tests --- modules/bigquery-dataset/README.md | 10 ++++----- .../examples/authorized_resources.yaml | 22 +++++++++---------- .../examples/authorized_resources_views.yaml | 21 +++++++++--------- 3 files changed, 26 insertions(+), 27 deletions(-) diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index 5686d8a84a..c1f1c9428d 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -80,7 +80,7 @@ module "bigquery-dataset" { { project_id = "auth_routine_project" dataset_id = "auth_routine_dataset" - routine_id = "auth_routine" + routine_id = "auth_routine" } ] } @@ -107,12 +107,12 @@ module "bigquery-dataset" { } ] access = { - view_2 = { role = "READER", type = "view" } - view_3 = { role = "READER", type = "view" } + view_2 = { role = "READER", type = "view" } + view_3 = { role = "READER", type = "view" } } access_identities = { - view_2 = "view_project|view_dataset|view_id_2" - view_3 = "view_project|view_dataset|view_id_3" + view_2 = "view_project|view_dataset|view_id_2" + view_3 = "view_project|view_dataset|view_id_3" } } # tftest modules=1 resources=4 inventory=authorized_resources_views.yaml diff --git a/tests/modules/bigquery_dataset/examples/authorized_resources.yaml b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml index 9fc3f426dd..9c6b61f128 100644 --- a/tests/modules/bigquery_dataset/examples/authorized_resources.yaml +++ b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml @@ -18,21 +18,21 @@ values: project: my-project module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["auth_dataset_project_auth_dataset"]: dataset: - dataset: - dataset_id: view_project - project_id: view_dataset - target_types: ["VIEWS"] + - dataset: + - dataset_id: auth_dataset + project_id: auth_dataset_project + target_types: ["VIEWS"] module.bigquery-dataset.google_bigquery_dataset_access.authorized_routines["auth_routine_project_auth_routine_dataset_auth_routine"]: routine: - dataset_id: auth_dataset_project - project_id: auth_dataset - routine_id: auth_routine + - dataset_id: auth_routine_dataset + project_id: auth_routine_project + routine_id: auth_routine module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_project_view_dataset_view_id"]: view: - dataset_id: view_dataset - project_id: view_project - table_id: view_id + - dataset_id: view_dataset + project_id: view_project + table_id: view_id counts: google_bigquery_dataset: 1 - google_bigquery_dataset_access: 4 + google_bigquery_dataset_access: 3 diff --git a/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml b/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml index 3edf5fb767..25f0b3a8da 100644 --- a/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml +++ b/tests/modules/bigquery_dataset/examples/authorized_resources_views.yaml @@ -18,20 +18,19 @@ values: project: my-project module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_project_view_dataset_view_id_1"]: view: - dataset_id: view_dataset - project_id: view_project - table_id: view_id_1 + - dataset_id: view_dataset + project_id: view_project + table_id: view_id_1 module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_project_view_dataset_view_id_2"]: view: - dataset_id: view_dataset - project_id: view_project - table_id: view_id_2 + - dataset_id: view_dataset + project_id: view_project + table_id: view_id_2 module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_project_view_dataset_view_id_3"]: view: - dataset_id: view_dataset - project_id: view_project - table_id: view_id_3 - + - dataset_id: view_dataset + project_id: view_project + table_id: view_id_3 counts: google_bigquery_dataset: 1 - google_bigquery_dataset_access: 4 + google_bigquery_dataset_access: 3 From 19673c902da89b0a345caa93f4082b6f1c13231e Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Thu, 29 Jun 2023 16:36:09 +0000 Subject: [PATCH 13/16] alphabetical order --- modules/bigquery-dataset/outputs.tf | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/bigquery-dataset/outputs.tf b/modules/bigquery-dataset/outputs.tf index a3fb06d9f5..5c2ee4661d 100644 --- a/modules/bigquery-dataset/outputs.tf +++ b/modules/bigquery-dataset/outputs.tf @@ -23,13 +23,13 @@ output "dataset_id" { description = "Dataset id." value = google_bigquery_dataset.default.dataset_id depends_on = [ + google_bigquery_dataset_access.authorized_datasets, + google_bigquery_dataset_access.authorized_routines, + google_bigquery_dataset_access.authorized_views, google_bigquery_dataset_access.domain, google_bigquery_dataset_access.group_by_email, google_bigquery_dataset_access.special_group, - google_bigquery_dataset_access.user_by_email, - google_bigquery_dataset_access.authorized_views, - google_bigquery_dataset_access.authorized_datasets, - google_bigquery_dataset_access.authorized_routines + google_bigquery_dataset_access.user_by_email ] } @@ -37,13 +37,13 @@ output "id" { description = "Fully qualified dataset id." value = google_bigquery_dataset.default.id depends_on = [ + google_bigquery_dataset_access.authorized_datasets, + google_bigquery_dataset_access.authorized_routines, + google_bigquery_dataset_access.authorized_views, google_bigquery_dataset_access.domain, google_bigquery_dataset_access.group_by_email, google_bigquery_dataset_access.special_group, - google_bigquery_dataset_access.user_by_email, - google_bigquery_dataset_access.authorized_views, - google_bigquery_dataset_access.authorized_datasets, - google_bigquery_dataset_access.authorized_routines + google_bigquery_dataset_access.user_by_email ] } @@ -51,13 +51,13 @@ output "self_link" { description = "Dataset self link." value = google_bigquery_dataset.default.self_link depends_on = [ + google_bigquery_dataset_access.authorized_datasets, + google_bigquery_dataset_access.authorized_routines, + google_bigquery_dataset_access.authorized_views, google_bigquery_dataset_access.domain, google_bigquery_dataset_access.group_by_email, google_bigquery_dataset_access.special_group, - google_bigquery_dataset_access.user_by_email, - google_bigquery_dataset_access.authorized_views, - google_bigquery_dataset_access.authorized_datasets, - google_bigquery_dataset_access.authorized_routines + google_bigquery_dataset_access.user_by_email ] } From 69422ae6547d67e31591a7e97aa38e433ef51725 Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Fri, 30 Jun 2023 10:33:57 +0000 Subject: [PATCH 14/16] add detailed example to docs and corresponding test --- modules/bigquery-dataset/README.md | 68 ++++++++++++++++--- .../examples/authorized_resources.yaml | 24 ++++--- 2 files changed, 74 insertions(+), 18 deletions(-) diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index c1f1c9428d..0443fda4dc 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -56,18 +56,19 @@ module "bigquery-dataset" { ### Authorized Views, Datasets, and Routines -You can specify authorized views, datasets, and routines via the `authorized_views`, `authorized_datasets` and `authorized_routines` variables, respectively. +You can specify authorized [views](https://cloud.google.com/bigquery/docs/authorized-views), [datasets](https://cloud.google.com/bigquery/docs/authorized-datasets?hl=en), and [routines](https://cloud.google.com/bigquery/docs/authorized-routines) via the `authorized_views`, `authorized_datasets` and `authorized_routines` variables, respectively. ```hcl -module "bigquery-dataset" { +// Create private BigQuery dataset that will not be publicly accessible, except via the authorized BigQuery resources +module "bigquery-dataset-private" { source = "./fabric/modules/bigquery-dataset" - project_id = "my-project" - id = "my-dataset" + project_id = "private_project" + id = "private_dataset" authorized_views = [ { - project_id = "view_project" - dataset_id = "view_dataset" - table_id = "view_id" + project_id = "auth_view_project" + dataset_id = "auth_view_dataset" + table_id = "auth_view" } ] authorized_datasets = [ @@ -84,7 +85,58 @@ module "bigquery-dataset" { } ] } -# tftest modules=1 resources=4 inventory=authorized_resources.yaml + +// Create authorized view in a public dataset +module "bigquery-authorized-views-dataset-public" { + source = "./fabric/modules/bigquery-dataset" + project_id = "auth_view_project" + id = "auth_view_dataset" + views = { + auth_view = { + friendly_name = "Public" + labels = {} + query = "SELECT * FROM `private_project.private_dataset.private_table`" + use_legacy_sql = false + deletion_protection = true + } + } + depends_on = [module.bigquery-dataset-private] +} + +// Create public authorized dataset +module "bigquery-authorized-dataset-public" { + source = "./fabric/modules/bigquery-dataset" + project_id = "auth_dataset_project" + id = "auth_dataset" + depends_on = [module.bigquery-dataset-private] +} + +// Create public authorized routine +module "bigquery-authorized-authorized-routine-dataset-public" { + source = "./fabric/modules/bigquery-dataset" + project_id = "auth_routine_project" + id = "auth_routine_dataset" + depends_on = [module.bigquery-dataset-private] +} + +resource "google_bigquery_routine" "public-routine" { + dataset_id = module.bigquery-authorized-authorized-routine-dataset-public.dataset_id + routine_id = "auth_routine" + routine_type = "TABLE_VALUED_FUNCTION" + language = "SQL" + definition_body = <<-EOS + SELECT 1 + value AS value + EOS + arguments { + name = "value" + argument_kind = "FIXED_TYPE" + data_type = jsonencode({ "typeKind" = "INT64" }) + } + return_table_type = jsonencode({ "columns" = [ + { "name" = "value", "type" = { "typeKind" = "INT64" } }, + ] }) +} +# tftest modules=4 resources=9 inventory=authorized_resources.yaml ``` Authorized views can be specified both using the standard `access` options and the `authorized_views` blocks. The example configuration below uses both blocks, and will create a dataset with three authorized views `view_id_1`, `view_id_2`, and `view_id_3`. diff --git a/tests/modules/bigquery_dataset/examples/authorized_resources.yaml b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml index 9c6b61f128..518ce6fcb4 100644 --- a/tests/modules/bigquery_dataset/examples/authorized_resources.yaml +++ b/tests/modules/bigquery_dataset/examples/authorized_resources.yaml @@ -13,26 +13,30 @@ # limitations under the License. values: - module.bigquery-dataset.google_bigquery_dataset.default: - dataset_id: my-dataset - project: my-project - module.bigquery-dataset.google_bigquery_dataset_access.authorized_datasets["auth_dataset_project_auth_dataset"]: + module.bigquery-dataset-private.google_bigquery_dataset.default: + dataset_id: private_dataset + project: private_project + module.bigquery-dataset-private.google_bigquery_dataset_access.authorized_datasets["auth_dataset_project_auth_dataset"]: dataset: - dataset: - dataset_id: auth_dataset project_id: auth_dataset_project target_types: ["VIEWS"] - module.bigquery-dataset.google_bigquery_dataset_access.authorized_routines["auth_routine_project_auth_routine_dataset_auth_routine"]: + module.bigquery-dataset-private.google_bigquery_dataset_access.authorized_routines["auth_routine_project_auth_routine_dataset_auth_routine"]: routine: - dataset_id: auth_routine_dataset project_id: auth_routine_project routine_id: auth_routine - module.bigquery-dataset.google_bigquery_dataset_access.authorized_views["view_project_view_dataset_view_id"]: + module.bigquery-dataset-private.google_bigquery_dataset_access.authorized_views["auth_view_project_auth_view_dataset_auth_view"]: view: - - dataset_id: view_dataset - project_id: view_project - table_id: view_id + - dataset_id: auth_view_dataset + project_id: auth_view_project + table_id: auth_view counts: - google_bigquery_dataset: 1 + google_bigquery_dataset: 4 google_bigquery_dataset_access: 3 + google_bigquery_routine: 1 + google_bigquery_table: 1 + modules: 4 + resources: 9 From dae6111256bdedb0544c2e4bf7d86f0ec235858c Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Fri, 30 Jun 2023 10:37:51 +0000 Subject: [PATCH 15/16] clarify docs for variables --- modules/bigquery-dataset/README.md | 2 +- modules/bigquery-dataset/variables.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index 0443fda4dc..db26f505df 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -300,7 +300,7 @@ module "bigquery-dataset" { | [access_identities](variables.tf#L33) | Map of access identities used for basic access roles. View identities have the format 'project_id\|dataset_id\|table_id'. | map(string) | | {} | | [authorized_datasets](variables.tf#L39) | An array of datasets to be authorized on the dataset. | list(object({…})) | | [] | | [authorized_routines](variables.tf#L48) | An array of authorized routine to be authorized on the dataset. | list(object({…})) | | [] | -| [authorized_views](variables.tf#L58) | An array of views to be authorized for the dataset. | list(object({…})) | | [] | +| [authorized_views](variables.tf#L58) | An array of views to be authorized on the dataset. | list(object({…})) | | [] | | [dataset_access](variables.tf#L68) | Set access in the dataset resource instead of using separate resources. | bool | | false | | [description](variables.tf#L74) | Optional description. | string | | "Terraform managed." | | [encryption_key](variables.tf#L80) | Self link of the KMS key that will be used to protect destination table. | string | | null | diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf index 695435fdc0..66eb893482 100644 --- a/modules/bigquery-dataset/variables.tf +++ b/modules/bigquery-dataset/variables.tf @@ -56,7 +56,7 @@ variable "authorized_routines" { } variable "authorized_views" { - description = "An array of views to be authorized for the dataset." + description = "An array of views to be authorized on the dataset." type = list(object({ dataset_id = string, project_id = string, From 09c94620fa939696c5dcc8f5fa1255306e7d5066 Mon Sep 17 00:00:00 2001 From: Thinh Ha Date: Fri, 30 Jun 2023 14:29:52 +0100 Subject: [PATCH 16/16] remove unnecessary depends_on --- modules/bigquery-dataset/README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index db26f505df..b440859bcd 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -100,7 +100,6 @@ module "bigquery-authorized-views-dataset-public" { deletion_protection = true } } - depends_on = [module.bigquery-dataset-private] } // Create public authorized dataset @@ -108,7 +107,6 @@ module "bigquery-authorized-dataset-public" { source = "./fabric/modules/bigquery-dataset" project_id = "auth_dataset_project" id = "auth_dataset" - depends_on = [module.bigquery-dataset-private] } // Create public authorized routine @@ -116,7 +114,6 @@ module "bigquery-authorized-authorized-routine-dataset-public" { source = "./fabric/modules/bigquery-dataset" project_id = "auth_routine_project" id = "auth_routine_dataset" - depends_on = [module.bigquery-dataset-private] } resource "google_bigquery_routine" "public-routine" {