diff --git a/modules/bigquery/README.md b/modules/bigquery/README.md new file mode 100644 index 0000000000..55a7a09ef2 --- /dev/null +++ b/modules/bigquery/README.md @@ -0,0 +1,141 @@ +# Opinionated BigQuery module + +TODO(jccb): add description. + +## Example usage + +```hcl +module "bq" { + source = "../modules/bigquery/" + project_id = "my-project" + datasets = [ + { + id = "dataset1" + name = "dataset1" + description = "dataset1 description" + location = "EU" + labels = { + environment = "dev" + } + # default_partition_expiration_ms = 10000 + # default_table_expiration_ms - 10000 + } ] + + dataset_access = { + dataset1 = { + "me@myorg.com" = "roles/bigquery.dataViewer" + } + } + + tables = [ + { + table_id = "table1" + dataset_id = "dataset1" + description = "dataset1 description" + labels = { + team = "sales" + } + schema = [ + { + type = "DATE" + name = "date" + }, + { + type = "STRING" + name = "name" + } + ] + time_partitioning = { + field = "date" + type = "DAY" + } + } + ] + + views = [ + { + dataset = "dataset1" + table = "view1" + query = "select date from `my-project.dataset1.table1`" + } + ] +} +``` + +## Variables + +| name | description | type | required | +|----------------|-------------------------------------------------------------|:------:|:--------:| +| project_id | The ID of the project where the datasets, tables, and views | string | ✓ | +| datasets | List of datasets to be created (see structure below) | string | | +| tables | List of tables to be created (see structure below) | string | | +| views | List of views to be created (see structure below) | string | | +| dataset_access | Dataset-level permissions to be granted | string | | + +### dataset structure +The datasets list should contains objects with the following structure: + +``` hcl + { + id = "dataset id" + name = "dataset name" + description = "dataset description" + location = "dataset location" + labels = { # optional + label1 = "value1" + } + default_partition_expiration_ms = 10000 # optional + default_table_expiration_ms - 10000 # optional + } + +``` + +### table structure + +``` hcl + { + table_id = "table name" + dataset_id = "table dataset" + description = "table description" + labels = { + label1 = "value1" + } + schema = [ + { + type = "DATE" + name = "date" + } + { + type = "STRING" + name = "name" + } + ] + } + +``` + +You can optionally specify ```expiration_time```, +```time_partitioning``` and ```clustering``` for any table. + + +### views structure + +The structure for the views field is just a dataset (where the view +will be created), the table (view name), and the query for the view. Keep in mind that the query should use the complete name for any table it references. + +``` hcl + { + dataset = "dataset1" + table = "view1" + query = "select date from `my-project.dataset.table`" + } +``` + + +## Outputs + +| name | description | +|----------|----------------------------------------| +| datasets | Map of dataset objects keyed by name | +| tables | List of table objects | +| views | List of table objects created as views | diff --git a/modules/bigquery/main.tf b/modules/bigquery/main.tf new file mode 100644 index 0000000000..7421d22f01 --- /dev/null +++ b/modules/bigquery/main.tf @@ -0,0 +1,74 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_bigquery_dataset" "datasets" { + count = length(var.datasets) + project = var.project_id + dataset_id = var.datasets[count.index]["id"] + friendly_name = var.datasets[count.index]["name"] + description = var.datasets[count.index]["description"] + location = var.datasets[count.index]["location"] + labels = lookup(var.datasets[count.index], "labels", null) + default_table_expiration_ms = lookup(var.datasets[count.index], "default_table_expiration_ms", null) + default_partition_expiration_ms = lookup(var.datasets[count.index], "default_partition_expiration_ms", null) + + dynamic "access" { + for_each = lookup(var.dataset_access, var.datasets[count.index]["id"], tomap({})) + content { + user_by_email = access.key + role = access.value + } + } + + access { + role = "OWNER" + special_group = "projectOwners" + } +} + +resource "google_bigquery_table" "tables" { + count = length(var.tables) + project = var.project_id + dataset_id = var.tables[count.index]["dataset_id"] + table_id = var.tables[count.index]["table_id"] + labels = var.tables[count.index]["labels"] + expiration_time = lookup(var.tables[count.index], "expiration_time", null) + clustering = lookup(var.tables[count.index], "clustering", null) + schema = jsonencode(var.tables[count.index]["schema"]) + + dynamic "time_partitioning" { + for_each = lookup(var.tables[count.index], "time_partitioning", false) == false ? [] : [1] + content { + field = var.tables[count.index]["time_partitioning"]["field"] + type = var.tables[count.index]["time_partitioning"]["type"] + expiration_ms = lookup(var.tables[count.index], "expiration_ms", null) + } + } + + depends_on = [google_bigquery_dataset.datasets] +} + +resource "google_bigquery_table" "views" { + count = length(var.views) + project = var.project_id + dataset_id = var.views[count.index]["dataset"] + table_id = var.views[count.index]["table"] + view { + query = var.views[count.index]["query"] + use_legacy_sql = false + } + depends_on = [google_bigquery_table.tables] +} diff --git a/modules/bigquery/outputs.tf b/modules/bigquery/outputs.tf new file mode 100644 index 0000000000..6eabd78f13 --- /dev/null +++ b/modules/bigquery/outputs.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "datasets" { + value = zipmap(google_bigquery_dataset.datasets[*].dataset_id, google_bigquery_dataset.datasets) +} + +output "tables" { + value = google_bigquery_table.tables +} + +output "views" { + value = google_bigquery_table.views +} diff --git a/modules/bigquery/vars.tf b/modules/bigquery/vars.tf new file mode 100644 index 0000000000..077507be44 --- /dev/null +++ b/modules/bigquery/vars.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "Project ID" + type = string +} + +variable "datasets" { + description = "Datatase IDs" + default = [] + type = list + # type = list(object({ + # id = string + # name = string + # description = string + # location = string + # # labels = map(string) # optional + # # default_partition_expiration_ms = number # optional + # # default_table_expiration_ms = number # optional + # })) +} + +variable "tables" { + description = "Tables" + default = [] + type = list + # type = list(object({ + # table_id = string + # dataset_id = string + # labels = map(string) + # schema = string + # # expiration_time = number # optional + # # clustering = string # optional + # # time_partitioning = object({ # optional + # # field = string + # # type = string + # # expiration_ms = number # optional + # # } + # })) +} + +variable "views" { + description = "Views" + default = [] + type = list(object({ + dataset = string + table = string + query = string + })) +} + +variable "dataset_access" { + description = "Dataset permissions" + default = {} + type = map(map(string)) +}