BigQuery Module

modules/bigquery/README.md

@@ -0,0 +1,141 @@
+# Opinionated BigQuery module
+
+TODO(jccb): add description.
+
+## Example usage
+
+```hcl
+module "bq" {
+  source     = "../modules/bigquery/"
+  project_id = "my-project"
+  datasets = [
+    {
+      id          = "dataset1"
+      name        = "dataset1"
+      description = "dataset1 description"
+      location    = "EU"
+      labels = {
+        environment = "dev"
+      }
+      # default_partition_expiration_ms = 10000
+      # default_table_expiration_ms - 10000
+    }  ]
+
+  dataset_access = {
+    dataset1 = {
+      "[email protected]" = "roles/bigquery.dataViewer"
+    }
+  }
+
+  tables = [
+    {
+      table_id    = "table1"
+      dataset_id  = "dataset1"
+      description = "dataset1 description"
+      labels = {
+        team = "sales"
+      }
+      schema = [
+        {
+          type = "DATE"
+          name = "date"
+        },
+        {
+          type = "STRING"
+          name = "name"
+        }
+      ]
+      time_partitioning = {
+        field = "date"
+        type  = "DAY"
+      }
+    }
+  ]
+
+  views = [
+    {
+      dataset = "dataset1"
+      table   = "view1"
+      query   = "select date from `my-project.dataset1.table1`"
+    }
+  ]
+}
+```
+
+## Variables
+
+| name           | description                                                 | type   | required |
+|----------------|-------------------------------------------------------------|:------:|:--------:|
+| project_id     | The ID of the project where the datasets, tables, and views | string | ✓        |
+| datasets       | List of datasets to be created (see structure below)        | string |          |
+| tables         | List of tables to be created (see structure below)          | string |          |
+| views          | List of views to be created (see structure below)           | string |          |
+| dataset_access | Dataset-level permissions to be granted                     | string |          |
+
+### dataset structure
+The datasets list should contains objects with the following structure:
+
+``` hcl
+    {
+      id          = "dataset id"
+      name        = "dataset name"
+      description = "dataset description"
+      location    = "dataset location"
+      labels = { # optional
+        label1 = "value1"
+      }
+      default_partition_expiration_ms = 10000 # optional
+      default_table_expiration_ms - 10000 # optional
+    }
+
+```
+
+### table structure
+
+``` hcl
+    {
+      table_id    = "table name"
+      dataset_id  = "table dataset"
+      description = "table description"
+      labels = {
+        label1 = "value1"
+      }
+      schema = [
+        {
+          type = "DATE"
+          name = "date"
+        }
+        {
+          type = "STRING"
+          name = "name"
+        }
+      ]
+    }
+
+```
+
+You can optionally specify ```expiration_time```,
+```time_partitioning``` and ```clustering``` for any table.
+
+
+### views structure
+
+The structure for the views field is just a dataset (where the view
+will be created), the table (view name), and the query for the view. Keep in mind that the query should use the complete name for any table it references.
+
+``` hcl
+    {
+      dataset = "dataset1"
+      table   = "view1"
+      query   = "select date from `my-project.dataset.table`"
+    }
+```
+
+
+## Outputs
+
+| name     | description                            |
+|----------|----------------------------------------|
+| datasets | Map of dataset objects keyed by name   |
+| tables   | List of table objects                  |
+| views    | List of table objects created as views |

modules/bigquery/main.tf

@@ -0,0 +1,74 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+resource "google_bigquery_dataset" "datasets" {
+  count                           = length(var.datasets)
+  project                         = var.project_id
+  dataset_id                      = var.datasets[count.index]["id"]
+  friendly_name                   = var.datasets[count.index]["name"]
+  description                     = var.datasets[count.index]["description"]
+  location                        = var.datasets[count.index]["location"]
+  labels                          = lookup(var.datasets[count.index], "labels", null)
+  default_table_expiration_ms     = lookup(var.datasets[count.index], "default_table_expiration_ms", null)
+  default_partition_expiration_ms = lookup(var.datasets[count.index], "default_partition_expiration_ms", null)
+
+  dynamic "access" {
+    for_each = lookup(var.dataset_access, var.datasets[count.index]["id"], tomap({}))
+    content {
+      user_by_email = access.key
+      role          = access.value
+    }
+  }
+
+  access {
+    role          = "OWNER"
+    special_group = "projectOwners"
+  }
+}
+
+resource "google_bigquery_table" "tables" {
+  count           = length(var.tables)
+  project         = var.project_id
+  dataset_id      = var.tables[count.index]["dataset_id"]
+  table_id        = var.tables[count.index]["table_id"]
+  labels          = var.tables[count.index]["labels"]
+  expiration_time = lookup(var.tables[count.index], "expiration_time", null)
+  clustering      = lookup(var.tables[count.index], "clustering", null)
+  schema          = jsonencode(var.tables[count.index]["schema"])
+
+  dynamic "time_partitioning" {
+    for_each = lookup(var.tables[count.index], "time_partitioning", false) == false ? [] : [1]
+    content {
+      field         = var.tables[count.index]["time_partitioning"]["field"]
+      type          = var.tables[count.index]["time_partitioning"]["type"]
+      expiration_ms = lookup(var.tables[count.index], "expiration_ms", null)
+    }
+  }
+
+  depends_on = [google_bigquery_dataset.datasets]
+}
+
+resource "google_bigquery_table" "views" {
+  count      = length(var.views)
+  project    = var.project_id
+  dataset_id = var.views[count.index]["dataset"]
+  table_id   = var.views[count.index]["table"]
+  view {
+    query          = var.views[count.index]["query"]
+    use_legacy_sql = false
+  }
+  depends_on = [google_bigquery_table.tables]
+}

modules/bigquery/outputs.tf

@@ -0,0 +1,27 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "datasets" {
+  value = zipmap(google_bigquery_dataset.datasets[*].dataset_id, google_bigquery_dataset.datasets)
+}
+
+output "tables" {
+  value = google_bigquery_table.tables
+}
+
+output "views" {
+  value = google_bigquery_table.views
+}

modules/bigquery/vars.tf

@@ -0,0 +1,70 @@
+/**
+ * Copyright 2019 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "Project ID"
+  type        = string
+}
+
+variable "datasets" {
+  description = "Datatase IDs"
+  default     = []
+  type        = list
+  # type = list(object({
+  #   id          = string
+  #   name        = string
+  #   description = string
+  #   location    = string
+  #   # labels = map(string) # optional
+  #   # default_partition_expiration_ms = number # optional
+  #   # default_table_expiration_ms = number # optional
+  # }))
+}
+
+variable "tables" {
+  description = "Tables"
+  default     = []
+  type        = list
+  # type = list(object({
+  #   table_id   = string
+  #   dataset_id = string
+  #   labels     = map(string)
+  #   schema     = string
+  #   # expiration_time = number # optional
+  #   # clustering = string # optional
+  #   # time_partitioning = object({ # optional
+  #   #   field = string
+  #   #   type = string
+  #   #   expiration_ms = number # optional
+  #   # }
+  # }))
+}
+
+variable "views" {
+  description = "Views"
+  default     = []
+  type = list(object({
+    dataset = string
+    table   = string
+    query   = string
+  }))
+}
+
+variable "dataset_access" {
+  description = "Dataset permissions"
+  default     = {}
+  type        = map(map(string))
+}