From 7503bceaede40757b03f8fa2f356b0060c069d80 Mon Sep 17 00:00:00 2001 From: devuonocar <65766241+devuonocar@users.noreply.github.com> Date: Wed, 4 Oct 2023 14:25:56 +0200 Subject: [PATCH] Add materialized views for bigquery (#1726) * add field and partition filter to time_partitioning for bq table * add materialized view * set optional fields for table * Fix README.md * Update README.md * Fix README.md * Replace explicit dependencies with implicit ones * Fix indexes * Update table_id(s) values * Try to fix tests * Update tests * Restore explicit dependencies * Update README.md * Update README.md --------- Co-authored-by: Ludovico Magnocavallo --- modules/bigquery-dataset/README.md | 9 ++--- modules/bigquery-dataset/main.tf | 50 +++++++++++++++++++++++++-- modules/bigquery-dataset/variables.tf | 39 +++++++++++++++++++-- 3 files changed, 89 insertions(+), 9 deletions(-) diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index 944dee9982..67ec627312 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -281,7 +281,7 @@ module "bigquery-dataset" { | name | description | type | required | default | |---|---|:---:|:---:|:---:| | [id](variables.tf#L98) | Dataset id. | string | ✓ | | -| [project_id](variables.tf#L129) | Id of the project where datasets will be created. | string | ✓ | | +| [project_id](variables.tf#L162) | Id of the project where datasets will be created. | string | ✓ | | | [access](variables.tf#L17) | Map of access rules with role and identity type. Keys are arbitrary and must match those in the `access_identities` variable, types are `domain`, `group`, `special_group`, `user`, `view`. | map(object({…})) | | {} | | [access_identities](variables.tf#L33) | Map of access identities used for basic access roles. View identities have the format 'project_id\|dataset_id\|table_id'. | map(string) | | {} | | [authorized_datasets](variables.tf#L39) | An array of datasets to be authorized on the dataset. | list(object({…})) | | [] | @@ -294,9 +294,10 @@ module "bigquery-dataset" { | [iam](variables.tf#L92) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | | [labels](variables.tf#L103) | Dataset labels. | map(string) | | {} | | [location](variables.tf#L109) | Dataset location. | string | | "EU" | -| [options](variables.tf#L115) | Dataset options. | object({…}) | | {} | -| [tables](variables.tf#L134) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | -| [views](variables.tf#L163) | View definitions. | map(object({…})) | | {} | +| [materialized_views](variables.tf#L115) | Materialized views definitions. | map(object({…})) | | {} | +| [options](variables.tf#L148) | Dataset options. | object({…}) | | {} | +| [tables](variables.tf#L167) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | +| [views](variables.tf#L198) | View definitions. | map(object({…})) | | {} | ## Outputs diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf index 107cda2014..af276a4780 100644 --- a/modules/bigquery-dataset/main.tf +++ b/modules/bigquery-dataset/main.tf @@ -244,9 +244,10 @@ resource "google_bigquery_table" "default" { dynamic "time_partitioning" { for_each = try(each.value.partitioning.time, null) != null ? [""] : [] content { - expiration_ms = each.value.partitioning.time.expiration_ms - field = each.value.partitioning.field - type = each.value.partitioning.time.type + expiration_ms = each.value.partitioning.time.expiration_ms + field = each.value.partitioning.time.field + type = each.value.partitioning.time.type + require_partition_filter = each.value.partitioning.time.require_partition_filter } } } @@ -267,3 +268,46 @@ resource "google_bigquery_table" "views" { use_legacy_sql = each.value.use_legacy_sql } } + +resource "google_bigquery_table" "materialized_view" { + depends_on = [google_bigquery_table.default] + for_each = var.materialized_views + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + table_id = each.key + friendly_name = each.value.friendly_name + description = each.value.description + labels = each.value.labels + clustering = each.value.options.clustering + expiration_time = each.value.options.expiration_time + deletion_protection = each.value.deletion_protection + + dynamic "range_partitioning" { + for_each = try(each.value.partitioning.range, null) != null ? [""] : [] + content { + field = each.value.partitioning.field + range { + start = each.value.partitioning.range.start + end = each.value.partitioning.range.end + interval = each.value.partitioning.range.interval + } + } + } + + dynamic "time_partitioning" { + for_each = try(each.value.partitioning.time, null) != null ? [""] : [] + content { + expiration_ms = each.value.partitioning.time.expiration_ms + field = each.value.partitioning.time.field + type = each.value.partitioning.time.type + require_partition_filter = each.value.partitioning.time.require_partition_filter + } + } + + materialized_view { + query = each.value.query + enable_refresh = each.value.enable_refresh + refresh_interval_ms = each.value.refresh_interval_ms + allow_non_incremental_definition = each.value.allow_non_incremental_definition + } +} diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf index 1ecc5f4a43..ecf7c6c3fb 100644 --- a/modules/bigquery-dataset/variables.tf +++ b/modules/bigquery-dataset/variables.tf @@ -112,6 +112,39 @@ variable "location" { default = "EU" } +variable "materialized_views" { + description = "Materialized views definitions." + type = map(object({ + query = string + deletion_protection = optional(bool) + description = optional(string, "Terraform managed.") + friendly_name = optional(string) + labels = optional(map(string), {}) + enable_refresh = optional(bool) + refresh_interval_ms = optional(bool) + allow_non_incremental_definition = optional(bool) + options = optional(object({ + clustering = optional(list(string)) + expiration_time = optional(number) + }), {}) + partitioning = optional(object({ + field = optional(string) + range = optional(object({ + end = number + interval = number + start = number + })) + time = optional(object({ + type = string + expiration_ms = optional(number) + field = optional(string) + require_partition_filter = optional(bool) + })) + })) + })) + default = {} +} + variable "options" { description = "Dataset options." type = object({ @@ -152,8 +185,10 @@ variable "tables" { start = number })) time = optional(object({ - expiration_ms = number - type = string + type = string + expiration_ms = optional(number) + field = optional(string) + require_partition_filter = optional(bool) })) })) }))