From 7c3272eaa22a98068ac13ba523b2254eaa853bfb Mon Sep 17 00:00:00 2001 From: Luigi Bitonti Date: Wed, 15 Nov 2023 15:06:29 +0100 Subject: [PATCH] Updated bigquery module --- modules/bigquery-dataset/README.md | 6 +- modules/bigquery-dataset/main.tf | 154 ++++++++++++++++++++------ modules/bigquery-dataset/variables.tf | 80 ++++++++++--- 3 files changed, 193 insertions(+), 47 deletions(-) diff --git a/modules/bigquery-dataset/README.md b/modules/bigquery-dataset/README.md index ab523129c1..fc5cd2aef1 100644 --- a/modules/bigquery-dataset/README.md +++ b/modules/bigquery-dataset/README.md @@ -294,10 +294,10 @@ module "bigquery-dataset" { | [iam](variables.tf#L92) | IAM bindings in {ROLE => [MEMBERS]} format. Mutually exclusive with the access_* variables used for basic roles. | map(list(string)) | | {} | | [labels](variables.tf#L103) | Dataset labels. | map(string) | | {} | | [location](variables.tf#L109) | Dataset location. | string | | "EU" | -| [materialized_views](variables.tf#L115) | Materialized views definitions. | map(object({…})) | | {} | +| [materialized_views](variables.tf#L115) | Materialized views definitions. | map(object({…})) | | {} | | [options](variables.tf#L148) | Dataset options. | object({…}) | | {} | -| [tables](variables.tf#L167) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | -| [views](variables.tf#L198) | View definitions. | map(object({…})) | | {} | +| [tables](variables.tf#L167) | Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null. | map(object({…})) | | {} | +| [views](variables.tf#L252) | View definitions. | map(object({…})) | | {} | ## Outputs diff --git a/modules/bigquery-dataset/main.tf b/modules/bigquery-dataset/main.tf index af276a4780..c4a02ff55f 100644 --- a/modules/bigquery-dataset/main.tf +++ b/modules/bigquery-dataset/main.tf @@ -209,18 +209,20 @@ resource "google_bigquery_dataset_iam_binding" "bindings" { } resource "google_bigquery_table" "default" { - provider = google-beta - for_each = var.tables - project = var.project_id - dataset_id = google_bigquery_dataset.default.dataset_id - table_id = each.key - friendly_name = each.value.friendly_name - description = each.value.description - clustering = each.value.options.clustering - expiration_time = each.value.options.expiration_time - labels = each.value.labels - schema = each.value.schema - deletion_protection = each.value.deletion_protection + provider = google-beta + for_each = var.tables + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + table_id = each.key + friendly_name = each.value.friendly_name + description = each.value.description + clustering = each.value.options.clustering + expiration_time = each.value.options.expiration_time + labels = each.value.labels + max_staleness = each.value.options.max_staleness + schema = each.value.schema + deletion_protection = each.value.deletion_protection + require_partition_filter = each.value.require_partition_filter dynamic "encryption_configuration" { for_each = each.value.options.encryption_key != null ? [""] : [] @@ -229,6 +231,97 @@ resource "google_bigquery_table" "default" { } } + dynamic "external_data_configuration" { + for_each = each.value.external_data_configuration != null ? [""] : [] + content { + autodetect = each.value.external_data_configuration.autodetect + compression = each.value.external_data_configuration.compression + connection_id = each.value.external_data_configuration.connection_id + file_set_spec_type = each.value.external_data_configuration.file_set_spec_type + ignore_unknown_values = each.value.external_data_configuration.ignore_unknown_values + max_bad_records = each.value.external_data_configuration.max_bad_records + metadata_cache_mode = each.value.external_data_configuration.metadata_cache_mode + object_metadata = each.value.external_data_configuration.object_metadata + reference_file_schema_uri = each.value.external_data_configuration.reference_file_schema_uri + schema = each.value.external_data_configuration.schema + source_format = each.value.external_data_configuration.source_format + source_uris = each.value.external_data_configuration.source_uris + + dynamic "avro_options" { + for_each = each.value.external_data_configuration.avro_logical_types != null ? [""] : [] + content { + use_avro_logical_types = each.value.external_data_configuration.avro_logical_types + } + } + dynamic "csv_options" { + for_each = each.value.external_data_configuration.csv_options != null ? [""] : [] + content { + quote = each.value.external_data_configuration.csv_options.quote + allow_jagged_rows = each.value.external_data_configuration.csv_options.allow_jagged_rows + allow_quoted_newlines = each.value.external_data_configuration.csv_options.allow_quoted_newlines + encoding = each.value.external_data_configuration.csv_options.encoding + field_delimiter = each.value.external_data_configuration.csv_options.field_delimiter + skip_leading_rows = each.value.external_data_configuration.csv_options.skip_leading_rows + } + } + dynamic "json_options" { + for_each = each.value.external_data_configuration.json_options_encoding != null ? [""] : [] + content { + encoding = each.value.external_data_configuration.json_options_encoding + } + } + dynamic "google_sheets_options" { + for_each = each.value.external_data_configuration.google_sheets_options != null ? [""] : [] + content { + range = each.value.external_data_configuration.google_sheets_options.range + skip_leading_rows = each.value.external_data_configuration.google_sheets_options.skip_leading_rows + } + } + dynamic "hive_partitioning_options" { + for_each = each.value.external_data_configuration.hive_partitioning_options != null ? [""] : [] + content { + mode = each.value.external_data_configuration.hive_partitioning_options.mode + require_partition_filter = each.value.external_data_configuration.hive_partitioning_options.require_partition_filter + source_uri_prefix = each.value.external_data_configuration.hive_partitioning_options.source_uri_prefix + } + } + dynamic "parquet_options" { + for_each = each.value.external_data_configuration.parquet_options != null ? [""] : [] + content { + enum_as_string = each.value.external_data_configuration.parquet_options.enum_as_string + enable_list_inference = each.value.external_data_configuration.parquet_options.enable_list_inference + } + } + } + } + + dynamic "table_constraints" { + for_each = each.value.table_constraints != null ? [""] : [] + content { + dynamic "primary_key" { + for_each = each.value.table_constraints.primary_key_columns != null ? [""] : [] + content { + columns = each.value.table_constraints.primary_key_columns + } + } + dynamic "foreign_keys" { + for_each = each.value.table_constraints.foreign_keys != null ? [""] : [] + content { + name = each.value.table_constraints.foreign_keys.name + referenced_table { + project_id = each.value.table_constraints.foreign_keys.referenced_table.project_id + dataset_id = each.value.table_constraints.foreign_keys.referenced_table.dataset_id + table_id = each.value.table_constraints.foreign_keys.referenced_table.table_id + } + column_references { + referencing_column = each.value.table_constraints.foreign_keys.column_references.referencing_column + referenced_column = each.value.table_constraints.foreign_keys.column_references.referenced_column + } + } + } + } + } + dynamic "range_partitioning" { for_each = try(each.value.partitioning.range, null) != null ? [""] : [] content { @@ -244,10 +337,9 @@ resource "google_bigquery_table" "default" { dynamic "time_partitioning" { for_each = try(each.value.partitioning.time, null) != null ? [""] : [] content { - expiration_ms = each.value.partitioning.time.expiration_ms - field = each.value.partitioning.time.field - type = each.value.partitioning.time.type - require_partition_filter = each.value.partitioning.time.require_partition_filter + expiration_ms = each.value.partitioning.time.expiration_ms + field = each.value.partitioning.time.field + type = each.value.partitioning.time.type } } } @@ -270,17 +362,18 @@ resource "google_bigquery_table" "views" { } resource "google_bigquery_table" "materialized_view" { - depends_on = [google_bigquery_table.default] - for_each = var.materialized_views - project = var.project_id - dataset_id = google_bigquery_dataset.default.dataset_id - table_id = each.key - friendly_name = each.value.friendly_name - description = each.value.description - labels = each.value.labels - clustering = each.value.options.clustering - expiration_time = each.value.options.expiration_time - deletion_protection = each.value.deletion_protection + depends_on = [google_bigquery_table.default] + for_each = var.materialized_views + project = var.project_id + dataset_id = google_bigquery_dataset.default.dataset_id + table_id = each.key + friendly_name = each.value.friendly_name + description = each.value.description + labels = each.value.labels + clustering = each.value.options.clustering + expiration_time = each.value.options.expiration_time + deletion_protection = each.value.deletion_protection + require_partition_filter = each.value.require_partition_filter dynamic "range_partitioning" { for_each = try(each.value.partitioning.range, null) != null ? [""] : [] @@ -297,10 +390,9 @@ resource "google_bigquery_table" "materialized_view" { dynamic "time_partitioning" { for_each = try(each.value.partitioning.time, null) != null ? [""] : [] content { - expiration_ms = each.value.partitioning.time.expiration_ms - field = each.value.partitioning.time.field - type = each.value.partitioning.time.type - require_partition_filter = each.value.partitioning.time.require_partition_filter + expiration_ms = each.value.partitioning.time.expiration_ms + field = each.value.partitioning.time.field + type = each.value.partitioning.time.type } } diff --git a/modules/bigquery-dataset/variables.tf b/modules/bigquery-dataset/variables.tf index 8a33c057d9..5deb2d24c7 100644 --- a/modules/bigquery-dataset/variables.tf +++ b/modules/bigquery-dataset/variables.tf @@ -123,6 +123,7 @@ variable "materialized_views" { friendly_name = optional(string) labels = optional(map(string), {}) refresh_interval_ms = optional(bool) + require_partition_filter = optional(bool) options = optional(object({ clustering = optional(list(string)) expiration_time = optional(number) @@ -135,10 +136,9 @@ variable "materialized_views" { start = number })) time = optional(object({ - type = string - expiration_ms = optional(number) - field = optional(string) - require_partition_filter = optional(bool) + type = string + expiration_ms = optional(number) + field = optional(string) })) })) })) @@ -167,15 +167,55 @@ variable "project_id" { variable "tables" { description = "Table definitions. Options and partitioning default to null. Partitioning can only use `range` or `time`, set the unused one to null." type = map(object({ - deletion_protection = optional(bool) - description = optional(string, "Terraform managed.") - friendly_name = optional(string) - labels = optional(map(string), {}) - schema = optional(string) + deletion_protection = optional(bool) + description = optional(string, "Terraform managed.") + friendly_name = optional(string) + labels = optional(map(string), {}) + require_partition_filter = optional(bool) + schema = optional(string) + external_data_configuration = optional(object({ + autodetect = bool + source_uris = list(string) + avro_logical_types = optional(bool) + compression = optional(string) + connection_id = optional(string) + file_set_spec_type = optional(string) + ignore_unknown_values = optional(bool) + metadata_cache_mode = optional(string) + object_metadata = optional(string) + json_options_encoding = optional(string) + reference_file_schema_uri = optional(string) + schema = optional(string) + source_format = optional(string) + max_bad_records = optional(number) + csv_options = optional(object({ + quote = string + allow_jagged_rows = optional(bool) + allow_quoted_newlines = optional(bool) + encoding = optional(string) + field_delimiter = optional(string) + skip_leading_rows = optional(number) + })) + google_sheets_options = optional(object({ + range = optional(string) + skip_leading_rows = optional(number) + })) + hive_partitioning_options = optional(object({ + mode = optional(string) + require_partition_filter = optional(bool) + source_uri_prefix = optional(string) + })) + parquet_options = optional(object({ + enum_as_string = optional(bool) + enable_list_inference = optional(bool) + })) + + })) options = optional(object({ clustering = optional(list(string)) encryption_key = optional(string) expiration_time = optional(number) + max_staleness = optional(string) }), {}) partitioning = optional(object({ field = optional(string) @@ -185,10 +225,24 @@ variable "tables" { start = number })) time = optional(object({ - type = string - expiration_ms = optional(number) - field = optional(string) - require_partition_filter = optional(bool) + type = string + expiration_ms = optional(number) + field = optional(string) + })) + })) + table_constraints = optional(object({ + primary_key_columns = optional(list(string)) + foreign_keys = optional(object({ + referenced_table = object({ + project_id = string + dataset_id = string + table_id = string + }) + column_references = object({ + referencing_column = string + referenced_column = string + }) + name = optional(string) })) })) }))