From 54e9738c393b66a481b04d47ae5ab297a3dabb46 Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Sun, 20 Nov 2022 13:26:33 +0100 Subject: [PATCH 1/6] Add schemas to Pubsub topic module. Pubsub topics can now have schemas (https://cloud.google.com/pubsub/docs/admin#schemas). This PR adds an option to set the schema settings and create a new optional resource of type `google_pubsub_schema` attached to the `google_pubsub_topic`. --- modules/pubsub/README.md | 37 ++++++++++++++++++++++++++++++++++++- modules/pubsub/main.tf | 17 +++++++++++++++++ modules/pubsub/outputs.tf | 16 ++++++++++++++++ modules/pubsub/variables.tf | 10 ++++++++++ 4 files changed, 79 insertions(+), 1 deletion(-) diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index b75aaf6d98..d20d1cc42f 100644 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -1,6 +1,6 @@ # Google Cloud Pub/Sub Module -This module allows managing a single Pub/Sub topic, including multiple subscriptions and IAM bindings at the topic and subscriptions levels. +This module allows managing a single Pub/Sub topic, including multiple subscriptions and IAM bindings at the topic and subscriptions levels, as well as schemas. ## Examples @@ -20,6 +20,38 @@ module "pubsub" { # tftest modules=1 resources=3 ``` +### Topic with schema + +```hcl +module "topic_with_schema" { + source = "./fabric/modules/pubsub" + project_id = "my-project" + name = "my-topic" + schema = { + msg_encoding = "JSON" + schema_type = "AVRO" + definition = jsonencode({ + "type" = "record", + "name" = "Avro", + "fields" : [{ + "name" = "StringField", + "type" = "string" + }, + { + "name" = "FloatField", + "type" = "float" + }, + { + "name" = "BooleanField", + "type" = "boolean" + }, + ] + }) + } +} +# tftest modules=1 resources=3 +``` + ### Subscriptions Subscriptions are defined with the `subscriptions` variable, allowing optional configuration of per-subscription defaults. Push subscriptions need extra configuration, shown in the following example. @@ -104,6 +136,7 @@ module "pubsub" { | [message_retention_duration](variables.tf#L62) | Minimum duration to retain a message after it is published to the topic. | string | | null | | [push_configs](variables.tf#L78) | Push subscription configurations. | map(object({…})) | | {} | | [regions](variables.tf#L91) | List of regions used to set persistence policy. | list(string) | | [] | +| [schema](variables.tf#L118) | Topic schema. If set, all messages in this topic should follow this schema. | object({…}) | | null | | [subscription_iam](variables.tf#L97) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | | [subscriptions](variables.tf#L103) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…})) | | {} | @@ -112,6 +145,8 @@ module "pubsub" { | name | description | sensitive | |---|---|:---:| | [id](outputs.tf#L17) | Topic id. | | +| [schema](outputs.tf#L59) | Schema resource. | | +| [schema_id](outputs.tf#L51) | Schema resource id. | | | [subscription_id](outputs.tf#L25) | Subscription ids. | | | [subscriptions](outputs.tf#L35) | Subscription resources. | | | [topic](outputs.tf#L43) | Topic resource. | | diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf index 2645de0238..b0beb8b7d9 100644 --- a/modules/pubsub/main.tf +++ b/modules/pubsub/main.tf @@ -35,6 +35,14 @@ locals { } } +resource "google_pubsub_schema" "default" { + count = var.schema == null ? 0 : 1 + name = format("%s-%s", var.name, "schema") + type = var.schema.schema_type + definition = var.schema.definition + project = var.project_id +} + resource "google_pubsub_topic" "default" { project = var.project_id name = var.name @@ -48,6 +56,15 @@ resource "google_pubsub_topic" "default" { allowed_persistence_regions = var.regions } } + + depends_on = [google_pubsub_schema.default] + dynamic "schema_settings" { + for_each = var.schema == null ? [] : [""] + content { + schema = google_pubsub_schema.default[0].id + encoding = var.schema.msg_encoding + } + } } resource "google_pubsub_topic_iam_binding" "default" { diff --git a/modules/pubsub/outputs.tf b/modules/pubsub/outputs.tf index c26eb4d9a5..971a1f3415 100644 --- a/modules/pubsub/outputs.tf +++ b/modules/pubsub/outputs.tf @@ -47,3 +47,19 @@ output "topic" { google_pubsub_topic_iam_binding.default ] } + +output "schema_id" { + description = "Schema resource id." + value = google_pubsub_schema.default[0].id + depends_on = [ + google_pubsub_schema.default + ] +} + +output "schema" { + description = "Schema resource." + value = google_pubsub_schema.default[0] + depends_on = [ + google_pubsub_schema.default + ] +} diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf index 009a12fbe1..503ff69076 100644 --- a/modules/pubsub/variables.tf +++ b/modules/pubsub/variables.tf @@ -114,3 +114,13 @@ variable "subscriptions" { })) default = {} } + +variable "schema" { + description = "Topic schema. If set, all messages in this topic should follow this schema." + type = object({ + schema_type = string + definition = string + msg_encoding = optional(string, "ENCODING_UNSPECIFIED") + }) + default = null +} From 3f4fa7472988f986f21b4130ffdb5067f223f438 Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Sun, 20 Nov 2022 16:32:37 +0100 Subject: [PATCH 2/6] Address reviewer comments. --- modules/pubsub/README.md | 9 +++++---- modules/pubsub/main.tf | 3 +-- modules/pubsub/outputs.tf | 20 ++++++++++---------- modules/pubsub/variables.tf | 20 ++++++++++---------- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index d20d1cc42f..024f2889e2 100644 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -136,7 +136,7 @@ module "pubsub" { | [message_retention_duration](variables.tf#L62) | Minimum duration to retain a message after it is published to the topic. | string | | null | | [push_configs](variables.tf#L78) | Push subscription configurations. | map(object({…})) | | {} | | [regions](variables.tf#L91) | List of regions used to set persistence policy. | list(string) | | [] | -| [schema](variables.tf#L118) | Topic schema. If set, all messages in this topic should follow this schema. | object({…}) | | null | +| [schema](variables.tf#L118) | Topic schema. If set, all messages in this topic should follow this schema. | object({…}) | | null | | [subscription_iam](variables.tf#L97) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | | [subscriptions](variables.tf#L103) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…})) | | {} | @@ -145,10 +145,11 @@ module "pubsub" { | name | description | sensitive | |---|---|:---:| | [id](outputs.tf#L17) | Topic id. | | -| [schema](outputs.tf#L59) | Schema resource. | | -| [schema_id](outputs.tf#L51) | Schema resource id. | | +| [schema](outputs.tf#L43) | Schema resource. | | +| [schema_id](outputs.tf#L48) | Schema resource id. | | +| [schema_id](outputs.tf#L61) | Schema resource id. | | | [subscription_id](outputs.tf#L25) | Subscription ids. | | | [subscriptions](outputs.tf#L35) | Subscription resources. | | -| [topic](outputs.tf#L43) | Topic resource. | | +| [topic](outputs.tf#L53) | Topic resource. | | diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf index b0beb8b7d9..aa63e8c029 100644 --- a/modules/pubsub/main.tf +++ b/modules/pubsub/main.tf @@ -37,7 +37,7 @@ locals { resource "google_pubsub_schema" "default" { count = var.schema == null ? 0 : 1 - name = format("%s-%s", var.name, "schema") + name = "{$var.name}-schema" type = var.schema.schema_type definition = var.schema.definition project = var.project_id @@ -57,7 +57,6 @@ resource "google_pubsub_topic" "default" { } } - depends_on = [google_pubsub_schema.default] dynamic "schema_settings" { for_each = var.schema == null ? [] : [""] content { diff --git a/modules/pubsub/outputs.tf b/modules/pubsub/outputs.tf index 971a1f3415..9c5a6a6eee 100644 --- a/modules/pubsub/outputs.tf +++ b/modules/pubsub/outputs.tf @@ -40,6 +40,16 @@ output "subscriptions" { ] } +output "schema" { + description = "Schema resource." + value = try(google_pubsub_schema.default[0], null) +} + +output "schema_id" { + description = "Schema resource id." + value = try(google_pubsub_schema.default[0].id, null) +} + output "topic" { description = "Topic resource." value = google_pubsub_topic.default @@ -51,15 +61,5 @@ output "topic" { output "schema_id" { description = "Schema resource id." value = google_pubsub_schema.default[0].id - depends_on = [ - google_pubsub_schema.default - ] } -output "schema" { - description = "Schema resource." - value = google_pubsub_schema.default[0] - depends_on = [ - google_pubsub_schema.default - ] -} diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf index 503ff69076..c84168eaee 100644 --- a/modules/pubsub/variables.tf +++ b/modules/pubsub/variables.tf @@ -94,6 +94,16 @@ variable "regions" { default = [] } +variable "schema" { + description = "Topic schema. If set, all messages in this topic should follow this schema." + type = object({ + definition = string + msg_encoding = optional(string, "ENCODING_UNSPECIFIED") + schema_type = string + }) + default = null +} + variable "subscription_iam" { description = "IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format." type = map(map(list(string))) @@ -114,13 +124,3 @@ variable "subscriptions" { })) default = {} } - -variable "schema" { - description = "Topic schema. If set, all messages in this topic should follow this schema." - type = object({ - schema_type = string - definition = string - msg_encoding = optional(string, "ENCODING_UNSPECIFIED") - }) - default = null -} From 7b799bc076435184f9769857b69506a41f524937 Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Sun, 20 Nov 2022 16:34:33 +0100 Subject: [PATCH 3/6] Remove duplicated code --- modules/pubsub/outputs.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/modules/pubsub/outputs.tf b/modules/pubsub/outputs.tf index 9c5a6a6eee..3660d3679a 100644 --- a/modules/pubsub/outputs.tf +++ b/modules/pubsub/outputs.tf @@ -58,8 +58,3 @@ output "topic" { ] } -output "schema_id" { - description = "Schema resource id." - value = google_pubsub_schema.default[0].id -} - From 2a23df6bd3ae387d17187db02cc1c339566e531a Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Sun, 20 Nov 2022 16:35:08 +0100 Subject: [PATCH 4/6] Fix variables/outputs table after removing duplicated code. --- modules/pubsub/README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index 024f2889e2..1efe75aa50 100644 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -136,9 +136,9 @@ module "pubsub" { | [message_retention_duration](variables.tf#L62) | Minimum duration to retain a message after it is published to the topic. | string | | null | | [push_configs](variables.tf#L78) | Push subscription configurations. | map(object({…})) | | {} | | [regions](variables.tf#L91) | List of regions used to set persistence policy. | list(string) | | [] | -| [schema](variables.tf#L118) | Topic schema. If set, all messages in this topic should follow this schema. | object({…}) | | null | -| [subscription_iam](variables.tf#L97) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | -| [subscriptions](variables.tf#L103) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…})) | | {} | +| [schema](variables.tf#L97) | Topic schema. If set, all messages in this topic should follow this schema. | object({…}) | | null | +| [subscription_iam](variables.tf#L107) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | map(map(list(string))) | | {} | +| [subscriptions](variables.tf#L113) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | map(object({…})) | | {} | ## Outputs @@ -147,7 +147,6 @@ module "pubsub" { | [id](outputs.tf#L17) | Topic id. | | | [schema](outputs.tf#L43) | Schema resource. | | | [schema_id](outputs.tf#L48) | Schema resource id. | | -| [schema_id](outputs.tf#L61) | Schema resource id. | | | [subscription_id](outputs.tf#L25) | Subscription ids. | | | [subscriptions](outputs.tf#L35) | Subscription resources. | | | [topic](outputs.tf#L53) | Topic resource. | | From 5fa9b5e0b0709bae6f1e8b9e974fd63276249862 Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Sun, 20 Nov 2022 16:36:36 +0100 Subject: [PATCH 5/6] Fix example code for topic with schemas --- modules/pubsub/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index 1efe75aa50..46f6d4f22f 100644 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -33,7 +33,7 @@ module "topic_with_schema" { definition = jsonencode({ "type" = "record", "name" = "Avro", - "fields" : [{ + "fields" = [{ "name" = "StringField", "type" = "string" }, From 06750e723637ebd24a3321e8a27766bfbbdec884 Mon Sep 17 00:00:00 2001 From: Israel Herraiz Date: Sun, 20 Nov 2022 16:56:23 +0100 Subject: [PATCH 6/6] 1 + 1 = 2 --- modules/pubsub/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md index 46f6d4f22f..7042f37b0e 100644 --- a/modules/pubsub/README.md +++ b/modules/pubsub/README.md @@ -49,7 +49,7 @@ module "topic_with_schema" { }) } } -# tftest modules=1 resources=3 +# tftest modules=1 resources=2 ``` ### Subscriptions