Merge pull request #994 from iht/add_pubsub_schema

Add schemas to Pubsub topic module.
GoogleCloudPlatform · Nov 20, 2022 · f26e864 · f26e864
2 parents f4960dd + 06750e7
commit f26e864
Show file tree

Hide file tree

Showing 4 changed files with 76 additions and 4 deletions.
diff --git a/modules/pubsub/README.md b/modules/pubsub/README.md
@@ -1,6 +1,6 @@
 # Google Cloud Pub/Sub Module
 
-This module allows managing a single Pub/Sub topic, including multiple subscriptions and IAM bindings at the topic and subscriptions levels.
+This module allows managing a single Pub/Sub topic, including multiple subscriptions and IAM bindings at the topic and subscriptions levels, as well as schemas.
 
 
 ## Examples
@@ -20,6 +20,38 @@ module "pubsub" {
 # tftest modules=1 resources=3
 ```
 
+### Topic with schema
+
+```hcl
+module "topic_with_schema" {
+  source     = "./fabric/modules/pubsub"
+  project_id = "my-project"
+  name       = "my-topic"
+  schema = {
+    msg_encoding = "JSON"
+    schema_type = "AVRO"
+    definition = jsonencode({
+      "type" = "record",
+      "name" = "Avro",
+      "fields" = [{
+        "name" = "StringField",
+        "type" = "string"
+        },
+        {
+          "name" = "FloatField",
+          "type" = "float"
+        },
+        {
+          "name" = "BooleanField",
+          "type" = "boolean"
+        },
+      ]
+    })
+  }
+}
+# tftest modules=1 resources=2
+```
+
 ### Subscriptions
 
 Subscriptions are defined with the `subscriptions` variable, allowing optional configuration of per-subscription defaults. Push subscriptions need extra configuration, shown in the following example.
@@ -104,16 +136,19 @@ module "pubsub" {
 | [message_retention_duration](variables.tf#L62) | Minimum duration to retain a message after it is published to the topic. | <code>string</code> |  | <code>null</code> |
 | [push_configs](variables.tf#L78) | Push subscription configurations. | <code title="map&#40;object&#40;&#123;&#10;  attributes &#61; map&#40;string&#41;&#10;  endpoint   &#61; string&#10;  oidc_token &#61; object&#40;&#123;&#10;    audience              &#61; string&#10;    service_account_email &#61; string&#10;  &#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 | [regions](variables.tf#L91) | List of regions used to set persistence policy. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
-| [subscription_iam](variables.tf#L97) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | <code>map&#40;map&#40;list&#40;string&#41;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
-| [subscriptions](variables.tf#L103) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | <code title="map&#40;object&#40;&#123;&#10;  labels &#61; map&#40;string&#41;&#10;  options &#61; object&#40;&#123;&#10;    ack_deadline_seconds       &#61; number&#10;    message_retention_duration &#61; string&#10;    retain_acked_messages      &#61; bool&#10;    expiration_policy_ttl      &#61; string&#10;    filter                     &#61; string&#10;  &#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [schema](variables.tf#L97) | Topic schema. If set, all messages in this topic should follow this schema. | <code title="object&#40;&#123;&#10;  definition   &#61; string&#10;  msg_encoding &#61; optional&#40;string, &#34;ENCODING_UNSPECIFIED&#34;&#41;&#10;  schema_type  &#61; string&#10;&#125;&#41;">object&#40;&#123;&#8230;&#125;&#41;</code> |  | <code>null</code> |
+| [subscription_iam](variables.tf#L107) | IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format. | <code>map&#40;map&#40;list&#40;string&#41;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
+| [subscriptions](variables.tf#L113) | Topic subscriptions. Also define push configs for push subscriptions. If options is set to null subscription defaults will be used. Labels default to topic labels if set to null. | <code title="map&#40;object&#40;&#123;&#10;  labels &#61; map&#40;string&#41;&#10;  options &#61; object&#40;&#123;&#10;    ack_deadline_seconds       &#61; number&#10;    message_retention_duration &#61; string&#10;    retain_acked_messages      &#61; bool&#10;    expiration_policy_ttl      &#61; string&#10;    filter                     &#61; string&#10;  &#125;&#41;&#10;&#125;&#41;&#41;">map&#40;object&#40;&#123;&#8230;&#125;&#41;&#41;</code> |  | <code>&#123;&#125;</code> |
 
 ## Outputs
 
 | name | description | sensitive |
 |---|---|:---:|
 | [id](outputs.tf#L17) | Topic id. |  |
+| [schema](outputs.tf#L43) | Schema resource. |  |
+| [schema_id](outputs.tf#L48) | Schema resource id. |  |
 | [subscription_id](outputs.tf#L25) | Subscription ids. |  |
 | [subscriptions](outputs.tf#L35) | Subscription resources. |  |
-| [topic](outputs.tf#L43) | Topic resource. |  |
+| [topic](outputs.tf#L53) | Topic resource. |  |
 
 <!-- END TFDOC -->
diff --git a/modules/pubsub/main.tf b/modules/pubsub/main.tf
@@ -35,6 +35,14 @@ locals {
   }
 }
 
+resource "google_pubsub_schema" "default" {
+  count      = var.schema == null ? 0 : 1
+  name       = "{$var.name}-schema"
+  type       = var.schema.schema_type
+  definition = var.schema.definition
+  project    = var.project_id
+}
+
 resource "google_pubsub_topic" "default" {
   project                    = var.project_id
   name                       = var.name
@@ -48,6 +56,14 @@ resource "google_pubsub_topic" "default" {
       allowed_persistence_regions = var.regions
     }
   }
+
+  dynamic "schema_settings" {
+    for_each = var.schema == null ? [] : [""]
+    content {
+      schema   = google_pubsub_schema.default[0].id
+      encoding = var.schema.msg_encoding
+    }
+  }
 }
 
 resource "google_pubsub_topic_iam_binding" "default" {

diff --git a/modules/pubsub/outputs.tf b/modules/pubsub/outputs.tf
@@ -40,10 +40,21 @@ output "subscriptions" {
   ]
 }
 
+output "schema" {
+  description = "Schema resource."
+  value       = try(google_pubsub_schema.default[0], null)
+}
+
+output "schema_id" {
+  description = "Schema resource id."
+  value       = try(google_pubsub_schema.default[0].id, null)
+}
+
 output "topic" {
   description = "Topic resource."
   value       = google_pubsub_topic.default
   depends_on = [
     google_pubsub_topic_iam_binding.default
   ]
 }
+
diff --git a/modules/pubsub/variables.tf b/modules/pubsub/variables.tf
@@ -94,6 +94,16 @@ variable "regions" {
   default     = []
 }
 
+variable "schema" {
+  description = "Topic schema. If set, all messages in this topic should follow this schema."
+  type = object({
+    definition   = string
+    msg_encoding = optional(string, "ENCODING_UNSPECIFIED")
+    schema_type  = string
+  })
+  default = null
+}
+
 variable "subscription_iam" {
   description = "IAM bindings for subscriptions in {SUBSCRIPTION => {ROLE => [MEMBERS]}} format."
   type        = map(map(list(string)))