fix(metricstream): lookup recommended filters (#169)

Rather than hardcoding a single filter set into the repo, look it up
by retrieving the SAM manifest. This allows us to roll back (and
forwards) between SAM app releases, and reduces the scope for
misconfiguration. The tradeoff is we are now dependent on an extra HTTP
call to S3 on every plan / apply.

Makefile

@@ -3,14 +3,3 @@
 test-dir:
 	terraform -chdir=${DIR} init -upgrade
 	terraform -chdir=${DIR} test
-
-update-binaries-logwriter:
-	APP=logwriter RESOURCE=Subscriber utilities/update-binaries.sh > modules/subscriber/uris.csv
-
-update-binaries-forwarder:
-	APP=forwarder RESOURCE=Forwarder utilities/update-binaries.sh > modules/forwarder/uris.csv
-
-update-binaries: update-binaries-logwriter update-binaries-forwarder
-
-update-filters: utilities/update-filters.sh
-

modules/metricstream/README.md

@@ -24,7 +24,7 @@ module "metric_stream" {
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.2 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.0 |
 
 ## Providers
@@ -35,7 +35,9 @@ module "metric_stream" {
 
 ## Modules
 
-No modules.
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_sam_asset"></a> [sam\_asset](#module\_sam\_asset) | ../sam_asset | n/a |
 
 ## Resources
 
@@ -67,6 +69,7 @@ No modules.
 | <a name="input_name"></a> [name](#input\_name) | Name for resources. | `string` | n/a | yes |
 | <a name="input_output_format"></a> [output\_format](#input\_output\_format) | The output format for CloudWatch Metrics. | `string` | `"json"` | no |
 | <a name="input_prefix"></a> [prefix](#input\_prefix) | Optional prefix to write log records to. | `string` | `""` | no |
+| <a name="input_sam_release_version"></a> [sam\_release\_version](#input\_sam\_release\_version) | Release version for SAM apps as defined on github.com/observeinc/aws-sam-apps. | `string` | `""` | no |
 
 ## Outputs
 

modules/metricstream/main.tf

@@ -2,8 +2,8 @@ locals {
   account_id      = data.aws_caller_identity.current.account_id
   region          = data.aws_region.current.name
   name_prefix     = "${substr(var.name, 0, 37)}-"
-  recommended     = yamldecode(file("${path.module}/filters/recommended.yaml"))
   use_recommended = var.include_filters == null && var.exclude_filters == null
+  recommended     = local.use_recommended ? yamldecode(module.sam_asset[0].body) : null
   filter = local.use_recommended ? {
     # must convert from cloudformation CamelCase to terraform snake_case when falling back to recommended filter
     include_filters = try([for v in local.recommended["IncludeFilters"] : { namespace = v.Namespace, metric_names = v.MetricNames }], [])
@@ -17,3 +17,10 @@ locals {
 data "aws_caller_identity" "current" {}
 
 data "aws_region" "current" {}
+
+module "sam_asset" {
+  count           = local.use_recommended ? 1 : 0
+  source          = "../sam_asset"
+  asset           = "cloudwatchmetrics/filters/recommended.yaml"
+  release_version = var.sam_release_version
+}

modules/metricstream/variables.tf

@@ -80,3 +80,10 @@ variable "buffering_size" {
   nullable    = false
   default     = 1
 }
+
+variable "sam_release_version" {
+  description = "Release version for SAM apps as defined on github.com/observeinc/aws-sam-apps."
+  type        = string
+  default     = ""
+  nullable    = false
+}

modules/metricstream/versions.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">= 1.2"
+  required_version = ">= 1.3"
   required_providers {
     aws = {
       source  = "hashicorp/aws"

modules/stack/README.md

@@ -172,7 +172,7 @@ You can additionally configure other submodules in this manner:
 | <a name="input_destination"></a> [destination](#input\_destination) | Destination filedrop | <pre>object({<br>    arn    = optional(string, "")<br>    bucket = optional(string, "")<br>    prefix = optional(string, "")<br>    # exclusively for backward compatible HTTP endpoint<br>    uri = optional(string, "")<br>  })</pre> | n/a | yes |
 | <a name="input_forwarder"></a> [forwarder](#input\_forwarder) | Variables for forwarder module. | <pre>object({<br>    source_bucket_names                      = optional(list(string), [])<br>    source_topic_arns                        = optional(list(string), [])<br>    content_type_overrides                   = optional(list(object({ pattern = string, content_type = string })), [])<br>    max_file_size                            = optional(number)<br>    lambda_memory_size                       = optional(number)<br>    lambda_timeout                           = optional(number)<br>    lambda_env_vars                          = optional(map(string))<br>    retention_in_days                        = optional(number)<br>    queue_max_receive_count                  = optional(number)<br>    queue_delay_seconds                      = optional(number)<br>    queue_message_retention_seconds          = optional(number)<br>    queue_batch_size                         = optional(number)<br>    queue_maximum_batching_window_in_seconds = optional(number)<br>    code_uri                                 = optional(string)<br>    sam_release_version                      = optional(string)<br>  })</pre> | `{}` | no |
 | <a name="input_logwriter"></a> [logwriter](#input\_logwriter) | Variables for AWS CloudWatch Logs collection. | <pre>object({<br>    log_group_name_patterns         = optional(list(string))<br>    log_group_name_prefixes         = optional(list(string))<br>    exclude_log_group_name_prefixes = optional(list(string))<br>    buffering_interval              = optional(number)<br>    buffering_size                  = optional(number)<br>    filter_name                     = optional(string)<br>    filter_pattern                  = optional(string)<br>    num_workers                     = optional(number)<br>    discovery_rate                  = optional(string, "24 hours")<br>    lambda_memory_size              = optional(number)<br>    lambda_timeout                  = optional(number)<br>    code_uri                        = optional(string)<br>    sam_release_version             = optional(string)<br>  })</pre> | `null` | no |
-| <a name="input_metricstream"></a> [metricstream](#input\_metricstream) | Variables for AWS CloudWatch Metrics Stream collection. | <pre>object({<br>    include_filters    = optional(list(object({ namespace = string, metric_names = optional(list(string)) })))<br>    exclude_filters    = optional(list(object({ namespace = string, metric_names = optional(list(string)) })))<br>    buffering_interval = optional(number)<br>    buffering_size     = optional(number)<br>  })</pre> | `null` | no |
+| <a name="input_metricstream"></a> [metricstream](#input\_metricstream) | Variables for AWS CloudWatch Metrics Stream collection. | <pre>object({<br>    include_filters     = optional(list(object({ namespace = string, metric_names = optional(list(string)) })))<br>    exclude_filters     = optional(list(object({ namespace = string, metric_names = optional(list(string)) })))<br>    buffering_interval  = optional(number)<br>    buffering_size      = optional(number)<br>    sam_release_version = optional(string)<br>  })</pre> | `null` | no |
 | <a name="input_name"></a> [name](#input\_name) | Name of role. Since this name must be unique within the<br>account, it will be reused for most of the resources created by this<br>module. | `string` | n/a | yes |
 | <a name="input_s3_bucket_lifecycle_expiration"></a> [s3\_bucket\_lifecycle\_expiration](#input\_s3\_bucket\_lifecycle\_expiration) | Expiration in days for S3 objects in collection bucket | `number` | `4` | no |
 | <a name="input_sam_release_version"></a> [sam\_release\_version](#input\_sam\_release\_version) | Release version for SAM apps as defined on github.com/observeinc/aws-sam-apps. | `string` | `null` | no |

modules/stack/metricstream.tf

@@ -5,8 +5,9 @@ module "metricstream" {
   name       = "${var.name}-metricstream"
   bucket_arn = aws_s3_bucket.this.arn
 
-  include_filters    = var.metricstream.include_filters
-  exclude_filters    = var.metricstream.exclude_filters
-  buffering_interval = var.metricstream.buffering_interval
-  buffering_size     = var.metricstream.buffering_size
+  include_filters     = var.metricstream.include_filters
+  exclude_filters     = var.metricstream.exclude_filters
+  buffering_interval  = var.metricstream.buffering_interval
+  buffering_size      = var.metricstream.buffering_size
+  sam_release_version = try(coalesce(var.metricstream.sam_release_version, var.sam_release_version), null)
 }

modules/stack/variables.tf

@@ -99,10 +99,11 @@ variable "metricstream" {
     Variables for AWS CloudWatch Metrics Stream collection.
   EOF
   type = object({
-    include_filters    = optional(list(object({ namespace = string, metric_names = optional(list(string)) })))
-    exclude_filters    = optional(list(object({ namespace = string, metric_names = optional(list(string)) })))
-    buffering_interval = optional(number)
-    buffering_size     = optional(number)
+    include_filters     = optional(list(object({ namespace = string, metric_names = optional(list(string)) })))
+    exclude_filters     = optional(list(object({ namespace = string, metric_names = optional(list(string)) })))
+    buffering_interval  = optional(number)
+    buffering_size      = optional(number)
+    sam_release_version = optional(string)
   })
   default = null
 }