feat(forwarder): allow specifying EventBridge pattern

This commit allows users to configure a more specific eventbridge
pattern to trigger the forwarder lambda. This can be useful when
attempting to subscribe only a subset of objects, and additionally
provides a mechanism to disable eventbridge outright.

examples/forwarder-s3-eventbridge-pattern/README.md

@@ -0,0 +1,68 @@
+# Forwarding a subset of S3 files via EventBridge
+
+This module demonstrates how to forward a subset of objects from an S3 bucket
+which send bucket notifications to EventBridge.
+
+To run this example you need to execute:
+
+```
+$ terraform init
+$ terraform plan
+$ terraform apply
+```
+
+After applying, you can copy files to the source bucket, and verify if they are written to the destination:
+
+```
+aws s3 cp ./main.tf s3://`terraform output -raw source_bucket`/a/foo/main.tf
+aws s3 cp ./main.tf s3://`terraform output -raw source_bucket`/a/bar/main.tf
+aws s3 cp ./main.tf s3://`terraform output -raw source_bucket`/a/baz/main.tf
+...
+aws s3 ls `terraform output -raw destination_bucket`/a/
+        PRE bar/
+        PRE foo/
+```
+
+Note that this example may create resources which can cost money. Run terraform destroy when you don't need these resources.
+
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.0 |
+| <a name="requirement_observe"></a> [observe](#requirement\_observe) | ~> 0.14 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.0 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_forwarder"></a> [forwarder](#module\_forwarder) | ../..//modules/forwarder | n/a |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_s3_bucket.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
+| [aws_s3_bucket.source](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
+| [aws_s3_bucket_notification.source](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_notification) | resource |
+
+## Inputs
+
+No inputs.
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_destination_bucket"></a> [destination\_bucket](#output\_destination\_bucket) | Destination bucket name to read objects from. |
+| <a name="output_source_bucket"></a> [source\_bucket](#output\_source\_bucket) | Source bucket name to write objects to. |
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/forwarder-s3-eventbridge-pattern/main.tf

@@ -0,0 +1,41 @@
+locals {
+  name        = basename(abspath(path.root))
+  name_prefix = "${local.name}-"
+}
+
+resource "aws_s3_bucket" "source" {
+  bucket_prefix = "${local.name_prefix}src-"
+  force_destroy = true
+}
+
+resource "aws_s3_bucket_notification" "source" {
+  bucket      = aws_s3_bucket.source.id
+  eventbridge = true
+}
+
+resource "aws_s3_bucket" "destination" {
+  bucket_prefix = "${local.name_prefix}dst-"
+  force_destroy = true
+}
+
+module "forwarder" {
+  # Prefer using the hashicorp registry:
+  # source = "observeinc/collection/aws//modules/forwarder"
+  # For validation purposes we will instead refer to a local version of the
+  # module:
+  source = "../..//modules/forwarder"
+  name   = local.name
+
+  destination = {
+    bucket = aws_s3_bucket.destination.id
+  }
+
+  source_bucket_names = [aws_s3_bucket.source.id]
+
+  source_eventbridge_pattern = {
+    "detail.object.key" = [
+      { "wildcard" = "*/foo/*" },
+      { "wildcard" = "*/bar/*" },
+    ]
+  }
+}

examples/forwarder-s3-eventbridge-pattern/outputs.tf

@@ -0,0 +1,9 @@
+output "source_bucket" {
+  description = "Source bucket name to write objects to."
+  value       = aws_s3_bucket.source.id
+}
+
+output "destination_bucket" {
+  description = "Destination bucket name to read objects from."
+  value       = aws_s3_bucket.destination.id
+}

examples/forwarder-s3-eventbridge-pattern/tests/example.tftest.hcl

@@ -0,0 +1,6 @@
+mock_provider "observe" {
+  source = "../../modules/testing/observe_mock"
+}
+
+# only verifies module can be installed and removed correctly
+run "install" {}

examples/forwarder-s3-eventbridge-pattern/versions.tf

@@ -0,0 +1,15 @@
+terraform {
+  required_version = ">= 1.3"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 5.0"
+    }
+
+    observe = {
+      source  = "terraform.observeinc.com/observeinc/observe"
+      version = "~> 0.14"
+    }
+  }
+}

modules/forwarder/README.md

@@ -117,6 +117,7 @@ module "forwarder" {
 | <a name="input_retention_in_days"></a> [retention\_in\_days](#input\_retention\_in\_days) | Retention in days of cloudwatch log group | `number` | `365` | no |
 | <a name="input_sam_release_version"></a> [sam\_release\_version](#input\_sam\_release\_version) | Release version for SAM apps as defined on github.com/observeinc/aws-sam-apps. | `string` | `""` | no |
 | <a name="input_source_bucket_names"></a> [source\_bucket\_names](#input\_source\_bucket\_names) | A list of bucket names which the forwarder is allowed to read from.  This<br>list only affects permissions, and supports wildcards. In order to have<br>files copied to Filedrop, you must also subscribe S3 Bucket Notifications<br>to the forwarder. | `list(string)` | `[]` | no |
+| <a name="input_source_eventbridge_pattern"></a> [source\_eventbridge\_pattern](#input\_source\_eventbridge\_pattern) | EventBridge event pattern to trigger forwarder. By default, any<br>`s3:ObjectCreated` in EventBridge will trigger the forwarder. You can<br>disable this behavior by setting this variable to null. Alternatively, you<br>may make it more specific by extending the pattern. | `map(any)` | `{}` | no |
 | <a name="input_source_kms_key_arns"></a> [source\_kms\_key\_arns](#input\_source\_kms\_key\_arns) | A list of KMS Key ARNs the forwarder is allowed to use to decrypt objects in S3. | `list(string)` | `[]` | no |
 | <a name="input_source_topic_arns"></a> [source\_topic\_arns](#input\_source\_topic\_arns) | A list of SNS topics the forwarder is allowed to be subscribed to. | `list(string)` | `[]` | no |
 | <a name="input_verbosity"></a> [verbosity](#input\_verbosity) | Logging verbosity for Lambda. Highest log verbosity is 9. | `number` | `1` | no |

modules/forwarder/eventbridge.tf

@@ -1,19 +1,21 @@
 resource "aws_cloudwatch_event_rule" "this" {
+  count          = var.source_eventbridge_pattern != null ? 1 : 0
   name_prefix    = local.name_prefix
   description    = "Trigger copy for object created events"
   event_bus_name = "default"
 
   event_pattern = jsonencode(
-    {
+    merge({
       source      = ["aws.s3"]
       detail-type = ["Object Created"],
-    },
+    }, var.source_eventbridge_pattern),
   )
 }
 
 resource "aws_cloudwatch_event_target" "this" {
-  rule = aws_cloudwatch_event_rule.this.name
-  arn  = aws_sqs_queue.this.arn
+  count = var.source_eventbridge_pattern != null ? 1 : 0
+  rule  = aws_cloudwatch_event_rule.this[0].name
+  arn   = aws_sqs_queue.this.arn
 
   input_transformer {
     input_paths = {

modules/forwarder/variables.tf

@@ -92,6 +92,18 @@ variable "source_kms_key_arns" {
   }
 }
 
+variable "source_eventbridge_pattern" {
+  description = <<-EOF
+    EventBridge event pattern to trigger forwarder. By default, any
+    `s3:ObjectCreated` in EventBridge will trigger the forwarder. You can
+    disable this behavior by setting this variable to null. Alternatively, you
+    may make it more specific by extending the pattern.
+  EOF
+  type        = map(any)
+  nullable    = true
+  default     = {}
+}
+
 variable "max_file_size" {
   description = <<-EOF
     Max file size for objects to process (in bytes), default is 1GB