feat: Add MongoDB Cloud destination

fdmsantos · Oct 18, 2022 · 39ab429 · 39ab429
1 parent 60e1bb6
commit 39ab429
Show file tree

Hide file tree

Showing 8 changed files with 190 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -23,6 +23,7 @@ Supports all destinations and all Kinesis Firehose Features.
     * [Dynatrace](#dynatrace)
     * [Honeycomb](#honeycomb)
     * [Logic Monitor](#logic-monitor)
+    * [MongoDB](#mongodb)
   * [Server Side Encryption](#server-side-encryption)
   * [Data Transformation with Lambda](#data-transformation-with-lambda)
   * [Data Format Conversion](#data-format-conversion)
@@ -64,6 +65,7 @@ Supports all destinations and all Kinesis Firehose Features.
   - Dynatrace
   - Honeycomb
   - Logic Monitor
+  - MongoDB Cloud
 - Data Transformation With Lambda
 - Original Data Backup in S3
 - Logging and Encryption
@@ -344,6 +346,25 @@ module "firehose" {
 }
 ```
 
+#### MongoDB
+
+**To Enabled It:** `destination = "mongodb"`
+
+**Variables Prefix:** `http_endpoint_` and `mongodb_realm_webhook_url`
+
+**Check [HTTP Endpoint](#http-endpoint) to more details and [Destinations Mapping](#destinations-mapping) to see the difference between http_endpoint and mongodb destinations**
+
+```hcl
+module "firehose" {
+  source                    = "fdmsantos/kinesis-firehose/aws"
+  version                   = "x.x.x"
+  name                      = "firehose-delivery-stream"
+  destination               = "mongodb"
+  mongodb_realm_webhook_url = "<mongodb_realm_webhook_url>"
+  http_endpoint_access_key  = "<mongodb_api_key>"
+}
+```
+
 ### Server Side Encryption
 
 **Supported By:** Only Direct Put source
@@ -654,6 +675,7 @@ The destination variable configured in module is mapped to firehose valid destin
 | dynatrace                    | http_endpoint        | The difference regarding http_endpoint is the http_endpoint_url and http_endpoint_name variables aren't support, and it's necessary configure dynatrace_endpoint_location and dynatrace_api_url variable |
 | honeycomb                    | http_endpoint        | The difference regarding http_endpoint is the http_endpoint_url and http_endpoint_name variables aren't support, and it's necessary configure honeycomb_dataset_name variable                            |
 | logicmonitor                 | http_endpoint        | The difference regarding http_endpoint is the http_endpoint_url and http_endpoint_name variables aren't support, and it's necessary configure logicmonitor_account variable                              |
+| mongodb                      | http_endpoint        | The difference regarding http_endpoint is the http_endpoint_url and http_endpoint_name variables aren't support, and it's necessary configure mongodb_realm_webhook_url variable                         |
 
 ## Examples
 
@@ -673,6 +695,7 @@ The destination variable configured in module is mapped to firehose valid destin
 - [Dynatrace](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/tree/main/examples/http-endpoint/dynatrace) - Creates a Kinesis Firehose Stream with dynatrace europe as destination.
 - [Honeycomb](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/tree/main/examples/http-endpoint/honeycomb) - Creates a Kinesis Firehose Stream with honeycomb as destination.
 - [LogicMonitor](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/tree/main/examples/http-endpoint/logicmonitor) - Creates a Kinesis Firehose Stream with Logic Monitor as destination.
+- [MongoDB](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/tree/main/examples/http-endpoint/mongodb) - Creates a Kinesis Firehose Stream with MongoDB as destination.
 
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
@@ -845,6 +868,7 @@ No modules.
 | <a name="input_kinesis_source_stream_arn"></a> [kinesis\_source\_stream\_arn](#input\_kinesis\_source\_stream\_arn) | The kinesis stream used as the source of the firehose delivery stream | `string` | `null` | no |
 | <a name="input_kinesis_source_use_existing_role"></a> [kinesis\_source\_use\_existing\_role](#input\_kinesis\_source\_use\_existing\_role) | Indicates if want use the kinesis firehose role to kinesis data stream access. | `bool` | `true` | no |
 | <a name="input_logicmonitor_account"></a> [logicmonitor\_account](#input\_logicmonitor\_account) | Account to use in Logic Monitor destination | `string` | `""` | no |
+| <a name="input_mongodb_realm_webhook_url"></a> [mongodb\_realm\_webhook\_url](#input\_mongodb\_realm\_webhook\_url) | Account to use in Logic Monitor destination | `string` | `""` | no |
 | <a name="input_name"></a> [name](#input\_name) | A name to identify the stream. This is unique to the AWS account and region the Stream is created in | `string` | n/a | yes |
 | <a name="input_newrelic_endpoint_type"></a> [newrelic\_endpoint\_type](#input\_newrelic\_endpoint\_type) | Endpoint type to New Relic destination | `string` | `"logs_eu"` | no |
 | <a name="input_policy_path"></a> [policy\_path](#input\_policy\_path) | Path of policies to that should be added to IAM role for Kinesis Firehose Stream | `string` | `null` | no |

diff --git a/examples/http-endpoint/mongodb/README.md b/examples/http-endpoint/mongodb/README.md
@@ -0,0 +1,70 @@
+# MongoDB
+
+Configuration in this directory creates kinesis firehose stream with Direct Put as source and mongo db cloud as destination.
+
+This example can be tested with Demo Data in Kinesis Firehose Console.
+
+## Usage
+
+To run this example you need to execute:
+
+```bash
+$ terraform init
+$ terraform plan
+$ terraform apply
+```
+
+It's necessary configure the following variables:
+
+```hcl
+mongodb_realm_webhook_url = "<mongodb_realm_webhook_url>"
+mongodb_api_key           = "<mongodb_api_key>"
+```
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.13.1 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 4.4 |
+| <a name="requirement_random"></a> [random](#requirement\_random) | >= 2.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.4 |
+| <a name="provider_random"></a> [random](#provider\_random) | >= 2.0 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_firehose"></a> [firehose](#module\_firehose) | ../../../ | n/a |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_kms_key.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
+| [aws_s3_bucket.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
+| [random_pet.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_mongodb_api_key"></a> [mongodb\_api\_key](#input\_mongodb\_api\_key) | MongoDB Api Key | `string` | n/a | yes |
+| <a name="input_mongodb_realm_webhook_url"></a> [mongodb\_realm\_webhook\_url](#input\_mongodb\_realm\_webhook\_url) | MongoDB Realm Webhook URL | `string` | n/a | yes |
+| <a name="input_name_prefix"></a> [name\_prefix](#input\_name\_prefix) | Name prefix to use in resources | `string` | `"firehose-to-mongodb"` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_firehose_role"></a> [firehose\_role](#output\_firehose\_role) | Firehose Role |
+| <a name="output_kinesis_firehose_arn"></a> [kinesis\_firehose\_arn](#output\_kinesis\_firehose\_arn) | The ARN of the Kinesis Firehose Stream |
+| <a name="output_kinesis_firehose_destination_id"></a> [kinesis\_firehose\_destination\_id](#output\_kinesis\_firehose\_destination\_id) | The Destination id of the Kinesis Firehose Stream |
+| <a name="output_kinesis_firehose_version_id"></a> [kinesis\_firehose\_version\_id](#output\_kinesis\_firehose\_version\_id) | The Version id of the Kinesis Firehose Stream |
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/examples/http-endpoint/mongodb/main.tf b/examples/http-endpoint/mongodb/main.tf
@@ -0,0 +1,33 @@
+resource "random_pet" "this" {
+  length = 2
+}
+
+resource "aws_s3_bucket" "s3" {
+  bucket        = "${var.name_prefix}-destination-bucket-${random_pet.this.id}"
+  force_destroy = true
+}
+
+resource "aws_kms_key" "this" {
+  description             = "${var.name_prefix}-kms-key"
+  deletion_window_in_days = 7
+}
+
+module "firehose" {
+  source                                               = "../../../"
+  name                                                 = "${var.name_prefix}-delivery-stream"
+  destination                                          = "mongodb"
+  buffer_interval                                      = 60
+  mongodb_realm_webhook_url                            = var.mongodb_realm_webhook_url
+  http_endpoint_access_key                             = var.mongodb_api_key
+  http_endpoint_retry_duration                         = 60
+  http_endpoint_enable_request_configuration           = true
+  http_endpoint_request_configuration_content_encoding = "GZIP"
+  s3_backup_mode                                       = "All"
+  s3_backup_prefix                                     = "backup/"
+  s3_backup_bucket_arn                                 = aws_s3_bucket.s3.arn
+  s3_backup_buffer_interval                            = 100
+  s3_backup_buffer_size                                = 100
+  s3_backup_compression                                = "GZIP"
+  s3_backup_enable_encryption                          = true
+  s3_backup_kms_key_arn                                = aws_kms_key.this.arn
+}
diff --git a/examples/http-endpoint/mongodb/outputs.tf b/examples/http-endpoint/mongodb/outputs.tf
@@ -0,0 +1,19 @@
+output "firehose_role" {
+  description = "Firehose Role"
+  value       = module.firehose.kinesis_firehose_role_arn
+}
+
+output "kinesis_firehose_arn" {
+  description = "The ARN of the Kinesis Firehose Stream"
+  value       = module.firehose.kinesis_firehose_arn
+}
+
+output "kinesis_firehose_destination_id" {
+  description = "The Destination id of the Kinesis Firehose Stream"
+  value       = module.firehose.kinesis_firehose_destination_id
+}
+
+output "kinesis_firehose_version_id" {
+  description = "The Version id of the Kinesis Firehose Stream"
+  value       = module.firehose.kinesis_firehose_version_id
+}
diff --git a/examples/http-endpoint/mongodb/variables.tf b/examples/http-endpoint/mongodb/variables.tf
@@ -0,0 +1,16 @@
+variable "name_prefix" {
+  description = "Name prefix to use in resources"
+  type        = string
+  default     = "firehose-to-mongodb"
+}
+
+variable "mongodb_realm_webhook_url" {
+  description = "MongoDB Realm Webhook URL"
+  type        = string
+}
+
+variable "mongodb_api_key" {
+  description = "MongoDB Api Key"
+  type        = string
+  sensitive   = true
+}
diff --git a/examples/http-endpoint/mongodb/versions.tf b/examples/http-endpoint/mongodb/versions.tf
@@ -0,0 +1,14 @@
+terraform {
+  required_version = ">= 0.13.1"
+
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 4.4"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = ">= 2.0"
+    }
+  }
+}
diff --git a/locals.tf b/locals.tf
@@ -16,7 +16,8 @@ locals {
     newrelic : "http_endpoint",
     dynatrace : "http_endpoint",
     honeycomb : "http_endpoint",
-    logicmonitor : "http_endpoint"
+    logicmonitor : "http_endpoint",
+    mongodb : "http_endpoint"
   }
   destination    = local.destinations[var.destination]
   s3_destination = local.destination == "extended_s3" ? true : false
@@ -173,6 +174,7 @@ locals {
     dynatrace : local.dynatrace_endpoint_url[var.dynatrace_endpoint_location]
     honeycomb : "${var.honeycomb_api_host}/1/kinesis_events/${var.honeycomb_dataset_name}"
     logicmonitor : "https://${var.logicmonitor_account}.logicmonitor.com"
+    mongodb : var.mongodb_realm_webhook_url
   }
 
   http_endpoint_name = {
@@ -183,6 +185,7 @@ locals {
     dynatrace : "Dynatrace"
     honeycomb : "Honeycomb"
     logicmonitor : "LogicMonitor"
+    mongodb : "MongoDB Cloud"
   }
 
   http_endpoint_destinations_parameters = {

diff --git a/variables.tf b/variables.tf
@@ -14,7 +14,7 @@ variable "destination" {
   type        = string
   validation {
     error_message = "Please use a valid destination!"
-    condition     = contains(["s3", "extended_s3", "redshift", "opensearch", "elasticsearch", "splunk", "http_endpoint", "datadog", "coralogix", "newrelic", "dynatrace", "honeycomb", "logicmonitor"], var.destination)
+    condition     = contains(["s3", "extended_s3", "redshift", "opensearch", "elasticsearch", "splunk", "http_endpoint", "datadog", "coralogix", "newrelic", "dynatrace", "honeycomb", "logicmonitor", "mongodb"], var.destination)
   }
 }
 
@@ -990,6 +990,15 @@ variable "logicmonitor_account" {
   default     = ""
 }
 
+######
+# MongoDB Destination Variables
+######
+variable "mongodb_realm_webhook_url" {
+  type        = string
+  description = "Account to use in Logic Monitor destination"
+  default     = ""
+}
+
 ######
 # Coralogix Destination Variables
 ######