diff --git a/README.md b/README.md index 76e5390..e9bef48 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Supports all destinations and all Kinesis Firehose Features. * [Datadog](#datadog) * [New Relic](#new-relic) * [Coralogix](#coralogix) + * [Dynatrace](#dynatrace) * [Server Side Encryption](#server-side-encryption) * [Data Transformation with Lambda](#data-transformation-with-lambda) * [Data Format Conversion](#data-format-conversion) @@ -58,6 +59,7 @@ Supports all destinations and all Kinesis Firehose Features. - DataDog - Coralogix - New Relic + - Dynatrace - Data Transformation With Lambda - Original Data Backup in S3 - Logging and Encryption @@ -270,12 +272,32 @@ module "firehose" { ```hcl module "firehose" { - source = "fdmsantos/kinesis-firehose/aws" - version = "x.x.x" - name = "firehose-delivery-stream" - destination = "coralogix" - coralogix_endpoint_type = "coralogix_ireland" - http_endpoint_access_key = "" + source = "fdmsantos/kinesis-firehose/aws" + version = "x.x.x" + name = "firehose-delivery-stream" + destination = "coralogix" + coralogix_endpoint_location = "ireland" + http_endpoint_access_key = "" +} +``` + +#### Dynatrace + +**To Enabled It:** `destination = "dynatrace"` + +**Variables Prefix:** `http_endpoint_`, `dynatrace_endpoint_location` and `dynatrace_api_url` + +**Check [HTTP Endpoint](#http-endpoint) to more details and [Destinations Mapping](#destinations-mapping) to see the difference between http_endpoint and dynatrace destinations** + +```hcl +module "firehose" { + source = "fdmsantos/kinesis-firehose/aws" + version = "x.x.x" + name = "firehose-delivery-stream" + destination = "dynatrace" + dynatrace_endpoint_location = "eu" + dynatrace_api_url = "https://xyazb123456.live.dynatrace.com" + http_endpoint_access_key = "" } ``` @@ -576,16 +598,17 @@ module "firehose" { The destination variable configured in module is mapped to firehose valid destination. -| Module Destination | Firehose Destination | Differences | -|------------------------------|----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| s3 and extended_s3 | extended_s3 | There is no difference between s3 or extended_s3 destinations | -| redshift | redshift | | -| splunk | splunk | | -| opensearch and elasticsearch | elasticsearch | There is no difference between opensearch or elasticsearch destinations | -| http_endpoint | http_endpoint | | -| datadog | http_endpoint | The difference regarding http_endpoint is the http_endpoint_url and http_endpoint_name variables aren't support, and it's necessary configure datadog_endpoint_type variable | -| newrelic | http_endpoint | The difference regarding http_endpoint is the http_endpoint_url and http_endpoint_name variables aren't support, and it's necessary configure newrelic_endpoint_type variable | -| coralogix | http_endpoint | The difference regarding http_endpoint is the http_endpoint_url and http_endpoint_name variables aren't support, and it's necessary configure coralogix_endpoint_type variable | +| Module Destination | Firehose Destination | Differences | +|------------------------------|----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| s3 and extended_s3 | extended_s3 | There is no difference between s3 or extended_s3 destinations | +| redshift | redshift | | +| splunk | splunk | | +| opensearch and elasticsearch | elasticsearch | There is no difference between opensearch or elasticsearch destinations | +| http_endpoint | http_endpoint | | +| datadog | http_endpoint | The difference regarding http_endpoint is the http_endpoint_url and http_endpoint_name variables aren't support, and it's necessary configure datadog_endpoint_type variable | +| newrelic | http_endpoint | The difference regarding http_endpoint is the http_endpoint_url and http_endpoint_name variables aren't support, and it's necessary configure newrelic_endpoint_type variable | +| coralogix | http_endpoint | The difference regarding http_endpoint is the http_endpoint_url and http_endpoint_name variables aren't support, and it's necessary configure coralogix_endpoint_location variable | +| dynatrace | http_endpoint | The difference regarding http_endpoint is the http_endpoint_url and http_endpoint_name variables aren't support, and it's necessary configure dynatrace_endpoint_location and dynatrace_api_url variable | ## Examples @@ -602,6 +625,7 @@ The destination variable configured in module is mapped to firehose valid destin - [Datadog](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/tree/main/examples/http-endpoint/datadog) - Creates a Kinesis Firehose Stream with datadog europe metrics as destination. - [New Relic](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/tree/main/examples/http-endpoint/newrelic) - Creates a Kinesis Firehose Stream with New Relic europe metrics as destination. - [Coralogix](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/tree/main/examples/http-endpoint/coralogix) - Creates a Kinesis Firehose Stream with coralogix ireland as destination. +- [Dynatrace](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/tree/main/examples/http-endpoint/dynatrace) - Creates a Kinesis Firehose Stream with dynatrace europe as destination. ## Requirements @@ -687,7 +711,7 @@ No modules. | [buffer\_interval](#input\_buffer\_interval) | Buffer incoming data for the specified period of time, in seconds, before delivering it to the destination | `number` | `300` | no | | [buffer\_size](#input\_buffer\_size) | Buffer incoming data to the specified size, in MBs, before delivering it to the destination. | `number` | `5` | no | | [configure\_existing\_application\_role](#input\_configure\_existing\_application\_role) | Set it to True if want use existing application role to add the firehose Policy | `bool` | `false` | no | -| [coralogix\_endpoint\_type](#input\_coralogix\_endpoint\_type) | Endpoint type to coralogix destination | `string` | `"coralogix_ireland"` | no | +| [coralogix\_endpoint\_location](#input\_coralogix\_endpoint\_location) | Endpoint Location to coralogix destination | `string` | `"ireland"` | no | | [coralogix\_parameter\_application\_name](#input\_coralogix\_parameter\_application\_name) | By default, your delivery stream arn will be used as applicationName | `string` | `null` | no | | [coralogix\_parameter\_subsystem\_name](#input\_coralogix\_parameter\_subsystem\_name) | By default, your delivery stream name will be used as subsystemName | `string` | `null` | no | | [coralogix\_parameter\_use\_dynamic\_values](#input\_coralogix\_parameter\_use\_dynamic\_values) | To use dynamic values for applicationName and subsystemName | `bool` | `false` | no | @@ -736,6 +760,8 @@ No modules. | [dynamic\_partition\_record\_deaggregation\_delimiter](#input\_dynamic\_partition\_record\_deaggregation\_delimiter) | Specifies the delimiter to be used for parsing through the records in the delivery stream and deaggregating them | `string` | `null` | no | | [dynamic\_partition\_record\_deaggregation\_type](#input\_dynamic\_partition\_record\_deaggregation\_type) | Data deaggregation is the process of parsing through the records in a delivery stream and separating the records based either on valid JSON or on the specified delimiter | `string` | `"JSON"` | no | | [dynamic\_partitioning\_retry\_duration](#input\_dynamic\_partitioning\_retry\_duration) | Total amount of seconds Firehose spends on retries | `number` | `300` | no | +| [dynatrace\_api\_url](#input\_dynatrace\_api\_url) | API URL to Dynatrace destination | `string` | `null` | no | +| [dynatrace\_endpoint\_location](#input\_dynatrace\_endpoint\_location) | Endpoint Location to Dynatrace destination | `string` | `"eu"` | no | | [elasticsearch\_domain\_arn](#input\_elasticsearch\_domain\_arn) | The ARN of the Amazon ES domain. The pattern needs to be arn:.* | `string` | `null` | no | | [elasticsearch\_enable\_vpc](#input\_elasticsearch\_enable\_vpc) | Indicates if destination is configured in VPC. Supported only to Elasticsearch destinations | `bool` | `false` | no | | [elasticsearch\_index\_name](#input\_elasticsearch\_index\_name) | The Elasticsearch index name | `string` | `null` | no | @@ -769,6 +795,7 @@ No modules. | [kinesis\_source\_stream\_arn](#input\_kinesis\_source\_stream\_arn) | The kinesis stream used as the source of the firehose delivery stream | `string` | `null` | no | | [kinesis\_source\_use\_existing\_role](#input\_kinesis\_source\_use\_existing\_role) | Indicates if want use the kinesis firehose role to kinesis data stream access. | `bool` | `true` | no | | [name](#input\_name) | A name to identify the stream. This is unique to the AWS account and region the Stream is created in | `string` | n/a | yes | +| [newrelic\_endpoint\_type](#input\_newrelic\_endpoint\_type) | Endpoint type to New Relic destination | `string` | `"logs_eu"` | no | | [policy\_path](#input\_policy\_path) | Path of policies to that should be added to IAM role for Kinesis Firehose Stream | `string` | `null` | no | | [redshift\_cluster\_endpoint](#input\_redshift\_cluster\_endpoint) | The redshift endpoint | `string` | `null` | no | | [redshift\_cluster\_identifier](#input\_redshift\_cluster\_identifier) | Redshift Cluster identifier. Necessary to associate the iam role to cluster | `string` | `null` | no | diff --git a/examples/http-endpoint/coralogix/main.tf b/examples/http-endpoint/coralogix/main.tf index 408ab71..d5ba395 100644 --- a/examples/http-endpoint/coralogix/main.tf +++ b/examples/http-endpoint/coralogix/main.tf @@ -17,7 +17,7 @@ module "firehose" { name = "${var.name_prefix}-delivery-stream" destination = "coralogix" buffer_interval = 60 - coralogix_endpoint_type = "coralogix_ireland" + coralogix_endpoint_location = "ireland" http_endpoint_access_key = var.coralogix_private_key http_endpoint_retry_duration = 30 http_endpoint_enable_request_configuration = true diff --git a/examples/http-endpoint/dynatrace/README.md b/examples/http-endpoint/dynatrace/README.md new file mode 100644 index 0000000..fcbda48 --- /dev/null +++ b/examples/http-endpoint/dynatrace/README.md @@ -0,0 +1,70 @@ +# Datadog + +Configuration in this directory creates kinesis firehose stream with Direct Put as source and dynatrace as destination to Europe. + +This example can be tested with Demo Data in Kinesis Firehose Console. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +It's necessary configure the following variables: + +```hcl +dynatrace_api_token = "" +dynatrace_api_url = "" +``` + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 4.4 | +| [random](#requirement\_random) | >= 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 4.4 | +| [random](#provider\_random) | >= 2.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [firehose](#module\_firehose) | ../../../ | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_kms_key.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_s3_bucket.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [random_pet.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [dynatrace\_api\_token](#input\_dynatrace\_api\_token) | Dynatrace Api Token | `string` | n/a | yes | +| [dynatrace\_api\_url](#input\_dynatrace\_api\_url) | Dynatrace Api url | `string` | n/a | yes | +| [name\_prefix](#input\_name\_prefix) | Name prefix to use in resources | `string` | `"firehose-to-dynatrace"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [firehose\_role](#output\_firehose\_role) | Firehose Role | +| [kinesis\_firehose\_arn](#output\_kinesis\_firehose\_arn) | The ARN of the Kinesis Firehose Stream | +| [kinesis\_firehose\_destination\_id](#output\_kinesis\_firehose\_destination\_id) | The Destination id of the Kinesis Firehose Stream | +| [kinesis\_firehose\_version\_id](#output\_kinesis\_firehose\_version\_id) | The Version id of the Kinesis Firehose Stream | + diff --git a/examples/http-endpoint/dynatrace/main.tf b/examples/http-endpoint/dynatrace/main.tf new file mode 100644 index 0000000..a0fd454 --- /dev/null +++ b/examples/http-endpoint/dynatrace/main.tf @@ -0,0 +1,44 @@ +resource "random_pet" "this" { + length = 2 +} + +resource "aws_s3_bucket" "s3" { + bucket = "${var.name_prefix}-destination-bucket-${random_pet.this.id}" + force_destroy = true +} + +resource "aws_kms_key" "this" { + description = "${var.name_prefix}-kms-key" + deletion_window_in_days = 7 +} + +module "firehose" { + source = "../../../" + name = "${var.name_prefix}-delivery-stream" + destination = "dynatrace" + buffer_interval = 60 + dynatrace_endpoint_location = "eu" + dynatrace_api_url = var.dynatrace_api_url + http_endpoint_access_key = var.dynatrace_api_token + http_endpoint_retry_duration = 60 + http_endpoint_enable_request_configuration = true + http_endpoint_request_configuration_content_encoding = "GZIP" + http_endpoint_request_configuration_common_attributes = [ + { + name = "testname" + value = "testvalue" + }, + { + name = "testname2" + value = "testvalue2" + } + ] + s3_backup_mode = "All" + s3_backup_prefix = "backup/" + s3_backup_bucket_arn = aws_s3_bucket.s3.arn + s3_backup_buffer_interval = 100 + s3_backup_buffer_size = 100 + s3_backup_compression = "GZIP" + s3_backup_enable_encryption = true + s3_backup_kms_key_arn = aws_kms_key.this.arn +} diff --git a/examples/http-endpoint/dynatrace/outputs.tf b/examples/http-endpoint/dynatrace/outputs.tf new file mode 100644 index 0000000..cd497c9 --- /dev/null +++ b/examples/http-endpoint/dynatrace/outputs.tf @@ -0,0 +1,19 @@ +output "firehose_role" { + description = "Firehose Role" + value = module.firehose.kinesis_firehose_role_arn +} + +output "kinesis_firehose_arn" { + description = "The ARN of the Kinesis Firehose Stream" + value = module.firehose.kinesis_firehose_arn +} + +output "kinesis_firehose_destination_id" { + description = "The Destination id of the Kinesis Firehose Stream" + value = module.firehose.kinesis_firehose_destination_id +} + +output "kinesis_firehose_version_id" { + description = "The Version id of the Kinesis Firehose Stream" + value = module.firehose.kinesis_firehose_version_id +} diff --git a/examples/http-endpoint/dynatrace/variables.tf b/examples/http-endpoint/dynatrace/variables.tf new file mode 100644 index 0000000..e725cd0 --- /dev/null +++ b/examples/http-endpoint/dynatrace/variables.tf @@ -0,0 +1,16 @@ +variable "name_prefix" { + description = "Name prefix to use in resources" + type = string + default = "firehose-to-dynatrace" +} + +variable "dynatrace_api_token" { + description = "Dynatrace Api Token" + type = string + sensitive = true +} + +variable "dynatrace_api_url" { + description = "Dynatrace Api url" + type = string +} diff --git a/examples/http-endpoint/dynatrace/versions.tf b/examples/http-endpoint/dynatrace/versions.tf new file mode 100644 index 0000000..5c3fca6 --- /dev/null +++ b/examples/http-endpoint/dynatrace/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.4" + } + random = { + source = "hashicorp/random" + version = ">= 2.0" + } + } +} diff --git a/examples/http-endpoint/newrelic/README.md b/examples/http-endpoint/newrelic/README.md index e74314d..11d28cd 100644 --- a/examples/http-endpoint/newrelic/README.md +++ b/examples/http-endpoint/newrelic/README.md @@ -21,4 +21,48 @@ newrelic_api_key = "" ``` +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | >= 4.4 | +| [random](#requirement\_random) | >= 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | >= 4.4 | +| [random](#provider\_random) | >= 2.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [firehose](#module\_firehose) | ../../../ | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_kms_key.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_s3_bucket.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [random_pet.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [name\_prefix](#input\_name\_prefix) | Name prefix to use in resources | `string` | `"firehose-to-newrelic"` | no | +| [newrelic\_api\_key](#input\_newrelic\_api\_key) | New Relic Api Key | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [firehose\_role](#output\_firehose\_role) | Firehose Role | +| [kinesis\_firehose\_arn](#output\_kinesis\_firehose\_arn) | The ARN of the Kinesis Firehose Stream | +| [kinesis\_firehose\_destination\_id](#output\_kinesis\_firehose\_destination\_id) | The Destination id of the Kinesis Firehose Stream | +| [kinesis\_firehose\_version\_id](#output\_kinesis\_firehose\_version\_id) | The Version id of the Kinesis Firehose Stream | diff --git a/examples/http-endpoint/newrelic/main.tf b/examples/http-endpoint/newrelic/main.tf index 62e63ba..0dccd74 100644 --- a/examples/http-endpoint/newrelic/main.tf +++ b/examples/http-endpoint/newrelic/main.tf @@ -17,7 +17,7 @@ module "firehose" { name = "${var.name_prefix}-delivery-stream" destination = "newrelic" buffer_interval = 60 - newrelic_endpoint_type = "metrics_eu" + newrelic_endpoint_type = "metrics_eu" http_endpoint_access_key = var.newrelic_api_key http_endpoint_retry_duration = 60 http_endpoint_enable_request_configuration = true diff --git a/locals.tf b/locals.tf index 27b476a..daedb5f 100644 --- a/locals.tf +++ b/locals.tf @@ -13,7 +13,8 @@ locals { http_endpoint : "http_endpoint", datadog : "http_endpoint", coralogix : "http_endpoint", - newrelic: "http_endpoint" + newrelic : "http_endpoint", + dynatrace : "http_endpoint" } destination = local.destinations[var.destination] s3_destination = local.destination == "extended_s3" ? true : false @@ -165,19 +166,22 @@ locals { http_endpoint_url = { http_endpoint : var.http_endpoint_url datadog : local.datadog_endpoint_url[var.datadog_endpoint_type] - coralogix : local.coralogix_endpoint_url[var.coralogix_endpoint_type] - newrelic: local.newrelic_endpoint_url[var.newrelic_endpoint_type] + coralogix : local.coralogix_endpoint_url[var.coralogix_endpoint_location] + newrelic : local.newrelic_endpoint_url[var.newrelic_endpoint_type] + dynatrace : local.dynatrace_endpoint_url[var.dynatrace_endpoint_location] } http_endpoint_name = { http_endpoint : var.http_endpoint_name datadog : "Datadog" coralogix : "Coralogix" - newrelic: "New Relic" + newrelic : "New Relic" + dynatrace : "Dynatrace" } http_endpoint_destinations_parameters = { coralogix : local.coralogix_parameters + dynatrace : local.dynatrace_parameters } # DataDog @@ -189,13 +193,35 @@ locals { metrics_eu : "https://awsmetrics-intake.datadoghq.eu/v1/input" } + # New Relic + newrelic_endpoint_url = { + logs_us : "https://aws-api.newrelic.om/firehose/v1" + logs_eu : "https://aws-api.eu.newrelic.com/firehose/v1" + metrics_us : "https://aws-api.newrelic.com/cloudwatch-metrics/v1" + metrics_eu : "https://aws-api.eu01.nr-data.net/cloudwatch-metrics/v1" + } + + # Dynatrace + dynatrace_endpoint_url = { + us : "https://us.aws.cloud.dynatrace.com" + eu : "https://eu.aws.cloud.dynatrace.com" + global : "https://aws.cloud.dynatrace.com" + } + + dynatrace_parameters = concat( + var.destination == "dynatrace" ? [{ + name = "dt-url" + value = var.dynatrace_api_url + }] : [], + ) + # Coralogix coralogix_endpoint_url = { - coralogix_us : "https://firehose-ingress.coralogix.us/firehose" - coralogix_singapore : "https://firehose-ingress.coralogixsg.com/firehose" - coralogix_ireland : "https://firehose-ingress.coralogix.com/firehose" - coralogix_india : "https://firehose-ingress.coralogix.in/firehose" - coralogix_stockholm : "https://firehose-ingress.eu2.coralogix.com/firehose" + us : "https://firehose-ingress.coralogix.us/firehose" + singapore : "https://firehose-ingress.coralogixsg.com/firehose" + ireland : "https://firehose-ingress.coralogix.com/firehose" + india : "https://firehose-ingress.coralogix.in/firehose" + stockholm : "https://firehose-ingress.eu2.coralogix.com/firehose" } coralogix_parameters = concat( @@ -213,14 +239,6 @@ locals { }] : [] ) - # New Relic - newrelic_endpoint_url = { - logs_us : "https://aws-api.newrelic.om/firehose/v1" - logs_eu : "https://aws-api.eu.newrelic.com/firehose/v1" - metrics_us : "https://aws-api.newrelic.com/cloudwatch-metrics/v1" - metrics_eu : "https://aws-api.eu01.nr-data.net/cloudwatch-metrics/v1" - } - # Networking firehose_cidr_blocks = { redshift : { diff --git a/variables.tf b/variables.tf index c901238..f86519c 100644 --- a/variables.tf +++ b/variables.tf @@ -14,7 +14,7 @@ variable "destination" { type = string validation { error_message = "Please use a valid destination!" - condition = contains(["s3", "extended_s3", "redshift", "opensearch", "elasticsearch", "splunk", "http_endpoint", "datadog", "coralogix", "newrelic"], var.destination) + condition = contains(["s3", "extended_s3", "redshift", "opensearch", "elasticsearch", "splunk", "http_endpoint", "datadog", "coralogix", "newrelic", "dynatrace"], var.destination) } } @@ -947,17 +947,35 @@ variable "newrelic_endpoint_type" { } } +###### +# Dynatrace Destination Variables +###### +variable "dynatrace_endpoint_location" { + description = "Endpoint Location to Dynatrace destination" + type = string + default = "eu" + validation { + error_message = "Please use a valid endpoint type!" + condition = contains(["us", "eu", "global"], var.dynatrace_endpoint_location) + } +} + +variable "dynatrace_api_url" { + description = "API URL to Dynatrace destination" + type = string + default = null +} ###### # Coralogix Destination Variables ###### -variable "coralogix_endpoint_type" { - description = "Endpoint type to coralogix destination" +variable "coralogix_endpoint_location" { + description = "Endpoint Location to coralogix destination" type = string - default = "coralogix_ireland" + default = "ireland" validation { error_message = "Please use a valid endpoint type!" - condition = contains(["coralogix_us", "coralogix_singapore", "coralogix_ireland", "coralogix_india", "coralogix_stockholm"], var.coralogix_endpoint_type) + condition = contains(["us", "singapore", "ireland", "india", "stockholm"], var.coralogix_endpoint_location) } }