Add ability to send in Redshift password via SSM parameter (closes #14)

snowplow-devops · Oct 11, 2024 · da828c2 · da828c2
1 parent 843f6de
commit da828c2
Show file tree

Hide file tree

Showing 5 changed files with 54 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -120,6 +120,15 @@ module "rs_loader" {
 }
 ```
 
+## Password options
+
+The push the Redshift password into the environment you can either:
+
+1. Use the `redshift_password` variable and supply the password in plaintext or;
+2. Upload the password into an AWS SSM parameter and then include the name of that parameter in the `redshift_password_ssm_param` variable
+
+_Note_: The SSM parameter, if set, will override the use of the plaintext input.
+
 ## Requirements
 
 | Name | Version |
@@ -173,7 +182,6 @@ module "rs_loader" {
 | <a name="input_redshift_database"></a> [redshift\_database](#input\_redshift\_database) | Redshift database name | `string` | n/a | yes |
 | <a name="input_redshift_host"></a> [redshift\_host](#input\_redshift\_host) | Redshift cluster hostname | `string` | n/a | yes |
 | <a name="input_redshift_loader_user"></a> [redshift\_loader\_user](#input\_redshift\_loader\_user) | Name of the user that will be used for loading data | `string` | n/a | yes |
-| <a name="input_redshift_password"></a> [redshift\_password](#input\_redshift\_password) | Password for redshift\_loader\_user used by loader to perform loading | `string` | n/a | yes |
 | <a name="input_redshift_schema"></a> [redshift\_schema](#input\_redshift\_schema) | Redshift schema name | `string` | n/a | yes |
 | <a name="input_sqs_queue_name"></a> [sqs\_queue\_name](#input\_sqs\_queue\_name) | SQS queue name | `string` | n/a | yes |
 | <a name="input_ssh_key_name"></a> [ssh\_key\_name](#input\_ssh\_key\_name) | The name of the SSH key-pair to attach to all EC2 nodes deployed | `string` | n/a | yes |
@@ -186,8 +194,8 @@ module "rs_loader" {
 | <a name="input_cloudwatch_logs_enabled"></a> [cloudwatch\_logs\_enabled](#input\_cloudwatch\_logs\_enabled) | Whether application logs should be reported to CloudWatch | `bool` | `true` | no |
 | <a name="input_cloudwatch_logs_retention_days"></a> [cloudwatch\_logs\_retention\_days](#input\_cloudwatch\_logs\_retention\_days) | The length of time in days to retain logs for | `number` | `7` | no |
 | <a name="input_config_override_b64"></a> [config\_override\_b64](#input\_config\_override\_b64) | App config uploaded as a base64 encoded blob. This variable facilitates dev flow, if config is incorrect this can break the deployment. | `string` | `""` | no |
-| <a name="input_custom_iglu_resolvers"></a> [custom\_iglu\_resolvers](#input\_custom\_iglu\_resolvers) | The custom Iglu Resolvers that will be used by Stream Shredder | <pre>list(object({<br>    name            = string<br>    priority        = number<br>    uri             = string<br>    api_key         = string<br>    vendor_prefixes = list(string)<br>  }))</pre> | `[]` | no |
-| <a name="input_default_iglu_resolvers"></a> [default\_iglu\_resolvers](#input\_default\_iglu\_resolvers) | The default Iglu Resolvers that will be used by Stream Shredder | <pre>list(object({<br>    name            = string<br>    priority        = number<br>    uri             = string<br>    api_key         = string<br>    vendor_prefixes = list(string)<br>  }))</pre> | <pre>[<br>  {<br>    "api_key": "",<br>    "name": "Iglu Central",<br>    "priority": 10,<br>    "uri": "http://iglucentral.com",<br>    "vendor_prefixes": []<br>  },<br>  {<br>    "api_key": "",<br>    "name": "Iglu Central - Mirror 01",<br>    "priority": 20,<br>    "uri": "http://mirror01.iglucentral.com",<br>    "vendor_prefixes": []<br>  }<br>]</pre> | no |
+| <a name="input_custom_iglu_resolvers"></a> [custom\_iglu\_resolvers](#input\_custom\_iglu\_resolvers) | The custom Iglu Resolvers that will be used by Stream Shredder | <pre>list(object({<br/>    name            = string<br/>    priority        = number<br/>    uri             = string<br/>    api_key         = string<br/>    vendor_prefixes = list(string)<br/>  }))</pre> | `[]` | no |
+| <a name="input_default_iglu_resolvers"></a> [default\_iglu\_resolvers](#input\_default\_iglu\_resolvers) | The default Iglu Resolvers that will be used by Stream Shredder | <pre>list(object({<br/>    name            = string<br/>    priority        = number<br/>    uri             = string<br/>    api_key         = string<br/>    vendor_prefixes = list(string)<br/>  }))</pre> | <pre>[<br/>  {<br/>    "api_key": "",<br/>    "name": "Iglu Central",<br/>    "priority": 10,<br/>    "uri": "http://iglucentral.com",<br/>    "vendor_prefixes": []<br/>  },<br/>  {<br/>    "api_key": "",<br/>    "name": "Iglu Central - Mirror 01",<br/>    "priority": 20,<br/>    "uri": "http://mirror01.iglucentral.com",<br/>    "vendor_prefixes": []<br/>  }<br/>]</pre> | no |
 | <a name="input_folder_monitoring_enabled"></a> [folder\_monitoring\_enabled](#input\_folder\_monitoring\_enabled) | Whether folder monitoring should be activated or not | `bool` | `false` | no |
 | <a name="input_folder_monitoring_period"></a> [folder\_monitoring\_period](#input\_folder\_monitoring\_period) | How often to folder should be checked by folder monitoring | `string` | `"8 hours"` | no |
 | <a name="input_folder_monitoring_since"></a> [folder\_monitoring\_since](#input\_folder\_monitoring\_since) | Specifies since when folder monitoring will check | `string` | `"14 days"` | no |
@@ -203,6 +211,8 @@ module "rs_loader" {
 | <a name="input_redshift_aws_s3_folder_monitoring_transformer_output_stage_url"></a> [redshift\_aws\_s3\_folder\_monitoring\_transformer\_output\_stage\_url](#input\_redshift\_aws\_s3\_folder\_monitoring\_transformer\_output\_stage\_url) | AWS bucket URL of transformer output stage - must be within 'redshift\_aws\_s3\_bucket\_name' (NOTE: must be set if 'folder\_monitoring\_enabled' is true) | `string` | `""` | no |
 | <a name="input_redshift_jsonpaths_bucket"></a> [redshift\_jsonpaths\_bucket](#input\_redshift\_jsonpaths\_bucket) | S3 path that holds JSONPaths | `string` | `""` | no |
 | <a name="input_redshift_max_error"></a> [redshift\_max\_error](#input\_redshift\_max\_error) | Redshift max error setting which controls amount of acceptable loading errors | `number` | `1` | no |
+| <a name="input_redshift_password"></a> [redshift\_password](#input\_redshift\_password) | Password for redshift\_loader\_user used by loader to perform loading | `string` | `""` | no |
+| <a name="input_redshift_password_ssm_param"></a> [redshift\_password\_ssm\_param](#input\_redshift\_password\_ssm\_param) | SSM parameter containing the password for redshift\_loader\_user used by loader to perform loading | `string` | `""` | no |
 | <a name="input_redshift_port"></a> [redshift\_port](#input\_redshift\_port) | Redshift port | `number` | `5439` | no |
 | <a name="input_retry_period"></a> [retry\_period](#input\_retry\_period) | How often batch of failed folders should be pulled into a discovery queue | `string` | `"10 min"` | no |
 | <a name="input_retry_queue_enabled"></a> [retry\_queue\_enabled](#input\_retry\_queue\_enabled) | Whether retry queue should be enabled or not | `bool` | `false` | no |
@@ -214,7 +224,7 @@ module "rs_loader" {
 | <a name="input_sp_tracking_app_id"></a> [sp\_tracking\_app\_id](#input\_sp\_tracking\_app\_id) | App id for Snowplow tracking | `string` | `""` | no |
 | <a name="input_sp_tracking_collector_url"></a> [sp\_tracking\_collector\_url](#input\_sp\_tracking\_collector\_url) | Collector URL for Snowplow tracking | `string` | `""` | no |
 | <a name="input_sp_tracking_enabled"></a> [sp\_tracking\_enabled](#input\_sp\_tracking\_enabled) | Whether Snowplow tracking should be activated or not | `bool` | `false` | no |
-| <a name="input_ssh_ip_allowlist"></a> [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | <pre>[<br>  "0.0.0.0/0"<br>]</pre> | no |
+| <a name="input_ssh_ip_allowlist"></a> [ssh\_ip\_allowlist](#input\_ssh\_ip\_allowlist) | The list of CIDR ranges to allow SSH traffic from | `list(any)` | <pre>[<br/>  "0.0.0.0/0"<br/>]</pre> | no |
 | <a name="input_statsd_enabled"></a> [statsd\_enabled](#input\_statsd\_enabled) | Whether Statsd should be enabled or not | `bool` | `false` | no |
 | <a name="input_statsd_host"></a> [statsd\_host](#input\_statsd\_host) | Hostname of StatsD server | `string` | `""` | no |
 | <a name="input_statsd_port"></a> [statsd\_port](#input\_statsd\_port) | Port of StatsD server | `number` | `8125` | no |

diff --git a/main.tf b/main.tf
@@ -41,6 +41,19 @@ locals {
     ]
   }]
   private_ecr_registry_statement_final = local.is_private_ecr_registry ? local.private_ecr_registry_statement : []
+
+  is_redshift_password_ssm_param = var.redshift_password_ssm_param != ""
+  ssm_param_statement = [{
+    Action = [
+      "ssm:GetParameter",
+      "ssm:GetParameters"
+    ]
+    Effect = "Allow"
+    Resource = [
+      "arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter${var.redshift_password_ssm_param}"
+    ]
+  }]
+  ssm_param_statement_final = local.is_redshift_password_ssm_param ? local.ssm_param_statement : []
 }
 
 module "telemetry" {
@@ -100,6 +113,7 @@ resource "aws_iam_policy" "iam_policy" {
     Version = "2012-10-17",
     Statement = concat(
       local.private_ecr_registry_statement_final,
+      local.ssm_param_statement_final,
       [
         {
           Effect = "Allow",
@@ -346,7 +360,6 @@ locals {
     rs_port                              = var.redshift_port
     rs_schema                            = var.redshift_schema
     rs_username                          = var.redshift_loader_user
-    rs_password                          = var.redshift_password
     rs_max_error                         = var.redshift_max_error
     rs_jsonpaths_bucket                  = var.redshift_jsonpaths_bucket
     temp_credentials_role_arn            = aws_iam_role.sts_credentials_role.arn
@@ -403,6 +416,10 @@ locals {
     is_private_ecr_registry = local.is_private_ecr_registry
     private_ecr_registry    = var.private_ecr_registry
     region                  = data.aws_region.current.name
+
+    is_redshift_password_ssm_param = local.is_redshift_password_ssm_param
+    rs_password                    = var.redshift_password
+    rs_password_ssm_param          = var.redshift_password_ssm_param
   })
 }
 

diff --git a/templates/config.json.tmpl b/templates/config.json.tmpl
@@ -15,22 +15,22 @@
   # Warehouse connection details
   "storage" : {
     # The destination DB
-    "type": "redshift",
+    "type": "redshift"
 
     # Redshift hostname
-    "host": "${rs_host}",
+    "host": "${rs_host}"
     # Database name
     "database": "${rs_database}",
     # Database port. Optional, default value 5439
-    "port": ${rs_port},
+    "port": ${rs_port}
     # DB schema name
-    "schema": "${rs_schema}",
+    "schema": "${rs_schema}"
     # DB user with permissions to load data
-    "username": "${rs_username}",
+    "username": "${rs_username}"
     # DB password
-    "password": "${rs_password}",
+    "password": $${REDSHIFT_PASSWORD}
     # Custom JDBC configuration. Optional, default value { "ssl": true }
-    "jdbc": { "ssl": true },
+    "jdbc": { "ssl": true }
     # MAXERROR, amount of acceptable loading errors. Optional, default value 10
     "maxError": ${rs_max_error}
 

diff --git a/templates/user-data.sh.tmpl b/templates/user-data.sh.tmpl
@@ -1,7 +1,13 @@
-%{ if is_private_ecr_registry }
+%{ if is_private_ecr_registry ~}
 aws ecr get-login-password --region ${region} | docker login --username AWS --password-stdin ${private_ecr_registry}
 %{ endif ~}
 
+%{ if is_redshift_password_ssm_param ~}
+redshift_password=$(aws ssm get-parameter --name "${rs_password_ssm_param}" --region "${region}" --with-decryption --query "Parameter.Value" --output text)
+%{ else ~}
+redshift_password='${rs_password}'
+%{ endif ~}
+
 # Launch the loader
 sudo docker run \
   -d \
@@ -20,6 +26,7 @@ sudo docker run \
   --env JDK_JAVA_OPTIONS='${java_opts}' \
   --env ACCEPT_LIMITED_USE_LICENSE=${accept_limited_use_license} \
   --env INSTANCE_ID=$(get_instance_id) \
+  --env REDSHIFT_PASSWORD=$${redshift_password} \
   ${private_ecr_registry}snowplow/rdb-loader-redshift:${version} \
   --config ${config_b64} \
   --iglu-config ${iglu_resolver_b64}

diff --git a/variables.tf b/variables.tf
@@ -363,6 +363,13 @@ variable "redshift_password" {
   description = "Password for redshift_loader_user used by loader to perform loading"
   type        = string
   sensitive   = true
+  default     = ""
+}
+
+variable "redshift_password_ssm_param" {
+  description = "SSM parameter containing the password for redshift_loader_user used by loader to perform loading"
+  type        = string
+  default     = ""
 }
 
 variable "redshift_max_error" {