From 65b0b228445ffad60084fa21208be3c8a446c713 Mon Sep 17 00:00:00 2001 From: lobsterdore Date: Wed, 24 May 2023 19:51:24 +0100 Subject: [PATCH] feat: Adds Option to Ignore Changes to GSIs This relates to this issue - https://github.com/hashicorp/terraform-provider-aws/issues/671. When using autoscaling with a provisioned table that has a GSI applying a TF change whilst the indices are scaled will reset capacity, which can be dangerous. This change has an option to ignore changes to global_secondary_index, which seems to be the only way to deal with this issue at present. --- README.md | 2 + autoscaling.tf | 8 +-- examples/autoscaling/main.tf | 15 +++--- examples/global-tables/main.tf | 11 ++-- main.tf | 93 +++++++++++++++++++++++++++++++++- outputs.tf | 8 +-- variables.tf | 6 +++ wrappers/main.tf | 11 ++-- 8 files changed, 128 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 64867d9..ba8751c 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,7 @@ No modules. | [aws_appautoscaling_target.table_read](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/appautoscaling_target) | resource | | [aws_appautoscaling_target.table_write](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/appautoscaling_target) | resource | | [aws_dynamodb_table.autoscaled](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table) | resource | +| [aws_dynamodb_table.autoscaled_gsi_ignore](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table) | resource | | [aws_dynamodb_table.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/dynamodb_table) | resource | ## Inputs @@ -95,6 +96,7 @@ No modules. | [deletion\_protection\_enabled](#input\_deletion\_protection\_enabled) | Enables deletion protection for table | `bool` | `null` | no | | [global\_secondary\_indexes](#input\_global\_secondary\_indexes) | Describe a GSI for the table; subject to the normal limits on the number of GSIs, projected attributes, etc. | `any` | `[]` | no | | [hash\_key](#input\_hash\_key) | The attribute to use as the hash (partition) key. Must also be defined as an attribute | `string` | `null` | no | +| [ignore\_changes\_global\_secondary\_index](#input\_ignore\_changes\_global\_secondary\_index) | Whether to ignore changes lifecycle to global secondary indices, useful for provisioned tables with scaling | `bool` | `false` | no | | [local\_secondary\_indexes](#input\_local\_secondary\_indexes) | Describe an LSI on the table; these can only be allocated at creation so you cannot change this definition after you have created the resource. | `any` | `[]` | no | | [name](#input\_name) | Name of the DynamoDB table | `string` | `null` | no | | [point\_in\_time\_recovery\_enabled](#input\_point\_in\_time\_recovery\_enabled) | Whether to enable point-in-time recovery | `bool` | `false` | no | diff --git a/autoscaling.tf b/autoscaling.tf index 50e765e..a38c24c 100644 --- a/autoscaling.tf +++ b/autoscaling.tf @@ -3,7 +3,7 @@ resource "aws_appautoscaling_target" "table_read" { max_capacity = var.autoscaling_read["max_capacity"] min_capacity = var.read_capacity - resource_id = "table/${aws_dynamodb_table.autoscaled[0].name}" + resource_id = "table/${try(aws_dynamodb_table.autoscaled[0].name, aws_dynamodb_table.autoscaled_gsi_ignore[0].name, "")}" scalable_dimension = "dynamodb:table:ReadCapacityUnits" service_namespace = "dynamodb" } @@ -33,7 +33,7 @@ resource "aws_appautoscaling_target" "table_write" { max_capacity = var.autoscaling_write["max_capacity"] min_capacity = var.write_capacity - resource_id = "table/${aws_dynamodb_table.autoscaled[0].name}" + resource_id = "table/${try(aws_dynamodb_table.autoscaled[0].name, aws_dynamodb_table.autoscaled_gsi_ignore[0].name, "")}" scalable_dimension = "dynamodb:table:WriteCapacityUnits" service_namespace = "dynamodb" } @@ -63,7 +63,7 @@ resource "aws_appautoscaling_target" "index_read" { max_capacity = each.value["read_max_capacity"] min_capacity = each.value["read_min_capacity"] - resource_id = "table/${aws_dynamodb_table.autoscaled[0].name}/index/${each.key}" + resource_id = "table/${try(aws_dynamodb_table.autoscaled[0].name, aws_dynamodb_table.autoscaled_gsi_ignore[0].name, "")}/index/${each.key}" scalable_dimension = "dynamodb:index:ReadCapacityUnits" service_namespace = "dynamodb" } @@ -93,7 +93,7 @@ resource "aws_appautoscaling_target" "index_write" { max_capacity = each.value["write_max_capacity"] min_capacity = each.value["write_min_capacity"] - resource_id = "table/${aws_dynamodb_table.autoscaled[0].name}/index/${each.key}" + resource_id = "table/${try(aws_dynamodb_table.autoscaled[0].name, aws_dynamodb_table.autoscaled_gsi_ignore[0].name, "")}/index/${each.key}" scalable_dimension = "dynamodb:index:WriteCapacityUnits" service_namespace = "dynamodb" } diff --git a/examples/autoscaling/main.tf b/examples/autoscaling/main.tf index 7a57c01..52dbc4e 100644 --- a/examples/autoscaling/main.tf +++ b/examples/autoscaling/main.tf @@ -9,13 +9,14 @@ resource "random_pet" "this" { module "dynamodb_table" { source = "../../" - name = "my-table-${random_pet.this.id}" - hash_key = "id" - range_key = "title" - billing_mode = "PROVISIONED" - read_capacity = 5 - write_capacity = 5 - autoscaling_enabled = true + name = "my-table-${random_pet.this.id}" + hash_key = "id" + range_key = "title" + billing_mode = "PROVISIONED" + read_capacity = 5 + write_capacity = 5 + autoscaling_enabled = true + ignore_changes_global_secondary_index = true autoscaling_read = { scale_in_cooldown = 50 diff --git a/examples/global-tables/main.tf b/examples/global-tables/main.tf index b996557..d2d6da5 100644 --- a/examples/global-tables/main.tf +++ b/examples/global-tables/main.tf @@ -41,11 +41,12 @@ resource "aws_kms_key" "secondary" { module "dynamodb_table" { source = "../../" - name = "my-table-${random_pet.this.id}" - hash_key = "id" - range_key = "title" - stream_enabled = true - stream_view_type = "NEW_AND_OLD_IMAGES" + name = "my-table-${random_pet.this.id}" + hash_key = "id" + ignore_changes_global_secondary_index = true + range_key = "title" + stream_enabled = true + stream_view_type = "NEW_AND_OLD_IMAGES" server_side_encryption_enabled = true server_side_encryption_kms_key_arn = aws_kms_key.primary.arn diff --git a/main.tf b/main.tf index e622b25..65a3f4e 100644 --- a/main.tf +++ b/main.tf @@ -86,7 +86,7 @@ resource "aws_dynamodb_table" "this" { } resource "aws_dynamodb_table" "autoscaled" { - count = var.create_table && var.autoscaling_enabled ? 1 : 0 + count = var.create_table && var.autoscaling_enabled && !var.ignore_changes_global_secondary_index ? 1 : 0 name = var.name billing_mode = var.billing_mode @@ -175,3 +175,94 @@ resource "aws_dynamodb_table" "autoscaled" { ignore_changes = [read_capacity, write_capacity] } } + +resource "aws_dynamodb_table" "autoscaled_gsi_ignore" { + count = var.create_table && var.autoscaling_enabled && var.ignore_changes_global_secondary_index ? 1 : 0 + + name = var.name + billing_mode = var.billing_mode + hash_key = var.hash_key + range_key = var.range_key + read_capacity = var.read_capacity + write_capacity = var.write_capacity + stream_enabled = var.stream_enabled + stream_view_type = var.stream_view_type + table_class = var.table_class + deletion_protection_enabled = var.deletion_protection_enabled + + ttl { + enabled = var.ttl_enabled + attribute_name = var.ttl_attribute_name + } + + point_in_time_recovery { + enabled = var.point_in_time_recovery_enabled + } + + dynamic "attribute" { + for_each = var.attributes + + content { + name = attribute.value.name + type = attribute.value.type + } + } + + dynamic "local_secondary_index" { + for_each = var.local_secondary_indexes + + content { + name = local_secondary_index.value.name + range_key = local_secondary_index.value.range_key + projection_type = local_secondary_index.value.projection_type + non_key_attributes = lookup(local_secondary_index.value, "non_key_attributes", null) + } + } + + dynamic "global_secondary_index" { + for_each = var.global_secondary_indexes + + content { + name = global_secondary_index.value.name + hash_key = global_secondary_index.value.hash_key + projection_type = global_secondary_index.value.projection_type + range_key = lookup(global_secondary_index.value, "range_key", null) + read_capacity = lookup(global_secondary_index.value, "read_capacity", null) + write_capacity = lookup(global_secondary_index.value, "write_capacity", null) + non_key_attributes = lookup(global_secondary_index.value, "non_key_attributes", null) + } + } + + dynamic "replica" { + for_each = var.replica_regions + + content { + region_name = replica.value.region_name + kms_key_arn = lookup(replica.value, "kms_key_arn", null) + propagate_tags = lookup(replica.value, "propagate_tags", null) + point_in_time_recovery = lookup(replica.value, "point_in_time_recovery", null) + } + } + + server_side_encryption { + enabled = var.server_side_encryption_enabled + kms_key_arn = var.server_side_encryption_kms_key_arn + } + + tags = merge( + var.tags, + { + "Name" = format("%s", var.name) + }, + ) + + timeouts { + create = lookup(var.timeouts, "create", null) + delete = lookup(var.timeouts, "delete", null) + update = lookup(var.timeouts, "update", null) + } + + lifecycle { + ignore_changes = [global_secondary_index, read_capacity, write_capacity] + } +} diff --git a/outputs.tf b/outputs.tf index 310cfc5..fa07482 100644 --- a/outputs.tf +++ b/outputs.tf @@ -1,19 +1,19 @@ output "dynamodb_table_arn" { description = "ARN of the DynamoDB table" - value = try(aws_dynamodb_table.this[0].arn, aws_dynamodb_table.autoscaled[0].arn, "") + value = try(aws_dynamodb_table.this[0].arn, aws_dynamodb_table.autoscaled[0].arn, aws_dynamodb_table.autoscaled_gsi_ignore[0].arn, "") } output "dynamodb_table_id" { description = "ID of the DynamoDB table" - value = try(aws_dynamodb_table.this[0].id, aws_dynamodb_table.autoscaled[0].id, "") + value = try(aws_dynamodb_table.this[0].id, aws_dynamodb_table.autoscaled[0].id, aws_dynamodb_table.autoscaled_gsi_ignore[0].id, "") } output "dynamodb_table_stream_arn" { description = "The ARN of the Table Stream. Only available when var.stream_enabled is true" - value = var.stream_enabled ? try(aws_dynamodb_table.this[0].stream_arn, aws_dynamodb_table.autoscaled[0].stream_arn, "") : null + value = var.stream_enabled ? try(aws_dynamodb_table.this[0].stream_arn, aws_dynamodb_table.autoscaled[0].stream_arn, aws_dynamodb_table.autoscaled_gsi_ignore[0].stream_arn, "") : null } output "dynamodb_table_stream_label" { description = "A timestamp, in ISO 8601 format of the Table Stream. Only available when var.stream_enabled is true" - value = var.stream_enabled ? try(aws_dynamodb_table.this[0].stream_label, aws_dynamodb_table.autoscaled[0].stream_label, "") : null + value = var.stream_enabled ? try(aws_dynamodb_table.this[0].stream_label, aws_dynamodb_table.autoscaled[0].stream_label, aws_dynamodb_table.autoscaled_gsi_ignore[0].stream_label, "") : null } diff --git a/variables.tf b/variables.tf index a633d9e..aa13fb1 100644 --- a/variables.tf +++ b/variables.tf @@ -167,3 +167,9 @@ variable "deletion_protection_enabled" { type = bool default = null } + +variable "ignore_changes_global_secondary_index" { + description = "Whether to ignore changes lifecycle to global secondary indices, useful for provisioned tables with scaling" + type = bool + default = false +} diff --git a/wrappers/main.tf b/wrappers/main.tf index 5c7996d..031be25 100644 --- a/wrappers/main.tf +++ b/wrappers/main.tf @@ -33,9 +33,10 @@ module "wrapper" { scale_out_cooldown = 0 target_value = 70 }) - autoscaling_read = try(each.value.autoscaling_read, var.defaults.autoscaling_read, {}) - autoscaling_write = try(each.value.autoscaling_write, var.defaults.autoscaling_write, {}) - autoscaling_indexes = try(each.value.autoscaling_indexes, var.defaults.autoscaling_indexes, {}) - table_class = try(each.value.table_class, var.defaults.table_class, null) - deletion_protection_enabled = try(each.value.deletion_protection_enabled, var.defaults.deletion_protection_enabled, null) + autoscaling_read = try(each.value.autoscaling_read, var.defaults.autoscaling_read, {}) + autoscaling_write = try(each.value.autoscaling_write, var.defaults.autoscaling_write, {}) + autoscaling_indexes = try(each.value.autoscaling_indexes, var.defaults.autoscaling_indexes, {}) + table_class = try(each.value.table_class, var.defaults.table_class, null) + deletion_protection_enabled = try(each.value.deletion_protection_enabled, var.defaults.deletion_protection_enabled, null) + ignore_changes_global_secondary_index = try(each.value.ignore_changes_global_secondary_index, var.defaults.ignore_changes_global_secondary_index, false) }