diff --git a/.changelog/37868.txt b/.changelog/37868.txt new file mode 100644 index 00000000000..6b5c7b496cc --- /dev/null +++ b/.changelog/37868.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +resource/aws_fsx_lustre_file_system: Add `metadata_configuration` argument +``` \ No newline at end of file diff --git a/go.mod b/go.mod index 57d585e86a8..ef0bb73901f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/ProtonMail/go-crypto v1.1.0-alpha.2 github.com/YakDriver/go-version v0.1.0 github.com/YakDriver/regexache v0.23.0 - github.com/aws/aws-sdk-go v1.53.17 + github.com/aws/aws-sdk-go v1.53.18 github.com/aws/aws-sdk-go-v2 v1.27.1 github.com/aws/aws-sdk-go-v2/config v1.27.17 github.com/aws/aws-sdk-go-v2/credentials v1.17.17 diff --git a/go.sum b/go.sum index fc1d44f40be..4f342a97f5a 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/aws/aws-sdk-go v1.53.17 h1:TwtYMzVBTaqPVj/pcemHRIgk01OycWEcEUyUUX0tpCI= -github.com/aws/aws-sdk-go v1.53.17/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.53.18 h1:BeMeCK5e3bDGJj675FhnO94zRci8O35ombWXRvYomJs= +github.com/aws/aws-sdk-go v1.53.18/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.27.1 h1:xypCL2owhog46iFxBKKpBcw+bPTX/RJzwNj8uSilENw= github.com/aws/aws-sdk-go-v2 v1.27.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= diff --git a/internal/service/fsx/lustre_file_system.go b/internal/service/fsx/lustre_file_system.go index a0621369eb1..955691c2c34 100644 --- a/internal/service/fsx/lustre_file_system.go +++ b/internal/service/fsx/lustre_file_system.go @@ -179,6 +179,28 @@ func resourceLustreFileSystem() *schema.Resource { }, }, }, + "metadata_configuration": { + Type: schema.TypeList, + Optional: true, + Computed: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + names.AttrMode: { + Type: schema.TypeString, + Optional: true, + Computed: true, + ValidateDiagFunc: validation.ToDiagFunc(validation.StringInSlice(fsx.MetadataConfigurationMode_Values(), false)), + }, + names.AttrIOPS: { + Type: schema.TypeInt, + Optional: true, + Computed: true, + ValidateDiagFunc: validation.ToDiagFunc(validation.IntInSlice([]int{1500, 3000, 6000, 12000, 24000, 36000, 48000, 60000, 72000, 84000, 96000, 108000, 120000, 132000, 144000, 156000, 168000, 180000, 192000})), + }, + }, + }, + }, "mount_name": { Type: schema.TypeString, Computed: true, @@ -277,12 +299,13 @@ func resourceLustreFileSystem() *schema.Resource { CustomizeDiff: customdiff.Sequence( verify.SetTagsDiff, - resourceLustreFileSystemSchemaCustomizeDiff, + resourceLustreFileSystemStorageCapacityCustomizeDiff, + resourceLustreFileSystemMetadataConfigCustomizeDiff, ), } } -func resourceLustreFileSystemSchemaCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta interface{}) error { +func resourceLustreFileSystemStorageCapacityCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta any) error { // we want to force a new resource if the new storage capacity is less than the old one if d.HasChange("storage_capacity") { o, n := d.GetChange("storage_capacity") @@ -296,6 +319,51 @@ func resourceLustreFileSystemSchemaCustomizeDiff(_ context.Context, d *schema.Re return nil } +func resourceLustreFileSystemMetadataConfigCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta any) error { + //metadata_configuration is only supported when deployment_type is persistent2 + if v, ok := d.GetOk("metadata_configuration"); ok { + if len(v.([]any)) > 0 { + deploymentType := d.Get("deployment_type").(string) + if deploymentType != fsx.LustreDeploymentTypePersistent2 { + return fmt.Errorf("metadata_configuration can only be set when deployment type is " + fsx.LustreDeploymentTypePersistent2) + } + } + } + + // we want to force a new resource if the new Iops is less than the old one + if d.HasChange("metadata_configuration") { + if v, ok := d.GetOk("metadata_configuration"); ok && len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil { + if mode := d.Get("metadata_configuration.0.mode"); mode == fsx.MetadataConfigurationModeUserProvisioned { + o, n := d.GetChange("metadata_configuration") + + oldV := o.([]interface{}) + newV := n.([]interface{}) + var metaOld map[string]interface{} + var metaNew map[string]interface{} + + for _, v := range oldV { + metaOld = v.(map[string]interface{}) + } + + for _, v := range newV { + metaNew = v.(map[string]interface{}) + } + + if len(metaNew) > 0 && len(metaOld) > 0 { + if metaNew[names.AttrIOPS].(int) < metaOld[names.AttrIOPS].(int) { + log.Printf("[DEBUG] Forcing new due to metadata iops decrease. old iops: %d new iops: %d", metaOld[names.AttrIOPS].(int), metaNew[names.AttrIOPS].(int)) + if err := d.ForceNew("metadata_configuration.0.iops"); err != nil { + return err + } + } + } + } + } + } + + return nil +} + func resourceLustreFileSystemCreate(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics { var diags diag.Diagnostics conn := meta.(*conns.AWSClient).FSxConn(ctx) @@ -382,6 +450,11 @@ func resourceLustreFileSystemCreate(ctx context.Context, d *schema.ResourceData, inputB.LustreConfiguration.LogConfiguration = expandLustreLogCreateConfiguration(v.([]interface{})) } + if v, ok := d.GetOk("metadata_configuration"); ok && len(v.([]interface{})) > 0 { + inputC.LustreConfiguration.MetadataConfiguration = expandLustreMetadataCreateConfiguration(v.([]interface{})) + inputB.LustreConfiguration.MetadataConfiguration = expandLustreMetadataCreateConfiguration(v.([]interface{})) + } + if v, ok := d.GetOk("per_unit_storage_throughput"); ok { inputC.LustreConfiguration.PerUnitStorageThroughput = aws.Int64(int64(v.(int))) inputB.LustreConfiguration.PerUnitStorageThroughput = aws.Int64(int64(v.(int))) @@ -469,6 +542,9 @@ func resourceLustreFileSystemRead(ctx context.Context, d *schema.ResourceData, m if err := d.Set("log_configuration", flattenLustreLogConfiguration(lustreConfig.LogConfiguration)); err != nil { return sdkdiag.AppendErrorf(diags, "setting log_configuration: %s", err) } + if err := d.Set("metadata_configuration", flattenLustreMetadataConfiguration(lustreConfig.MetadataConfiguration)); err != nil { + return sdkdiag.AppendErrorf(diags, "setting metadata_configuration: %s", err) + } d.Set("mount_name", lustreConfig.MountName) d.Set("network_interface_ids", aws.StringValueSlice(filesystem.NetworkInterfaceIds)) d.Set(names.AttrOwnerID, filesystem.OwnerId) @@ -518,6 +594,10 @@ func resourceLustreFileSystemUpdate(ctx context.Context, d *schema.ResourceData, input.LustreConfiguration.LogConfiguration = expandLustreLogCreateConfiguration(d.Get("log_configuration").([]interface{})) } + if d.HasChange("metadata_configuration") { + input.LustreConfiguration.MetadataConfiguration = expandLustreMetadataUpdateConfiguration(d.Get("metadata_configuration").([]interface{})) + } + if d.HasChange("per_unit_storage_throughput") { input.LustreConfiguration.PerUnitStorageThroughput = aws.Int64(int64(d.Get("per_unit_storage_throughput").(int))) } @@ -647,6 +727,56 @@ func flattenLustreLogConfiguration(adopts *fsx.LustreLogConfiguration) []map[str return []map[string]interface{}{m} } +func expandLustreMetadataCreateConfiguration(l []interface{}) *fsx.CreateFileSystemLustreMetadataConfiguration { + if len(l) == 0 || l[0] == nil { + return nil + } + + data := l[0].(map[string]interface{}) + req := &fsx.CreateFileSystemLustreMetadataConfiguration{ + Mode: aws.String(data[names.AttrMode].(string)), + } + + if v, ok := data[names.AttrIOPS].(int); ok && v != 0 { + req.Iops = aws.Int64(int64(v)) + } + + return req +} + +func expandLustreMetadataUpdateConfiguration(l []interface{}) *fsx.UpdateFileSystemLustreMetadataConfiguration { + if len(l) == 0 || l[0] == nil { + return nil + } + + data := l[0].(map[string]interface{}) + req := &fsx.UpdateFileSystemLustreMetadataConfiguration{ + Mode: aws.String(data[names.AttrMode].(string)), + } + + if v, ok := data[names.AttrIOPS].(int); ok && v != 0 { + req.Iops = aws.Int64(int64(v)) + } + + return req +} + +func flattenLustreMetadataConfiguration(adopts *fsx.FileSystemLustreMetadataConfiguration) []map[string]interface{} { + if adopts == nil { + return []map[string]interface{}{} + } + + m := map[string]interface{}{ + names.AttrMode: aws.StringValue(adopts.Mode), + } + + if adopts.Iops != nil { + m[names.AttrIOPS] = aws.Int64Value(adopts.Iops) + } + + return []map[string]interface{}{m} +} + func logStateFunc(v interface{}) string { value := v.(string) // API returns the specific log stream arn instead of provided log group diff --git a/internal/service/fsx/lustre_file_system_test.go b/internal/service/fsx/lustre_file_system_test.go index c7d0d52b9d6..d7af5edc7f1 100644 --- a/internal/service/fsx/lustre_file_system_test.go +++ b/internal/service/fsx/lustre_file_system_test.go @@ -53,6 +53,7 @@ func TestAccFSxLustreFileSystem_basic(t *testing.T) { resource.TestCheckResourceAttr(resourceName, "imported_file_chunk_size", acctest.Ct0), resource.TestCheckResourceAttr(resourceName, "log_configuration.#", acctest.Ct1), resource.TestCheckResourceAttr(resourceName, "log_configuration.0.level", "DISABLED"), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct0), resource.TestCheckResourceAttrSet(resourceName, "mount_name"), resource.TestCheckResourceAttr(resourceName, "network_interface_ids.#", acctest.Ct2), acctest.CheckResourceAttrAccountID(resourceName, names.AttrOwnerID), @@ -768,6 +769,130 @@ func TestAccFSxLustreFileSystem_logConfig(t *testing.T) { }) } +func TestAccFSxLustreFileSystem_metadataConfig(t *testing.T) { + ctx := acctest.Context(t) + var filesystem1, filesystem2 fsx.FileSystem + resourceName := "aws_fsx_lustre_file_system.test" + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t); acctest.PreCheckPartitionHasService(t, fsx.EndpointsID) }, + ErrorCheck: acctest.ErrorCheck(t, names.FSxServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckLustreFileSystemDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccLustreFileSystemConfig_metadata(rName, "AUTOMATIC"), + Check: resource.ComposeTestCheckFunc( + testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem1), + testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem1), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "AUTOMATIC"), + resource.TestCheckResourceAttrSet(resourceName, "metadata_configuration.0.iops"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{names.AttrSecurityGroupIDs}, + }, + { + Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 1500), + Check: resource.ComposeTestCheckFunc( + testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem2), + testAccCheckLustreFileSystemNotRecreated(&filesystem1, &filesystem2), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "1500"), + ), + }, + }, + }) +} + +func TestAccFSxLustreFileSystem_metadataConfig_increase(t *testing.T) { + ctx := acctest.Context(t) + var filesystem1, filesystem2 fsx.FileSystem + resourceName := "aws_fsx_lustre_file_system.test" + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t); acctest.PreCheckPartitionHasService(t, fsx.EndpointsID) }, + ErrorCheck: acctest.ErrorCheck(t, names.FSxServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckLustreFileSystemDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 1500), + Check: resource.ComposeTestCheckFunc( + testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem1), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "1500"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{names.AttrSecurityGroupIDs}, + }, + { + Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 3000), + Check: resource.ComposeTestCheckFunc( + testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem2), + testAccCheckLustreFileSystemNotRecreated(&filesystem1, &filesystem2), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "3000"), + ), + }, + }, + }) +} + +func TestAccFSxLustreFileSystem_metadataConfig_decrease(t *testing.T) { + ctx := acctest.Context(t) + var filesystem1, filesystem2 fsx.FileSystem + resourceName := "aws_fsx_lustre_file_system.test" + rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix) + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { acctest.PreCheck(ctx, t); acctest.PreCheckPartitionHasService(t, fsx.EndpointsID) }, + ErrorCheck: acctest.ErrorCheck(t, names.FSxServiceID), + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories, + CheckDestroy: testAccCheckLustreFileSystemDestroy(ctx), + Steps: []resource.TestStep{ + { + Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 3000), + Check: resource.ComposeTestCheckFunc( + testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem1), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "3000"), + ), + }, + { + ResourceName: resourceName, + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{names.AttrSecurityGroupIDs}, + }, + { + Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 1500), + Check: resource.ComposeTestCheckFunc( + testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem2), + testAccCheckLustreFileSystemRecreated(&filesystem1, &filesystem2), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"), + resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "1500"), + ), + }, + }, + }) +} + func TestAccFSxLustreFileSystem_rootSquashConfig(t *testing.T) { ctx := acctest.Context(t) var filesystem fsx.FileSystem @@ -1608,6 +1733,45 @@ resource "aws_fsx_lustre_file_system" "test" { `, rName, status)) } +func testAccLustreFileSystemConfig_metadata(rName, mode string) string { + return acctest.ConfigCompose(testAccLustreFileSystemConfig_base(rName), fmt.Sprintf(` +resource "aws_fsx_lustre_file_system" "test" { + storage_capacity = 1200 + subnet_ids = aws_subnet.test[*].id + deployment_type = "PERSISTENT_2" + per_unit_storage_throughput = 125 + + metadata_configuration { + mode = %[2]q + } + + tags = { + Name = %[1]q + } +} +`, rName, mode)) +} + +func testAccLustreFileSystemConfig_metadata_iops(rName, mode string, iops int) string { + return acctest.ConfigCompose(testAccLustreFileSystemConfig_base(rName), fmt.Sprintf(` +resource "aws_fsx_lustre_file_system" "test" { + storage_capacity = 1200 + subnet_ids = aws_subnet.test[*].id + deployment_type = "PERSISTENT_2" + per_unit_storage_throughput = 125 + + metadata_configuration { + mode = %[2]q + iops = %[3]d + } + + tags = { + Name = %[1]q + } +} +`, rName, mode, iops)) +} + func testAccLustreFileSystemConfig_rootSquash(rName, uid string) string { return acctest.ConfigCompose(testAccLustreFileSystemConfig_base(rName), fmt.Sprintf(` resource "aws_fsx_lustre_file_system" "test" { diff --git a/website/docs/r/fsx_lustre_file_system.html.markdown b/website/docs/r/fsx_lustre_file_system.html.markdown index 420781b0d83..ea89e8d6905 100644 --- a/website/docs/r/fsx_lustre_file_system.html.markdown +++ b/website/docs/r/fsx_lustre_file_system.html.markdown @@ -47,6 +47,7 @@ This resource supports the following arguments: * `data_compression_type` - (Optional) Sets the data compression configuration for the file system. Valid values are `LZ4` and `NONE`. Default value is `NONE`. Unsetting this value reverts the compression type back to `NONE`. * `file_system_type_version` - (Optional) Sets the Lustre version for the file system that you're creating. Valid values are 2.10 for `SCRATCH_1`, `SCRATCH_2` and `PERSISTENT_1` deployment types. Valid values for 2.12 include all deployment types. * `log_configuration` - (Optional) The Lustre logging configuration used when creating an Amazon FSx for Lustre file system. When logging is enabled, Lustre logs error and warning events for data repositories associated with your file system to Amazon CloudWatch Logs. +* `metadata_configuration` - (Optional) The Lustre metadata configuration used when creating an Amazon FSx for Lustre file system. This can be used to specify a user provisioned metadata scale. This is only supported when `deployment_type` is set to `PERSISTENT_2`. See Metadata Configuration below. * `root_squash_configuration` - (Optional) The Lustre root squash configuration used when creating an Amazon FSx for Lustre file system. When enabled, root squash restricts root-level access from clients that try to access your file system as a root user. ### log_configuration @@ -54,6 +55,13 @@ This resource supports the following arguments: * `destination` - (Optional) The Amazon Resource Name (ARN) that specifies the destination of the logs. The name of the Amazon CloudWatch Logs log group must begin with the `/aws/fsx` prefix. If you do not provide a destination, Amazon FSx will create and use a log stream in the CloudWatch Logs `/aws/fsx/lustre` log group. * `level` - (Optional) Sets which data repository events are logged by Amazon FSx. Valid values are `WARN_ONLY`, `FAILURE_ONLY`, `ERROR_ONLY`, `WARN_ERROR` and `DISABLED`. Default value is `DISABLED`. +### metadata_configuration + +* `mode` - (Optional) Mode for the metadata configuration of the file system. Valid values are `AUTOMATIC`, and `USER_PROVISIONED`. +* `iops` - (Optional) Amount of IOPS provisioned for metadata. This parameter should only be used when the mode is set to `USER_PROVISIONED`. Valid Values are `1500`,`3000`,`6000` and `12000` through `192000` in increments of `12000`. + +!> **WARNING:** Updating the value of `iops` from a higher to a lower value will force a recreation of the resource. Any data on the file system will be lost when recreating. + ### root_squash_configuration * `no_squash_nids` - (Optional) When root squash is enabled, you can optionally specify an array of NIDs of clients for which root squash does not apply. A client NID is a Lustre Network Identifier used to uniquely identify a client. You can specify the NID as either a single address or a range of addresses: 1. A single address is described in standard Lustre NID format by specifying the client’s IP address followed by the Lustre network ID (for example, 10.0.1.6@tcp). 2. An address range is described using a dash to separate the range (for example, 10.0.[2-10].[1-255]@tcp).