Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[new feature] fsx lustre file system metadata #37868

Merged
merged 6 commits into from
Jun 7, 2024
Merged

[new feature] fsx lustre file system metadata #37868

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8285463
updates
awsaxeman Jun 6, 2024
bbe0645
add changelog
awsaxeman Jun 6, 2024
0760d87
Updates AWS SDK
gdavison Jun 7, 2024
b071263
Adds test check for `metadata_configuration.iops` when `mode` is `AUT…
gdavison Jun 7, 2024
6319818
Linting fixes
gdavison Jun 7, 2024
5a2755b
linting fix
gdavison Jun 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .changelog/37868.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:enhancement
resource/aws_fsx_lustre_file_system: Add `metadata_configuration` argument
```
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/ProtonMail/go-crypto v1.1.0-alpha.2
github.com/YakDriver/go-version v0.1.0
github.com/YakDriver/regexache v0.23.0
github.com/aws/aws-sdk-go v1.53.17
github.com/aws/aws-sdk-go v1.53.18
github.com/aws/aws-sdk-go-v2 v1.27.1
github.com/aws/aws-sdk-go-v2/config v1.27.17
github.com/aws/aws-sdk-go-v2/credentials v1.17.17
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go v1.53.17 h1:TwtYMzVBTaqPVj/pcemHRIgk01OycWEcEUyUUX0tpCI=
github.com/aws/aws-sdk-go v1.53.17/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go v1.53.18 h1:BeMeCK5e3bDGJj675FhnO94zRci8O35ombWXRvYomJs=
github.com/aws/aws-sdk-go v1.53.18/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go-v2 v1.27.1 h1:xypCL2owhog46iFxBKKpBcw+bPTX/RJzwNj8uSilENw=
github.com/aws/aws-sdk-go-v2 v1.27.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
Expand Down
134 changes: 132 additions & 2 deletions internal/service/fsx/lustre_file_system.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,28 @@ func resourceLustreFileSystem() *schema.Resource {
},
},
},
"metadata_configuration": {
Type: schema.TypeList,
Optional: true,
Computed: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
names.AttrMode: {
Type: schema.TypeString,
Optional: true,
Computed: true,
ValidateDiagFunc: validation.ToDiagFunc(validation.StringInSlice(fsx.MetadataConfigurationMode_Values(), false)),
},
names.AttrIOPS: {
Type: schema.TypeInt,
Optional: true,
Computed: true,
ValidateDiagFunc: validation.ToDiagFunc(validation.IntInSlice([]int{1500, 3000, 6000, 12000, 24000, 36000, 48000, 60000, 72000, 84000, 96000, 108000, 120000, 132000, 144000, 156000, 168000, 180000, 192000})),
},
},
},
},
"mount_name": {
Type: schema.TypeString,
Computed: true,
Expand Down Expand Up @@ -277,12 +299,13 @@ func resourceLustreFileSystem() *schema.Resource {

CustomizeDiff: customdiff.Sequence(
verify.SetTagsDiff,
resourceLustreFileSystemSchemaCustomizeDiff,
resourceLustreFileSystemStorageCapacityCustomizeDiff,
resourceLustreFileSystemMetadataConfigCustomizeDiff,
),
}
}

func resourceLustreFileSystemSchemaCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta interface{}) error {
func resourceLustreFileSystemStorageCapacityCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta any) error {
// we want to force a new resource if the new storage capacity is less than the old one
if d.HasChange("storage_capacity") {
o, n := d.GetChange("storage_capacity")
Expand All @@ -296,6 +319,51 @@ func resourceLustreFileSystemSchemaCustomizeDiff(_ context.Context, d *schema.Re
return nil
}

func resourceLustreFileSystemMetadataConfigCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta any) error {
//metadata_configuration is only supported when deployment_type is persistent2
if v, ok := d.GetOk("metadata_configuration"); ok {
if len(v.([]any)) > 0 {
deploymentType := d.Get("deployment_type").(string)
if deploymentType != fsx.LustreDeploymentTypePersistent2 {
return fmt.Errorf("metadata_configuration can only be set when deployment type is " + fsx.LustreDeploymentTypePersistent2)
}
}
}

// we want to force a new resource if the new Iops is less than the old one
if d.HasChange("metadata_configuration") {
if v, ok := d.GetOk("metadata_configuration"); ok && len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil {
if mode := d.Get("metadata_configuration.0.mode"); mode == fsx.MetadataConfigurationModeUserProvisioned {
o, n := d.GetChange("metadata_configuration")

oldV := o.([]interface{})
newV := n.([]interface{})
var metaOld map[string]interface{}
var metaNew map[string]interface{}

for _, v := range oldV {
metaOld = v.(map[string]interface{})
}

for _, v := range newV {
metaNew = v.(map[string]interface{})
}

if len(metaNew) > 0 && len(metaOld) > 0 {
if metaNew[names.AttrIOPS].(int) < metaOld[names.AttrIOPS].(int) {
log.Printf("[DEBUG] Forcing new due to metadata iops decrease. old iops: %d new iops: %d", metaOld[names.AttrIOPS].(int), metaNew[names.AttrIOPS].(int))
if err := d.ForceNew("metadata_configuration.0.iops"); err != nil {
return err
}
}
}
}
}
}

return nil
}

func resourceLustreFileSystemCreate(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics {
var diags diag.Diagnostics
conn := meta.(*conns.AWSClient).FSxConn(ctx)
Expand Down Expand Up @@ -382,6 +450,11 @@ func resourceLustreFileSystemCreate(ctx context.Context, d *schema.ResourceData,
inputB.LustreConfiguration.LogConfiguration = expandLustreLogCreateConfiguration(v.([]interface{}))
}

if v, ok := d.GetOk("metadata_configuration"); ok && len(v.([]interface{})) > 0 {
inputC.LustreConfiguration.MetadataConfiguration = expandLustreMetadataCreateConfiguration(v.([]interface{}))
inputB.LustreConfiguration.MetadataConfiguration = expandLustreMetadataCreateConfiguration(v.([]interface{}))
}

if v, ok := d.GetOk("per_unit_storage_throughput"); ok {
inputC.LustreConfiguration.PerUnitStorageThroughput = aws.Int64(int64(v.(int)))
inputB.LustreConfiguration.PerUnitStorageThroughput = aws.Int64(int64(v.(int)))
Expand Down Expand Up @@ -469,6 +542,9 @@ func resourceLustreFileSystemRead(ctx context.Context, d *schema.ResourceData, m
if err := d.Set("log_configuration", flattenLustreLogConfiguration(lustreConfig.LogConfiguration)); err != nil {
return sdkdiag.AppendErrorf(diags, "setting log_configuration: %s", err)
}
if err := d.Set("metadata_configuration", flattenLustreMetadataConfiguration(lustreConfig.MetadataConfiguration)); err != nil {
return sdkdiag.AppendErrorf(diags, "setting metadata_configuration: %s", err)
}
d.Set("mount_name", lustreConfig.MountName)
d.Set("network_interface_ids", aws.StringValueSlice(filesystem.NetworkInterfaceIds))
d.Set(names.AttrOwnerID, filesystem.OwnerId)
Expand Down Expand Up @@ -518,6 +594,10 @@ func resourceLustreFileSystemUpdate(ctx context.Context, d *schema.ResourceData,
input.LustreConfiguration.LogConfiguration = expandLustreLogCreateConfiguration(d.Get("log_configuration").([]interface{}))
}

if d.HasChange("metadata_configuration") {
input.LustreConfiguration.MetadataConfiguration = expandLustreMetadataUpdateConfiguration(d.Get("metadata_configuration").([]interface{}))
}

if d.HasChange("per_unit_storage_throughput") {
input.LustreConfiguration.PerUnitStorageThroughput = aws.Int64(int64(d.Get("per_unit_storage_throughput").(int)))
}
Expand Down Expand Up @@ -647,6 +727,56 @@ func flattenLustreLogConfiguration(adopts *fsx.LustreLogConfiguration) []map[str
return []map[string]interface{}{m}
}

func expandLustreMetadataCreateConfiguration(l []interface{}) *fsx.CreateFileSystemLustreMetadataConfiguration {
if len(l) == 0 || l[0] == nil {
return nil
}

data := l[0].(map[string]interface{})
req := &fsx.CreateFileSystemLustreMetadataConfiguration{
Mode: aws.String(data[names.AttrMode].(string)),
}

if v, ok := data[names.AttrIOPS].(int); ok && v != 0 {
req.Iops = aws.Int64(int64(v))
}

return req
}

func expandLustreMetadataUpdateConfiguration(l []interface{}) *fsx.UpdateFileSystemLustreMetadataConfiguration {
if len(l) == 0 || l[0] == nil {
return nil
}

data := l[0].(map[string]interface{})
req := &fsx.UpdateFileSystemLustreMetadataConfiguration{
Mode: aws.String(data[names.AttrMode].(string)),
}

if v, ok := data[names.AttrIOPS].(int); ok && v != 0 {
req.Iops = aws.Int64(int64(v))
}

return req
}

func flattenLustreMetadataConfiguration(adopts *fsx.FileSystemLustreMetadataConfiguration) []map[string]interface{} {
if adopts == nil {
return []map[string]interface{}{}
}

m := map[string]interface{}{
names.AttrMode: aws.StringValue(adopts.Mode),
}

if adopts.Iops != nil {
m[names.AttrIOPS] = aws.Int64Value(adopts.Iops)
}

return []map[string]interface{}{m}
}

func logStateFunc(v interface{}) string {
value := v.(string)
// API returns the specific log stream arn instead of provided log group
Expand Down
164 changes: 164 additions & 0 deletions internal/service/fsx/lustre_file_system_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ func TestAccFSxLustreFileSystem_basic(t *testing.T) {
resource.TestCheckResourceAttr(resourceName, "imported_file_chunk_size", acctest.Ct0),
resource.TestCheckResourceAttr(resourceName, "log_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "log_configuration.0.level", "DISABLED"),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct0),
resource.TestCheckResourceAttrSet(resourceName, "mount_name"),
resource.TestCheckResourceAttr(resourceName, "network_interface_ids.#", acctest.Ct2),
acctest.CheckResourceAttrAccountID(resourceName, names.AttrOwnerID),
Expand Down Expand Up @@ -768,6 +769,130 @@ func TestAccFSxLustreFileSystem_logConfig(t *testing.T) {
})
}

func TestAccFSxLustreFileSystem_metadataConfig(t *testing.T) {
ctx := acctest.Context(t)
var filesystem1, filesystem2 fsx.FileSystem
resourceName := "aws_fsx_lustre_file_system.test"
rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix)

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { acctest.PreCheck(ctx, t); acctest.PreCheckPartitionHasService(t, fsx.EndpointsID) },
ErrorCheck: acctest.ErrorCheck(t, names.FSxServiceID),
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories,
CheckDestroy: testAccCheckLustreFileSystemDestroy(ctx),
Steps: []resource.TestStep{
{
Config: testAccLustreFileSystemConfig_metadata(rName, "AUTOMATIC"),
Check: resource.ComposeTestCheckFunc(
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem1),
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "AUTOMATIC"),
resource.TestCheckResourceAttrSet(resourceName, "metadata_configuration.0.iops"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{names.AttrSecurityGroupIDs},
},
{
Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 1500),
Check: resource.ComposeTestCheckFunc(
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem2),
testAccCheckLustreFileSystemNotRecreated(&filesystem1, &filesystem2),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "1500"),
),
},
},
})
}

func TestAccFSxLustreFileSystem_metadataConfig_increase(t *testing.T) {
ctx := acctest.Context(t)
var filesystem1, filesystem2 fsx.FileSystem
resourceName := "aws_fsx_lustre_file_system.test"
rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix)

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { acctest.PreCheck(ctx, t); acctest.PreCheckPartitionHasService(t, fsx.EndpointsID) },
ErrorCheck: acctest.ErrorCheck(t, names.FSxServiceID),
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories,
CheckDestroy: testAccCheckLustreFileSystemDestroy(ctx),
Steps: []resource.TestStep{
{
Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 1500),
Check: resource.ComposeTestCheckFunc(
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "1500"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{names.AttrSecurityGroupIDs},
},
{
Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 3000),
Check: resource.ComposeTestCheckFunc(
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem2),
testAccCheckLustreFileSystemNotRecreated(&filesystem1, &filesystem2),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "3000"),
),
},
},
})
}

func TestAccFSxLustreFileSystem_metadataConfig_decrease(t *testing.T) {
ctx := acctest.Context(t)
var filesystem1, filesystem2 fsx.FileSystem
resourceName := "aws_fsx_lustre_file_system.test"
rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix)

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { acctest.PreCheck(ctx, t); acctest.PreCheckPartitionHasService(t, fsx.EndpointsID) },
ErrorCheck: acctest.ErrorCheck(t, names.FSxServiceID),
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories,
CheckDestroy: testAccCheckLustreFileSystemDestroy(ctx),
Steps: []resource.TestStep{
{
Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 3000),
Check: resource.ComposeTestCheckFunc(
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "3000"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{names.AttrSecurityGroupIDs},
},
{
Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 1500),
Check: resource.ComposeTestCheckFunc(
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem2),
testAccCheckLustreFileSystemRecreated(&filesystem1, &filesystem2),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "1500"),
),
},
},
})
}

func TestAccFSxLustreFileSystem_rootSquashConfig(t *testing.T) {
ctx := acctest.Context(t)
var filesystem fsx.FileSystem
Expand Down Expand Up @@ -1608,6 +1733,45 @@ resource "aws_fsx_lustre_file_system" "test" {
`, rName, status))
}

func testAccLustreFileSystemConfig_metadata(rName, mode string) string {
return acctest.ConfigCompose(testAccLustreFileSystemConfig_base(rName), fmt.Sprintf(`
resource "aws_fsx_lustre_file_system" "test" {
storage_capacity = 1200
subnet_ids = aws_subnet.test[*].id
deployment_type = "PERSISTENT_2"
per_unit_storage_throughput = 125

metadata_configuration {
mode = %[2]q
}

tags = {
Name = %[1]q
}
}
`, rName, mode))
}

func testAccLustreFileSystemConfig_metadata_iops(rName, mode string, iops int) string {
return acctest.ConfigCompose(testAccLustreFileSystemConfig_base(rName), fmt.Sprintf(`
resource "aws_fsx_lustre_file_system" "test" {
storage_capacity = 1200
subnet_ids = aws_subnet.test[*].id
deployment_type = "PERSISTENT_2"
per_unit_storage_throughput = 125

metadata_configuration {
mode = %[2]q
iops = %[3]d
}

tags = {
Name = %[1]q
}
}
`, rName, mode, iops))
}

func testAccLustreFileSystemConfig_rootSquash(rName, uid string) string {
return acctest.ConfigCompose(testAccLustreFileSystemConfig_base(rName), fmt.Sprintf(`
resource "aws_fsx_lustre_file_system" "test" {
Expand Down
Loading
Loading