From 49ae92467937bc9517ed1d3845a63b3eaf958b63 Mon Sep 17 00:00:00 2001 From: Sanooj Date: Mon, 8 Jul 2019 14:09:41 +0530 Subject: [PATCH 1/2] New Resource: aws_emr_instance_fleet --- aws/provider.go | 1 + aws/resource_aws_emr_cluster.go | 310 ++++++++- aws/resource_aws_emr_cluster_test.go | 333 ++++++++++ aws/resource_aws_emr_instance_fleet.go | 597 ++++++++++++++++++ aws/resource_aws_emr_instance_fleet_test.go | 540 ++++++++++++++++ aws/validators.go | 79 +++ website/aws.erb | 4 + website/docs/r/emr_cluster.html.markdown | 14 + .../docs/r/emr_instance_fleet.html.markdown | 156 +++++ 9 files changed, 2000 insertions(+), 34 deletions(-) create mode 100644 aws/resource_aws_emr_instance_fleet.go create mode 100644 aws/resource_aws_emr_instance_fleet_test.go create mode 100644 website/docs/r/emr_instance_fleet.html.markdown diff --git a/aws/provider.go b/aws/provider.go index 24082a86343..b7e598013b2 100644 --- a/aws/provider.go +++ b/aws/provider.go @@ -478,6 +478,7 @@ func Provider() terraform.ResourceProvider { "aws_elb_attachment": resourceAwsElbAttachment(), "aws_emr_cluster": resourceAwsEMRCluster(), "aws_emr_instance_group": resourceAwsEMRInstanceGroup(), + "aws_emr_instance_fleet": resourceAwsEMRInstanceFleet(), "aws_emr_security_configuration": resourceAwsEMRSecurityConfiguration(), "aws_flow_log": resourceAwsFlowLog(), "aws_gamelift_alias": resourceAwsGameliftAlias(), diff --git a/aws/resource_aws_emr_cluster.go b/aws/resource_aws_emr_cluster.go index 9521377804d..de844b7de91 100644 --- a/aws/resource_aws_emr_cluster.go +++ b/aws/resource_aws_emr_cluster.go @@ -140,7 +140,7 @@ func resourceAwsEMRCluster() *schema.Resource { DiffSuppressFunc: func(k, old, new string, d *schema.ResourceData) bool { // EMR uses a proprietary filesystem called EMRFS // and both s3n & s3 protocols are mapped to that FS - // so they're equvivalent in this context (confirmed by AWS support) + // so they're equivalent in this context (confirmed by AWS support) old = strings.Replace(old, "s3n://", "s3://", -1) return old == new }, @@ -184,6 +184,11 @@ func resourceAwsEMRCluster() *schema.Resource { Optional: true, ForceNew: true, }, + "subnet_ids": { + Type: schema.TypeString, + Optional: true, + Computed: true, + }, "additional_master_security_groups": { Type: schema.TypeString, Optional: true, @@ -214,6 +219,16 @@ func resourceAwsEMRCluster() *schema.Resource { Optional: true, ForceNew: true, }, + "availability_zone": { + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + "availability_zones": { + Type: schema.TypeString, + Optional: true, + Computed: true, + }, }, }, }, @@ -255,6 +270,50 @@ func resourceAwsEMRCluster() *schema.Resource { }, }, }, + "instance_fleet": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "instance_fleet_type": { + Type: schema.TypeString, + Required: true, + ForceNew: true, + ValidateFunc: validateAwsEmrInstanceFleetType, + }, + "instance_type_configs": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + Elem: instanceTypeConfigSchema(), + }, + "launch_specifications": { + Type: schema.TypeSet, + Optional: true, + ForceNew: true, + MinItems: 1, + MaxItems: 1, + Elem: launchSpecificationsSchema(), + }, + "name": { + Type: schema.TypeString, + Optional: true, + ForceNew: true, + }, + "target_on_demand_capacity": { + Type: schema.TypeInt, + Optional: true, + Default: 0, + }, + "target_spot_capacity": { + Type: schema.TypeInt, + Optional: true, + Default: 0, + }, + }, + }, + }, "core_instance_group": { Type: schema.TypeList, Optional: true, @@ -735,6 +794,16 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error if v, ok := attributes["subnet_id"]; ok { instanceConfig.Ec2SubnetId = aws.String(v.(string)) } + if v, ok := attributes["subnet_ids"]; ok { + if len(v.(string)) > 0 { + strSlice := strings.Split(v.(string), ",") + for i, s := range strSlice { + strSlice[i] = strings.TrimSpace(s) + } + + instanceConfig.Ec2SubnetIds = aws.StringSlice(strSlice) + } + } if v, ok := attributes["additional_master_security_groups"]; ok { strSlice := strings.Split(v.(string), ",") @@ -766,6 +835,24 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error if v, ok := attributes["service_access_security_group"]; ok { instanceConfig.ServiceAccessSecurityGroup = aws.String(v.(string)) } + if v, ok := attributes["availability_zone"]; ok { + instanceConfig.Placement = &emr.PlacementType{ + AvailabilityZone: aws.String(v.(string)), + } + } + + if v, ok := attributes["availability_zones"]; ok { + if len(v.(string)) > 0 { + strSlice := strings.Split(v.(string), ",") + for i, s := range strSlice { + strSlice[i] = strings.TrimSpace(s) + } + + instanceConfig.Placement = &emr.PlacementType{ + AvailabilityZones: aws.StringSlice(strSlice), + } + } + } } // DEPRECATED: Remove in a future major version @@ -779,7 +866,10 @@ func resourceAwsEMRClusterCreate(d *schema.ResourceData, meta interface{}) error instanceConfig.InstanceGroups = instanceGroups } - + if v, ok := d.GetOk("instance_fleet"); ok { + instanceFleetConfigs := v.(*schema.Set).List() + instanceConfig.InstanceFleets = expandInstanceFleetConfigs(instanceFleetConfigs) + } emrApps := expandApplications(applications) params := &emr.RunJobFlowInput{ @@ -946,51 +1036,65 @@ func resourceAwsEMRClusterRead(d *schema.ResourceData, meta interface{}) error { d.Set("cluster_state", state) } + if _, ok := d.GetOk("instance_group"); ok { - instanceGroups, err := fetchAllEMRInstanceGroups(emrconn, d.Id()) + instanceGroups, err := fetchAllEMRInstanceGroups(emrconn, d.Id()) - if err != nil { - return err - } + if err != nil { + return err + } - coreGroup := emrCoreInstanceGroup(instanceGroups) - masterGroup := findMasterGroup(instanceGroups) + coreGroup := emrCoreInstanceGroup(instanceGroups) + masterGroup := findMasterGroup(instanceGroups) - d.Set("core_instance_count", 0) - d.Set("core_instance_type", "") - d.Set("master_instance_type", "") + d.Set("core_instance_count", 0) + d.Set("core_instance_type", "") + d.Set("master_instance_type", "") - if coreGroup != nil { - d.Set("core_instance_type", coreGroup.InstanceType) - d.Set("core_instance_count", coreGroup.RequestedInstanceCount) - } + if coreGroup != nil { + d.Set("core_instance_type", coreGroup.InstanceType) + d.Set("core_instance_count", coreGroup.RequestedInstanceCount) + } - if masterGroup != nil { - d.Set("master_instance_type", masterGroup.InstanceType) - } + if masterGroup != nil { + d.Set("master_instance_type", masterGroup.InstanceType) + } - flattenedInstanceGroups, err := flattenInstanceGroups(instanceGroups) - if err != nil { - return fmt.Errorf("error flattening instance groups: %s", err) - } - if err := d.Set("instance_group", flattenedInstanceGroups); err != nil { - return fmt.Errorf("error setting instance_group: %s", err) - } + flattenedInstanceGroups, err := flattenInstanceGroups(instanceGroups) + if err != nil { + return fmt.Errorf("error flattening instance groups: %s", err) + } + if err := d.Set("instance_group", flattenedInstanceGroups); err != nil { + return fmt.Errorf("error setting instance_group: %s", err) + } - flattenedCoreInstanceGroup, err := flattenEmrCoreInstanceGroup(coreGroup) + flattenedCoreInstanceGroup, err := flattenEmrCoreInstanceGroup(coreGroup) - if err != nil { - return fmt.Errorf("error flattening core_instance_group: %s", err) - } + if err != nil { + return fmt.Errorf("error flattening core_instance_group: %s", err) + } - if err := d.Set("core_instance_group", flattenedCoreInstanceGroup); err != nil { - return fmt.Errorf("error setting core_instance_group: %s", err) - } + if err := d.Set("core_instance_group", flattenedCoreInstanceGroup); err != nil { + return fmt.Errorf("error setting core_instance_group: %s", err) + } - if err := d.Set("master_instance_group", flattenEmrMasterInstanceGroup(masterGroup)); err != nil { - return fmt.Errorf("error setting master_instance_group: %s", err) + if err := d.Set("master_instance_group", flattenEmrMasterInstanceGroup(masterGroup)); err != nil { + return fmt.Errorf("error setting master_instance_group: %s", err) + } } + if _, ok := d.GetOk("instance_fleet"); ok { + instanceFleets, err := fetchAllEMRInstanceFleets(emrconn, d.Id()) + if err == nil { + coreFleet := emrCoreInstanceFleet(instanceFleets) + if coreFleet != nil { + d.Set("core_instance_type", coreFleet.InstanceFleetType) + } + if err := d.Set("instance_fleet", flattenInstanceFleets(instanceFleets)); err != nil { + log.Printf("[ERR] Error setting EMR instance fleets: %s", err) + } + } + } d.Set("name", cluster.Name) d.Set("service_role", cluster.ServiceRole) @@ -1418,6 +1522,10 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{ if ia.Ec2SubnetId != nil { attrs["subnet_id"] = *ia.Ec2SubnetId } + if len(ia.RequestedEc2SubnetIds) > 0 { + strs := aws.StringValueSlice(ia.RequestedEc2SubnetIds) + attrs["subnet_ids"] = strings.Join(strs, ",") + } if ia.IamInstanceProfile != nil { attrs["instance_profile"] = *ia.IamInstanceProfile } @@ -1441,6 +1549,14 @@ func flattenEc2Attributes(ia *emr.Ec2InstanceAttributes) []map[string]interface{ attrs["service_access_security_group"] = *ia.ServiceAccessSecurityGroup } + if ia.Ec2AvailabilityZone != nil { + attrs["availability_zone"] = *ia.Ec2AvailabilityZone + } + if len(ia.RequestedEc2AvailabilityZones) > 0 { + strs := aws.StringValueSlice(ia.RequestedEc2AvailabilityZones) + attrs["availability_zones"] = strings.Join(strs, ",") + } + result = append(result, attrs) return result @@ -1664,6 +1780,123 @@ func flattenInstanceGroups(igs []*emr.InstanceGroup) (*schema.Set, error) { return schema.NewSet(resourceAwsEMRClusterInstanceGroupHash, instanceGroupSet), nil } +func flattenInstanceFleets(ifleets []*emr.InstanceFleet) []map[string]interface{} { + result := make([]map[string]interface{}, 0) + + for _, fleet := range ifleets { + attrs := make(map[string]interface{}) + + attrs["instance_fleet_type"] = *fleet.InstanceFleetType + + instanceTypeConfigs := make([]map[string]interface{}, 0) + for _, its := range fleet.InstanceTypeSpecifications { + itsAttrs := make(map[string]interface{}) + + if its.BidPrice != nil { + itsAttrs["bid_price"] = *its.BidPrice + } else { + itsAttrs["bid_price"] = "" + } + + if its.BidPriceAsPercentageOfOnDemandPrice != nil { + itsAttrs["bid_price_as_percentage_of_on_demand_price"] = *its.BidPriceAsPercentageOfOnDemandPrice + } + + configurations := make([]map[string]interface{}, 0) + if its.Configurations != nil { + for _, cfg := range its.Configurations { + configurationsAttrs := make(map[string]interface{}) + + configurationsAttrs["classification"] = *cfg.Classification + + additionalConfigurations := make([]map[string]interface{}, 0) + if cfg.Configurations != nil { + for _, additionalCfg := range cfg.Configurations { + additionalConfigurationsAttrs := make(map[string]interface{}) + + additionalConfigurationsAttrs["classification"] = *additionalCfg.Classification + + additionalProperties := make(map[string]string) + for k, v := range additionalCfg.Properties { + additionalProperties[k] = *v + } + additionalConfigurationsAttrs["properties"] = additionalProperties + + additionalConfigurations = append(additionalConfigurations, additionalConfigurationsAttrs) + } + } + configurationsAttrs["configurations"] = additionalConfigurations + + properties := make(map[string]string) + for k, v := range cfg.Properties { + properties[k] = *v + } + configurationsAttrs["properties"] = properties + + configurations = append(configurations, configurationsAttrs) + } + } + itsAttrs["configurations"] = configurations + + ebsConfiguration := make([]map[string]interface{}, 0) + if its.EbsBlockDevices != nil { + for _, ebsbd := range its.EbsBlockDevices { + ebscAttrs := make(map[string]interface{}) + + if ebsbd.VolumeSpecification != nil { + if ebsbd.VolumeSpecification.Iops != nil { + ebscAttrs["iops"] = *ebsbd.VolumeSpecification.Iops + } else { + ebscAttrs["iops"] = "" + } + ebscAttrs["size"] = *ebsbd.VolumeSpecification.SizeInGB + ebscAttrs["type"] = *ebsbd.VolumeSpecification.VolumeType + ebscAttrs["volumes_per_instance"] = 1 + } + + ebsConfiguration = append(ebsConfiguration, ebscAttrs) + } + } + if len(ebsConfiguration) > 0 { + itsAttrs["ebs_config"] = ebsConfiguration + } + + itsAttrs["instance_type"] = *its.InstanceType + itsAttrs["weighted_capacity"] = *its.WeightedCapacity + + instanceTypeConfigs = append(instanceTypeConfigs, itsAttrs) + } + if len(instanceTypeConfigs) > 0 { + attrs["instance_type_configs"] = instanceTypeConfigs + } + + launchSpecifications := make(map[string]interface{}) + if fleet.LaunchSpecifications != nil { + spotSpecification := make(map[string]interface{}) + + if fleet.LaunchSpecifications.SpotSpecification.BlockDurationMinutes != nil { + spotSpecification["block_duration_minutes"] = *fleet.LaunchSpecifications.SpotSpecification.BlockDurationMinutes + } else { + spotSpecification["block_duration_minutes"] = "" + } + + spotSpecification["timeout_action"] = *fleet.LaunchSpecifications.SpotSpecification.TimeoutAction + spotSpecification["timeout_duration_minutes"] = *fleet.LaunchSpecifications.SpotSpecification.TimeoutDurationMinutes + + launchSpecifications["spot_specification"] = spotSpecification + } + attrs["launch_specifications"] = launchSpecifications + + attrs["name"] = *fleet.Name + attrs["target_on_demand_capacity"] = *fleet.TargetOnDemandCapacity + attrs["target_spot_capacity"] = *fleet.TargetSpotCapacity + + result = append(result, attrs) + } + + return result +} + func flattenEBSConfig(ebsBlockDevices []*emr.EbsBlockDevice) *schema.Set { ebsConfig := make([]interface{}, 0) @@ -1709,6 +1942,15 @@ func emrCoreInstanceGroup(grps []*emr.InstanceGroup) *emr.InstanceGroup { return nil } +func emrCoreInstanceFleet(flts []*emr.InstanceFleet) *emr.InstanceFleet { + for _, flt := range flts { + if aws.StringValue(flt.InstanceFleetType) == emr.InstanceFleetTypeCore { + return flt + } + } + return nil +} + func expandTags(m map[string]interface{}) []*emr.Tag { var result []*emr.Tag for k, v := range m { diff --git a/aws/resource_aws_emr_cluster_test.go b/aws/resource_aws_emr_cluster_test.go index 2ad3661a9c9..0c7b718fc40 100644 --- a/aws/resource_aws_emr_cluster_test.go +++ b/aws/resource_aws_emr_cluster_test.go @@ -885,6 +885,26 @@ func TestAccAWSEMRCluster_MasterInstanceGroup_Name(t *testing.T) { }) } +func TestAccAWSEMRCluster_instance_fleet(t *testing.T) { + var cluster emr.Cluster + r := acctest.RandInt() + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrClusterConfigInstanceFleets(r), + Check: resource.ComposeTestCheckFunc( + testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &cluster), + resource.TestCheckResourceAttr( + "aws_emr_cluster.tf-test-cluster", "instance_fleet.#", "2"), + ), + }, + }, + }) +} + func TestAccAWSEMRCluster_security_config(t *testing.T) { var cluster emr.Cluster r := acctest.RandInt() @@ -1708,6 +1728,7 @@ resource "aws_iam_policy" "iam_emr_profile_policy" { "elasticmapreduce:ListBootstrapActions", "elasticmapreduce:ListClusters", "elasticmapreduce:ListInstanceGroups", + "elasticmapreduce:ListInstanceFleets", "elasticmapreduce:ListInstances", "elasticmapreduce:ListSteps", "kinesis:CreateStream", @@ -2082,6 +2103,7 @@ resource "aws_iam_policy" "iam_emr_profile_policy" { "elasticmapreduce:ListBootstrapActions", "elasticmapreduce:ListClusters", "elasticmapreduce:ListInstanceGroups", + "elasticmapreduce:ListInstanceFleets", "elasticmapreduce:ListInstances", "elasticmapreduce:ListSteps", "kinesis:CreateStream", @@ -2397,6 +2419,7 @@ resource "aws_iam_policy" "iam_emr_profile_policy" { "elasticmapreduce:ListBootstrapActions", "elasticmapreduce:ListClusters", "elasticmapreduce:ListInstanceGroups", + "elasticmapreduce:ListInstanceFleets", "elasticmapreduce:ListInstances", "elasticmapreduce:ListSteps", "kinesis:CreateStream", @@ -2720,6 +2743,7 @@ resource "aws_iam_policy" "iam_emr_profile_policy" { "elasticmapreduce:ListBootstrapActions", "elasticmapreduce:ListClusters", "elasticmapreduce:ListInstanceGroups", + "elasticmapreduce:ListInstanceFleets", "elasticmapreduce:ListInstances", "elasticmapreduce:ListSteps", "kinesis:CreateStream", @@ -3122,6 +3146,7 @@ resource "aws_iam_policy" "iam_emr_profile_policy" { "elasticmapreduce:ListBootstrapActions", "elasticmapreduce:ListClusters", "elasticmapreduce:ListInstanceGroups", + "elasticmapreduce:ListInstanceFleets", "elasticmapreduce:ListInstances", "elasticmapreduce:ListSteps", "kinesis:CreateStream", @@ -3989,6 +4014,7 @@ resource "aws_iam_policy" "iam_emr_profile_policy" { "elasticmapreduce:ListBootstrapActions", "elasticmapreduce:ListClusters", "elasticmapreduce:ListInstanceGroups", + "elasticmapreduce:ListInstanceFleets", "elasticmapreduce:ListInstances", "elasticmapreduce:ListSteps", "kinesis:CreateStream", @@ -4323,6 +4349,7 @@ resource "aws_iam_policy" "iam_emr_profile_policy" { "elasticmapreduce:ListBootstrapActions", "elasticmapreduce:ListClusters", "elasticmapreduce:ListInstanceGroups", + "elasticmapreduce:ListInstanceFleets", "elasticmapreduce:ListInstances", "elasticmapreduce:ListSteps", "kinesis:CreateStream", @@ -4676,6 +4703,7 @@ resource "aws_iam_policy" "iam_emr_profile_policy" { "elasticmapreduce:ListBootstrapActions", "elasticmapreduce:ListClusters", "elasticmapreduce:ListInstanceGroups", + "elasticmapreduce:ListInstanceFleets", "elasticmapreduce:ListInstances", "elasticmapreduce:ListSteps", "kinesis:CreateStream", @@ -5027,6 +5055,7 @@ resource "aws_iam_policy" "iam_emr_profile_policy" { "elasticmapreduce:ListBootstrapActions", "elasticmapreduce:ListClusters", "elasticmapreduce:ListInstanceGroups", + "elasticmapreduce:ListInstanceFleets", "elasticmapreduce:ListInstances", "elasticmapreduce:ListSteps", "kinesis:CreateStream", @@ -5378,6 +5407,7 @@ resource "aws_iam_policy" "iam_emr_profile_policy" { "elasticmapreduce:ListBootstrapActions", "elasticmapreduce:ListClusters", "elasticmapreduce:ListInstanceGroups", + "elasticmapreduce:ListInstanceFleets", "elasticmapreduce:ListInstances", "elasticmapreduce:ListSteps", "kinesis:CreateStream", @@ -5523,6 +5553,303 @@ resource "aws_emr_cluster" "test" { `, rName, masterInstanceType) } +func testAccAWSEmrClusterConfigInstanceFleets(r int) string { + return fmt.Sprintf(` +provider "aws" { + region = "us-west-2" +} +resource "aws_emr_cluster" "tf-test-cluster" { + name = "emr-test-%d" + release_label = "emr-5.8.0" + applications = ["Spark"] + ec2_attributes { + subnet_ids = "${aws_subnet.main.id}" + emr_managed_master_security_group = "${aws_security_group.allow_all.id}" + emr_managed_slave_security_group = "${aws_security_group.allow_all.id}" + instance_profile = "${aws_iam_instance_profile.emr_profile.arn}" + } + instance_fleet { + instance_fleet_type = "MASTER" + instance_type_configs { + instance_type = "m3.xlarge" + } + + target_on_demand_capacity = 1 + } + instance_fleet{ + instance_fleet_type = "CORE" + instance_type_configs{ + bid_price_as_percentage_of_on_demand_price = 80 + ebs_optimized = true + ebs_config { + size = 100 + type = "gp2" + volumes_per_instance = 1 + } + instance_type = "m3.xlarge" + weighted_capacity = 1 + } + + launch_specifications { + spot_specification { + block_duration_minutes = 60 + timeout_action = "SWITCH_TO_ON_DEMAND" + timeout_duration_minutes = 10 + } + } + name = "instance-fleet-test" + target_on_demand_capacity = 0 + target_spot_capacity = 1 + } + + tags = { + role = "rolename" + dns_zone = "env_zone" + env = "env" + name = "name-env" + } + keep_job_flow_alive_when_no_steps = true + termination_protection = false + bootstrap_action { + path = "s3://elasticmapreduce/bootstrap-actions/run-if" + name = "runif" + args = ["instance.isMaster=true", "echo running on master node"] + } + configurations = "test-fixtures/emr_configurations.json" + depends_on = ["aws_main_route_table_association.a"] + service_role = "${aws_iam_role.iam_emr_default_role.arn}" +} +resource "aws_security_group" "allow_all" { + name = "allow_all_%d" + description = "Allow all inbound traffic" + vpc_id = "${aws_vpc.main.id}" + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + depends_on = ["aws_subnet.main"] + lifecycle { + ignore_changes = ["ingress", "egress"] + } + tags { + name = "emr_test" + } +} +resource "aws_vpc" "main" { + cidr_block = "168.31.0.0/16" + enable_dns_hostnames = true + tags { + name = "emr_test_%d" + } +} +resource "aws_subnet" "main" { + vpc_id = "${aws_vpc.main.id}" + cidr_block = "168.31.0.0/20" + tags { + name = "emr_test_%d" + } +} +resource "aws_internet_gateway" "gw" { + vpc_id = "${aws_vpc.main.id}" +} +resource "aws_route_table" "r" { + vpc_id = "${aws_vpc.main.id}" + route { + cidr_block = "0.0.0.0/0" + gateway_id = "${aws_internet_gateway.gw.id}" + } +} +resource "aws_main_route_table_association" "a" { + vpc_id = "${aws_vpc.main.id}" + route_table_id = "${aws_route_table.r.id}" +} +### +# IAM things +### +# IAM role for EMR Service +resource "aws_iam_role" "iam_emr_default_role" { + name = "iam_emr_default_role_%d" + assume_role_policy = < 0 { + config.InstanceTypeConfigs = expandInstanceTypeConfigs(v.List()) + } + + if v, ok := data.Get("launch_specifications").(*schema.Set); ok && v.Len() == 1 { + config.LaunchSpecifications = expandLaunchSpecification(v.List()[0]) + } + + return config +} + +func expandInstanceFleetConfigs(instanceFleetConfigs []interface{}) []*emr.InstanceFleetConfig { + configsOut := []*emr.InstanceFleetConfig{} + + for _, raw := range instanceFleetConfigs { + configAttributes := raw.(map[string]interface{}) + + configInstanceFleetType := configAttributes["instance_fleet_type"].(string) + configName := configAttributes["name"].(string) + configTargetOnDemandCapacity := configAttributes["target_on_demand_capacity"].(int) + configTargetSpotCapacity := configAttributes["target_spot_capacity"].(int) + + config := &emr.InstanceFleetConfig{ + InstanceFleetType: aws.String(configInstanceFleetType), + Name: aws.String(configName), + TargetOnDemandCapacity: aws.Int64(int64(configTargetOnDemandCapacity)), + TargetSpotCapacity: aws.Int64(int64(configTargetSpotCapacity)), + } + + if v, ok := configAttributes["instance_type_configs"].(*schema.Set); ok && v.Len() > 0 { + config.InstanceTypeConfigs = expandInstanceTypeConfigs(v.List()) + } + + if v, ok := configAttributes["launch_specifications"].(*schema.Set); ok && v.Len() == 1 { + config.LaunchSpecifications = expandLaunchSpecification(v.List()[0]) + } + + configsOut = append(configsOut, config) + } + + return configsOut +} + +func expandInstanceTypeConfigs(instanceTypeConfigs []interface{}) []*emr.InstanceTypeConfig { + configsOut := []*emr.InstanceTypeConfig{} + + for _, raw := range instanceTypeConfigs { + configAttributes := raw.(map[string]interface{}) + + configInstanceType := configAttributes["instance_type"].(string) + + config := &emr.InstanceTypeConfig{ + InstanceType: aws.String(configInstanceType), + } + + if bidPrice, ok := configAttributes["bid_price"]; ok { + if bidPrice != "" { + config.BidPrice = aws.String(bidPrice.(string)) + } + } + + if v, ok := configAttributes["bid_price_as_percentage_of_on_demand_price"].(float64); ok && v != 0 { + config.BidPriceAsPercentageOfOnDemandPrice = aws.Float64(v) + } + + if v, ok := configAttributes["weighted_capacity"].(int); ok { + config.WeightedCapacity = aws.Int64(int64(v)) + } + + if v, ok := configAttributes["configurations"].(*schema.Set); ok && v.Len() > 0 { + config.Configurations = expandConfigurations(v.List()) + } + + if v, ok := configAttributes["ebs_config"].(*schema.Set); ok && v.Len() == 1 { + config.EbsConfiguration = expandEbsConfiguration(v.List()) + + if v, ok := configAttributes["ebs_optimized"].(bool); ok { + config.EbsConfiguration.EbsOptimized = aws.Bool(v) + } + } + + configsOut = append(configsOut, config) + } + + return configsOut +} + +func expandConfigurations(configurations []interface{}) []*emr.Configuration { + configsOut := []*emr.Configuration{} + + for _, raw := range configurations { + configAttributes := raw.(map[string]interface{}) + + config := &emr.Configuration{} + + if v, ok := configAttributes["classification"].(string); ok { + config.Classification = aws.String(v) + } + + if rawConfig, ok := configAttributes["configurations"]; ok { + config.Configurations = expandConfigurations(rawConfig.([]interface{})) + } + + if v, ok := configAttributes["properties"]; ok { + properties := make(map[string]string) + for k, v := range v.(map[string]interface{}) { + properties[k] = v.(string) + } + config.Properties = aws.StringMap(properties) + } + + configsOut = append(configsOut, config) + } + + return configsOut +} + +func expandEbsConfiguration(ebsConfigurations []interface{}) *emr.EbsConfiguration { + ebsConfig := &emr.EbsConfiguration{} + + ebsConfigs := make([]*emr.EbsBlockDeviceConfig, 0) + for _, ebsConfiguration := range ebsConfigurations { + cfg := ebsConfiguration.(map[string]interface{}) + + ebs := &emr.EbsBlockDeviceConfig{} + + volumeSpec := &emr.VolumeSpecification{ + SizeInGB: aws.Int64(int64(cfg["size"].(int))), + VolumeType: aws.String(cfg["type"].(string)), + } + + if v, ok := cfg["iops"].(int); ok && v != 0 { + volumeSpec.Iops = aws.Int64(int64(v)) + } + + if v, ok := cfg["volumes_per_instance"].(int); ok && v != 0 { + ebs.VolumesPerInstance = aws.Int64(int64(v)) + } + + ebs.VolumeSpecification = volumeSpec + + ebsConfigs = append(ebsConfigs, ebs) + } + + ebsConfig.EbsBlockDeviceConfigs = ebsConfigs + + return ebsConfig +} + +func expandLaunchSpecification(launchSpecification interface{}) *emr.InstanceFleetProvisioningSpecifications { + configAttributes := launchSpecification.(map[string]interface{}) + + return &emr.InstanceFleetProvisioningSpecifications{ + SpotSpecification: expandSpotSpecification(configAttributes["spot_specification"].(*schema.Set).List()[0]), + } +} + +func expandSpotSpecification(spotSpecification interface{}) *emr.SpotProvisioningSpecification { + configAttributes := spotSpecification.(map[string]interface{}) + + spotProvisioning := &emr.SpotProvisioningSpecification{ + TimeoutAction: aws.String(configAttributes["timeout_action"].(string)), + TimeoutDurationMinutes: aws.Int64(int64(configAttributes["timeout_duration_minutes"].(int))), + } + + if v, ok := configAttributes["block_duration_minutes"]; ok && v != 0 { + spotProvisioning.BlockDurationMinutes = aws.Int64(int64(v.(int))) + } + + return spotProvisioning +} + +func resourceAwsEMRInstanceFleetCreate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).emrconn + + clusterId := d.Get("cluster_id").(string) + instanceFleetConfig := expandInstanceFleetConfig(d) + + addInstanceFleetInput := &emr.AddInstanceFleetInput{ + ClusterId: aws.String(clusterId), + InstanceFleet: instanceFleetConfig, + } + + log.Printf("[DEBUG] Creating EMR instance fleet params: %s", addInstanceFleetInput) + resp, err := conn.AddInstanceFleet(addInstanceFleetInput) + if err != nil { + return err + } + + log.Printf("[DEBUG] Created EMR instance fleet finished: %#v", resp) + if resp == nil { + return fmt.Errorf("error creating instance fleet: no instance fleet returned") + } + d.SetId(*resp.InstanceFleetId) + + return nil +} + +func resourceAwsEMRInstanceFleetRead(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).emrconn + fleet, err := fetchEMRInstanceFleet(conn, d.Get("cluster_id").(string), d.Id()) + if err != nil { + switch err { + case emrInstanceFleetNotFound: + log.Printf("[DEBUG] EMR Instance Fleet (%s) not found, removing", d.Id()) + d.SetId("") + return nil + default: + return err + } + } + + // Guard against the chance of fetchEMRInstanceFleet returning nil fleet but + // not a emrInstanceFleetNotFound error + if fleet == nil { + log.Printf("[DEBUG] EMR Instance Fleet (%s) not found, removing", d.Id()) + d.SetId("") + return nil + } + + d.Set("name", fleet.Name) + d.Set("provisioned_on_demand_capacity", fleet.ProvisionedOnDemandCapacity) + d.Set("provisioned_spot_capacity", fleet.ProvisionedSpotCapacity) + if fleet.Status != nil && fleet.Status.State != nil { + d.Set("status", fleet.Status.State) + } + + return nil +} + +func fetchAllEMRInstanceFleets(conn *emr.EMR, clusterId string) ([]*emr.InstanceFleet, error) { + listInstanceFleetsInput := &emr.ListInstanceFleetsInput{ + ClusterId: aws.String(clusterId), + } + + var fleets []*emr.InstanceFleet + marker := aws.String("initial") + for marker != nil { + log.Printf("[DEBUG] EMR Cluster Instance Marker: %s", *marker) + respFleets, errFleets := conn.ListInstanceFleets(listInstanceFleetsInput) + if errFleets != nil { + return nil, fmt.Errorf("[ERR] Error reading EMR cluster (%s): %s", clusterId, errFleets) + } + if respFleets == nil { + return nil, fmt.Errorf("[ERR] Error reading EMR Instance Fleet for cluster (%s)", clusterId) + } + + if respFleets.InstanceFleets != nil { + for _, f := range respFleets.InstanceFleets { + fleets = append(fleets, f) + } + } else { + log.Printf("[DEBUG] EMR Instance Fleet list was empty") + } + marker = respFleets.Marker + } + + if len(fleets) == 0 { + return nil, fmt.Errorf("[WARN] No instance fleets found for EMR Cluster (%s)", clusterId) + } + + return fleets, nil +} + +func fetchEMRInstanceFleet(conn *emr.EMR, clusterId, fleetId string) (*emr.InstanceFleet, error) { + fleets, err := fetchAllEMRInstanceFleets(conn, clusterId) + if err != nil { + return nil, err + } + + var instanceFleet *emr.InstanceFleet + for _, fleet := range fleets { + if fleetId == *fleet.Id { + instanceFleet = fleet + break + } + } + + if instanceFleet != nil { + return instanceFleet, nil + } + + return nil, emrInstanceFleetNotFound +} + +func resourceAwsEMRInstanceFleetUpdate(d *schema.ResourceData, meta interface{}) error { + conn := meta.(*AWSClient).emrconn + + log.Printf("[DEBUG] Modify EMR task fleet") + clusterId := d.Get("cluster_id").(string) + targetOnDemandCapacity := d.Get("target_on_demand_capacity").(int) + targetSpotCapacity := d.Get("target_spot_capacity").(int) + + modifyInstanceFleetInput := &emr.ModifyInstanceFleetInput{ + ClusterId: aws.String(clusterId), + InstanceFleet: &emr.InstanceFleetModifyConfig{ + InstanceFleetId: aws.String(d.Id()), + TargetOnDemandCapacity: aws.Int64(int64(targetOnDemandCapacity)), + TargetSpotCapacity: aws.Int64(int64(targetSpotCapacity)), + }, + } + + _, err := conn.ModifyInstanceFleet(modifyInstanceFleetInput) + if err != nil { + return err + } + + stateConf := &resource.StateChangeConf{ + Pending: []string{emr.InstanceFleetStateProvisioning, emr.InstanceFleetStateBootstrapping, emr.InstanceFleetStateResizing}, + Target: []string{emr.InstanceFleetStateRunning}, + Refresh: instanceFleetStateRefresh(conn, d.Get("cluster_id").(string), d.Id()), + Timeout: 10 * time.Minute, + Delay: 10 * time.Second, + MinTimeout: 3 * time.Second, + } + + _, err = stateConf.WaitForState() + if err != nil { + return fmt.Errorf("error waiting for instance (%s) to terminate: %s", d.Id(), err) + } + + return resourceAwsEMRInstanceFleetRead(d, meta) +} + +func instanceFleetStateRefresh(conn *emr.EMR, clusterID, ifID string) resource.StateRefreshFunc { + return func() (interface{}, string, error) { + fleet, err := fetchEMRInstanceFleet(conn, clusterID, ifID) + if err != nil { + return nil, "Not Found", err + } + + if fleet.Status == nil || fleet.Status.State == nil { + log.Printf("[WARN] ERM Instance Fleet found, but without state") + return nil, "Undefined", fmt.Errorf("undefined EMR Cluster Instance Fleet state") + } + + return fleet, *fleet.Status.State, nil + } +} + +func resourceAwsEMRInstanceFleetDelete(d *schema.ResourceData, meta interface{}) error { + log.Printf("[WARN] AWS EMR Instance Fleet does not support DELETE; resizing cluster to zero before removing from state") + conn := meta.(*AWSClient).emrconn + + clusterId := d.Get("cluster_id").(string) + + modifyInstanceFleetInput := &emr.ModifyInstanceFleetInput{ + ClusterId: aws.String(clusterId), + InstanceFleet: &emr.InstanceFleetModifyConfig{ + InstanceFleetId: aws.String(d.Id()), + TargetOnDemandCapacity: aws.Int64(0), + TargetSpotCapacity: aws.Int64(0), + }, + } + + _, err := conn.ModifyInstanceFleet(modifyInstanceFleetInput) + if err != nil { + return err + } + + return nil +} diff --git a/aws/resource_aws_emr_instance_fleet_test.go b/aws/resource_aws_emr_instance_fleet_test.go new file mode 100644 index 00000000000..009990b7647 --- /dev/null +++ b/aws/resource_aws_emr_instance_fleet_test.go @@ -0,0 +1,540 @@ +package aws + +import ( + "fmt" + "log" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/emr" + "github.com/hashicorp/terraform/helper/acctest" + "github.com/hashicorp/terraform/helper/resource" + "github.com/hashicorp/terraform/terraform" +) + +func TestAccAWSEMRInstanceFleet_basic(t *testing.T) { + var fleet emr.InstanceFleet + rInt := acctest.RandInt() + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrInstanceFleetDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrInstanceFleetConfig(rInt), + Check: resource.ComposeTestCheckFunc(testAccCheckAWSEmrInstanceFleetExists("aws_emr_instance_fleet.task", &fleet), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_fleet_type", "TASK"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_type_configs.#", "1"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_on_demand_capacity", "0"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_spot_capacity", "1"), + ), + }, + }, + }) +} + +func TestAccAWSEMRInstanceFleet_zero_count(t *testing.T) { + var fleet emr.InstanceFleet + rInt := acctest.RandInt() + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrInstanceFleetDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrInstanceFleetConfig(rInt), + Check: resource.ComposeTestCheckFunc(testAccCheckAWSEmrInstanceFleetExists("aws_emr_instance_fleet.task", &fleet), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_fleet_type", "TASK"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_type_configs.#", "1"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_on_demand_capacity", "0"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_spot_capacity", "1"), + ), + }, + { + Config: testAccAWSEmrInstanceFleetConfigZeroCount(rInt), + Check: resource.ComposeTestCheckFunc(testAccCheckAWSEmrInstanceFleetExists("aws_emr_instance_fleet.task", &fleet), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_fleet_type", "TASK"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_type_configs.#", "1"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_on_demand_capacity", "0"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_spot_capacity", "0"), + ), + }, + }, + }) +} + +func TestAccAWSEMRInstanceFleet_ebsBasic(t *testing.T) { + var fleet emr.InstanceFleet + rInt := acctest.RandInt() + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrInstanceFleetDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrInstanceFleetConfigEbsBasic(rInt), + Check: resource.ComposeTestCheckFunc(testAccCheckAWSEmrInstanceFleetExists("aws_emr_instance_fleet.task", &fleet), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_fleet_type", "TASK"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_type_configs.#", "1"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_on_demand_capacity", "0"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_spot_capacity", "1"), + ), + }, + }, + }) +} + +func TestAccAWSEMRInstanceFleet_full(t *testing.T) { + var fleet emr.InstanceFleet + rInt := acctest.RandInt() + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckAWSEmrInstanceFleetDestroy, + Steps: []resource.TestStep{ + { + Config: testAccAWSEmrInstanceFleetConfigFull(rInt), + Check: resource.ComposeTestCheckFunc(testAccCheckAWSEmrInstanceFleetExists("aws_emr_instance_fleet.task", &fleet), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_fleet_type", "TASK"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "instance_type_configs.#", "1"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_on_demand_capacity", "1"), + resource.TestCheckResourceAttr("aws_emr_instance_fleet.task", "target_spot_capacity", "1"), + ), + }, + }, + }) +} + +func testAccCheckAWSEmrInstanceFleetDestroy(s *terraform.State) error { + conn := testAccProvider.Meta().(*AWSClient).emrconn + + for _, rs := range s.RootModule().Resources { + if rs.Type != "aws_emr_cluster" { + continue + } + + params := &emr.DescribeClusterInput{ + ClusterId: aws.String(rs.Primary.ID), + } + + describe, err := conn.DescribeCluster(params) + + if err == nil { + if describe.Cluster != nil && + *describe.Cluster.Status.State == "WAITING" { + return fmt.Errorf("EMR Cluster still exists") + } + } + + providerErr, ok := err.(awserr.Error) + if !ok { + return err + } + + log.Printf("[ERROR] %v", providerErr) + } + + return nil +} + +func testAccCheckAWSEmrInstanceFleetExists(n string, v *emr.InstanceFleet) resource.TestCheckFunc { + return func(s *terraform.State) error { + rs, ok := s.RootModule().Resources[n] + if !ok { + return fmt.Errorf("Not found: %s", n) + } + if rs.Primary.ID == "" { + return fmt.Errorf("No task fleet id set") + } + meta := testAccProvider.Meta() + conn := meta.(*AWSClient).emrconn + f, err := fetchEMRInstanceFleet(conn, rs.Primary.Attributes["cluster_id"], rs.Primary.ID) + if err != nil { + return fmt.Errorf("EMR error: %v", err) + } + + if f == nil { + return fmt.Errorf("No match found for (%s)", n) + } + + v = f + return nil + } +} + +const testAccAWSEmrInstanceFleetBase = ` +provider "aws" { + region = "us-west-2" +} +resource "aws_emr_cluster" "tf-test-cluster" { + name = "emr-test-%d" + release_label = "emr-5.10.0" + applications = ["Spark"] + ec2_attributes { + subnet_id = "${aws_subnet.main.id}" + emr_managed_master_security_group = "${aws_security_group.allow_all.id}" + emr_managed_slave_security_group = "${aws_security_group.allow_all.id}" + instance_profile = "${aws_iam_instance_profile.emr_profile.arn}" + } + instance_fleet{ + instance_fleet_type = "MASTER" + instance_type_configs { + instance_type = "m3.xlarge" + } + + target_on_demand_capacity = 1 + } + instance_fleet { + instance_fleet_type = "CORE" + instance_type_configs{ + bid_price_as_percentage_of_on_demand_price = 100 + ebs_optimized = true + ebs_config { + size = 100 + type = "gp2" + volumes_per_instance = 1 + } + + instance_type = "m3.xlarge" + weighted_capacity = 1 + } + + launch_specifications { + spot_specification { + block_duration_minutes = 60 + timeout_action = "SWITCH_TO_ON_DEMAND" + timeout_duration_minutes = 5 + } + } + name = "instance-fleet-test" + target_on_demand_capacity = 0 + target_spot_capacity = 1 + } + tags = { + role = "rolename" + dns_zone = "env_zone" + env = "env" + name = "name-env" + } + bootstrap_action { + path = "s3://elasticmapreduce/bootstrap-actions/run-if" + name = "runif" + args = ["instance.isMaster=true", "echo running on master node"] + } + configurations = "test-fixtures/emr_configurations.json" + service_role = "${aws_iam_role.iam_emr_default_role.arn}" + depends_on = ["aws_internet_gateway.gw"] +} +resource "aws_security_group" "allow_all" { + name = "allow_all" + description = "Allow all inbound traffic" + vpc_id = "${aws_vpc.main.id}" + ingress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + depends_on = ["aws_subnet.main"] + lifecycle { + ignore_changes = ["ingress", "egress"] + } +} +resource "aws_vpc" "main" { + cidr_block = "172.32.0.0/16" + enable_dns_hostnames = true + +} +resource "aws_subnet" "main" { + vpc_id = "${aws_vpc.main.id}" + cidr_block = "172.32.0.0/20" + map_public_ip_on_launch = true +} +resource "aws_internet_gateway" "gw" { + vpc_id = "${aws_vpc.main.id}" +} +resource "aws_route_table" "r" { + vpc_id = "${aws_vpc.main.id}" + route { + cidr_block = "0.0.0.0/0" + gateway_id = "${aws_internet_gateway.gw.id}" + } +} +resource "aws_main_route_table_association" "a" { + vpc_id = "${aws_vpc.main.id}" + route_table_id = "${aws_route_table.r.id}" +} +### +# IAM role for EMR Service +resource "aws_iam_role" "iam_emr_default_role" { + name = "iam_emr_default_role_%d" + assume_role_policy = <aws_emr_instance_group +
  • + aws_emr_instance_fleet +
    • +
  • aws_emr_security_configuration
    • diff --git a/website/docs/r/emr_cluster.html.markdown b/website/docs/r/emr_cluster.html.markdown index a647c014ffc..111ecd26fd0 100644 --- a/website/docs/r/emr_cluster.html.markdown +++ b/website/docs/r/emr_cluster.html.markdown @@ -248,12 +248,15 @@ Attributes for the Amazon EC2 instances running the job flow * `key_name` - (Optional) Amazon EC2 key pair that can be used to ssh to the master node as the user called `hadoop` * `subnet_id` - (Optional) VPC subnet id where you want the job flow to launch. Cannot specify the `cc1.4xlarge` instance type for nodes of a job flow launched in a Amazon VPC +* `subnet_ids` - (Optional) Applies to clusters that use the instance fleet configuration. When multiple EC2 subnet IDs are specified, Amazon EMR evaluates them and launches instances in the optimal subnet. * `additional_master_security_groups` - (Optional) String containing a comma separated list of additional Amazon EC2 security group IDs for the master node * `additional_slave_security_groups` - (Optional) String containing a comma separated list of additional Amazon EC2 security group IDs for the slave nodes as a comma separated string * `emr_managed_master_security_group` - (Optional) Identifier of the Amazon EC2 EMR-Managed security group for the master node * `emr_managed_slave_security_group` - (Optional) Identifier of the Amazon EC2 EMR-Managed security group for the slave nodes * `service_access_security_group` - (Optional) Identifier of the Amazon EC2 service-access security group - required when the cluster runs on a private subnet * `instance_profile` - (Required) Instance Profile for EC2 instances of the cluster assume this role +* `availability_zone` - (Optional) The Amazon EC2 Availability Zone for the cluster. `availability_zone` is used for uniform instance groups, while `availability_zones` (plural) is used for instance fleets. +* `availability_zones` - (Optional) When multiple Availability Zones are specified, Amazon EMR evaluates them and launches instances in the optimal Availability Zone. `availability_zones` is used for instance fleets, while `availability_zone` (singular) is used for uniform instance groups. ~> **NOTE on EMR-Managed security groups:** These security groups will have any missing inbound or outbound access rules added and maintained by AWS, to ensure @@ -301,6 +304,17 @@ Supported nested arguments for the `master_instance_group` configuration block: * `ebs_config` - (Optional) Configuration block(s) for EBS volumes attached to each instance in the instance group. Detailed below. * `name` - (Optional) Friendly name given to the instance group. +## instance_fleet + +Attributes for each instance fleet in the cluster (See aws_emr_instance_fleet for more details) + +* `instance_fleet_type` - (Required) The node type that the instance fleet hosts. Valid values are `MASTER`, `CORE`, and `TASK`. Changing this forces a new resource to be created. +* `instance_type_configs` - (Optional) The instance type configurations that define the EC2 instances in the instance fleet. Array of `instance_type_config` blocks. +* `launch_specifications` - (Optional) The launch specification for the instance fleet. +* `name` - (Optional) The friendly name of the instance fleet. +* `target_on_demand_capacity` - (Optional) The target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision. +* `target_spot_capacity` - (Optional) The target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision. + ## ebs_config Attributes for the EBS volumes attached to each EC2 instance in the `instance_group` diff --git a/website/docs/r/emr_instance_fleet.html.markdown b/website/docs/r/emr_instance_fleet.html.markdown new file mode 100644 index 00000000000..08de2521511 --- /dev/null +++ b/website/docs/r/emr_instance_fleet.html.markdown @@ -0,0 +1,156 @@ +--- +layout: "aws" +page_title: "AWS: aws_emr_instance_fleet" +sidebar_current: "docs-aws-resource-emr-instance-fleet" +description: |- + Provides an Elastic MapReduce Cluster Instance Fleet +--- + +# aws_emr_instance_fleet + +Provides an Elastic MapReduce Cluster Instance Fleet configuration. +See [Amazon Elastic MapReduce Documentation](https://aws.amazon.com/documentation/emr/) for more information. + +~> **NOTE:** At this time, Instance Fleets cannot be destroyed through the API nor +web interface. Instance Fleets are destroyed when the EMR Cluster is destroyed. +Terraform will resize any Instance Fleet to zero when destroying the resource. + +## Example Usage + +```hcl +resource "aws_emr_instance_fleet" "task" { + cluster_id = "${aws_emr_cluster.tf-test-cluster.id}" + instance_fleet_type = "TASK" + instance_type_configs [ + { + bid_price_as_percentage_of_on_demand_price = 100 + configurations = [] + ebs_optimized = true + ebs_config = [ + { + iops = 300 + size = 10 + type = "gp2" + volumes_per_instance = 1 + } + ] + instance_type = "m3.xlarge" + "weighted_capacity" = 8 + } + ], + launch_specifications { + spot_specification { + block_duration_minutes = 60 + timeout_action = "TERMINATE_CLUSTER" + timeout_duration_minutes = 10 + } + }, + name = "my little instance fleet" + target_on_demand_capacity = 1 + target_spot_capacity = 1 +} +``` + +## Argument Reference + +The following arguments are supported: + +* `cluster_id` - (Required) ID of the EMR Cluster to attach to. Changing this forces a new resource to be created. + +* `instance_fleet_type` - (Required) The node type that the instance fleet hosts. Valid values are `MASTER`, `CORE`, and `TASK`. Changing this forces a new resource to be created. + +* `instance_type_configs` - (Optional) The instance type configurations that define the EC2 instances in the instance fleet. List of `instance_type_config` blocks. + +* `launch_specifications` - (Optional) The launch specification for the instance fleet. + + * `spot_specification` - (Required) The launch specification for Spot instances in the fleet, which determines the + defined duration and provisioning timeout behavior. + +* `name` - (Optional) The friendly name of the instance fleet. + +* `target_on_demand_capacity` - (Optional) The target capacity of On-Demand units for the instance fleet, which determines how many On-Demand instances to provision. + +* `target_spot_capacity` - (Optional) The target capacity of Spot units for the instance fleet, which determines how many Spot instances to provision. + + + +`instance_type_config` supports the following: + +* `bid_price` - (Optional) The bid price for each EC2 Spot instance type as defined by `instance_type`. +Expressed in USD. If neither `bid_price` nor `bid_price_as_percentage_of_on_demand_price` is provided, +`bid_price_as_percentage_of_on_demand_price` defaults to 100%. + +* `bid_price_as_percentage_of_on_demand_price` - (Optional) The bid price, as a percentage of On-Demand price, +for each EC2 Spot instance as defined by `instance_type`. Expressed as a number (for example, 20 specifies 20%). +If neither `bid_price` nor `bid_price_as_percentage_of_on_demand_price` is provided, +`bid_price_as_percentage_of_on_demand_price` defaults to 100%. + +* `configurations` - (Optional) A configuration classification that applies when provisioning cluster instances, +which can include configurations for applications and software that run on the cluster. List of `configuration` blocks. + +* `ebs_optimized` - (Optional) Indicates whether an Amazon EBS volume is EBS-optimized. + +* `ebs_config` - (Optional) The configuration of Amazon Elastic Block Storage (EBS) attached to each instance as +defined by `instance_type`. + +* `instance_type` - (Required) An EC2 instance type, such as m3.xlarge. + +* `weighted_capacity` - (Optional) The number of units that a provisioned instance of this type provides toward +fulfilling the target capacities defined in `aws_emr_instance_fleet`. This value is 1 for a master instance fleet, +and must be 1 or greater for core and task instance fleets. Defaults to 1 if not specified. + + + +`spot_specification` supports the following: + +* `block_duration_minutes` - (Optional) The defined duration for Spot instances (also known as Spot blocks) in minutes. +When specified, the Spot instance does not terminate before the defined duration expires, and defined duration pricing +for Spot instances applies. Valid values are 60, 120, 180, 240, 300, or 360. The duration period starts as soon as a +Spot instance receives its instance ID. At the end of the duration, Amazon EC2 marks the Spot instance for termination +and provides a Spot instance termination notice, which gives the instance a two-minute warning before it terminates. + +* `timeout_action` - (Required) The action to take when `target_spot_capacity` has not been fulfilled when the +`timeout_duration_minutes` has expired. Spot instances are not uprovisioned within the Spot provisioining timeout. +Valid values are `TERMINATE_CLUSTER` and `SWITCH_TO_ON_DEMAND`. `SWITCH_TO_ON_DEMAND` specifies that if no Spot +instances are available, On-Demand Instances should be provisioned to fulfill any remaining Spot capacity. + +* `timeout_duration_minutes` - (Required) The spot provisioning timeout period in minutes. If Spot instances are not +provisioned within this time period, the `timeout_action` is taken. Minimum value is 5 and maximum value is 1440. +The timeout applies only during initial provisioning, when the cluster is first created. + + + +`configuration` supports the following: + +* `classification` - (Optional) The classification within a configuration. + +* `configurations` - (Optional) A list of additional configurations to apply within a configuration object. + +* `properties` - (Optional) A set of properties specified within a configuration classification. + + + +`ebs_config` EBS volume specifications such as volume type, IOPS, and size (GiB) that will be requested for the EBS volume attached to an EC2 instance in the cluster. + +* `iops` - (Optional) The number of I/O operations per second (IOPS) that the volume supports. + +* `size_in_gb` - (Required) The volume size, in gibibytes (GiB). This can be a number from 1 - 1024. If the volume type is EBS-optimized, the minimum value is 10. + +* `volume_type` - (Required) The volume type. Volume types supported are `gp2`, `io1`, `standard`. + +* `volumes_per_instance` - (Optional) Number of EBS volumes with a specific volume configuration that will be associated with every instance in the instance group + + +## Attributes Reference + +The following attributes are exported: + +* `id` - The unique identifier of the instance fleet. + +* `provisioned_on_demand_capacity` The number of On-Demand units that have been provisioned for the instance +fleet to fulfill TargetOnDemandCapacity. This provisioned capacity might be less than or greater than TargetOnDemandCapacity. + +* `provisioned_spot_capacity` The number of Spot units that have been provisioned for this instance fleet +to fulfill TargetSpotCapacity. This provisioned capacity might be less than or greater than TargetSpotCapacity. + +* `status` The current status of the instance fleet. From b575a0734f8244234397020098a81a3da45bdc9b Mon Sep 17 00:00:00 2001 From: Sanooj Date: Tue, 16 Jul 2019 12:38:26 +0530 Subject: [PATCH 2/2] Fixed lint issues --- aws/resource_aws_emr_instance_fleet.go | 6 ++-- aws/validators.go | 47 -------------------------- 2 files changed, 2 insertions(+), 51 deletions(-) diff --git a/aws/resource_aws_emr_instance_fleet.go b/aws/resource_aws_emr_instance_fleet.go index 6d86d921c7d..12f033dbe20 100644 --- a/aws/resource_aws_emr_instance_fleet.go +++ b/aws/resource_aws_emr_instance_fleet.go @@ -84,7 +84,6 @@ func instanceTypeConfigSchema() *schema.Resource { "bid_price": { Type: schema.TypeString, Optional: true, - Required: false, ForceNew: true, }, "bid_price_as_percentage_of_on_demand_price": { @@ -481,9 +480,8 @@ func fetchAllEMRInstanceFleets(conn *emr.EMR, clusterId string) ([]*emr.Instance } if respFleets.InstanceFleets != nil { - for _, f := range respFleets.InstanceFleets { - fleets = append(fleets, f) - } + fleets = append(fleets, respFleets.InstanceFleets...) + } else { log.Printf("[DEBUG] EMR Instance Fleet list was empty") } diff --git a/aws/validators.go b/aws/validators.go index 4e57dd1f15a..03117d85c06 100644 --- a/aws/validators.go +++ b/aws/validators.go @@ -2405,50 +2405,3 @@ func validateAwsEmrInstanceFleetType(v interface{}, k string) (ws []string, erro } return } - -func validateAwsEmrRepoUpgradeOnBootSecurity(v interface{}, k string) (ws []string, errors []error) { - validTypes := map[string]struct{}{ - emr.RepoUpgradeOnBootSecurity: {}, - emr.RepoUpgradeOnBootNone: {}, - } - - value := v.(string) - - if _, ok := validTypes[value]; !ok { - errors = append(errors, fmt.Errorf( - "%q must be one of [%q, %q]", k, emr.RepoUpgradeOnBootSecurity, emr.RepoUpgradeOnBootNone)) - } - return -} - -func validateAwsEmrActionOnFailure(v interface{}, k string) (ws []string, errors []error) { - validTypes := map[string]struct{}{ - emr.ActionOnFailureTerminateJobFlow: {}, - emr.ActionOnFailureTerminateCluster: {}, - emr.ActionOnFailureCancelAndWait: {}, - emr.ActionOnFailureContinue: {}, - } - - value := v.(string) - - if _, ok := validTypes[value]; !ok { - errors = append(errors, fmt.Errorf( - "%q must be one of [%q, %q, %q, %q]", k, emr.ActionOnFailureTerminateJobFlow, emr.ActionOnFailureTerminateCluster, emr.ActionOnFailureCancelAndWait, emr.ActionOnFailureContinue)) - } - return -} - -func validateAwsEmrScaleDownBehaviour(v interface{}, k string) (ws []string, errors []error) { - validTypes := map[string]struct{}{ - emr.ScaleDownBehaviorTerminateAtInstanceHour: {}, - emr.ScaleDownBehaviorTerminateAtTaskCompletion: {}, - } - - value := v.(string) - - if _, ok := validTypes[value]; !ok { - errors = append(errors, fmt.Errorf( - "%q must be one of [%q, %q]", k, emr.ScaleDownBehaviorTerminateAtInstanceHour, emr.ScaleDownBehaviorTerminateAtTaskCompletion)) - } - return -}