From 6659ee8a4e761321029f30eb52f047649b8da9f5 Mon Sep 17 00:00:00 2001 From: Surya Vadan Adivikolanu Date: Tue, 9 Dec 2014 17:52:17 -0800 Subject: [PATCH 1/3] [EMR] Added support for Security Groups. --- awscli/customizations/emr/applicationutils.py | 2 +- awscli/customizations/emr/argumentschema.py | 23 +++++- awscli/customizations/emr/createcluster.py | 16 ++++ awscli/customizations/emr/describecluster.py | 2 +- awscli/customizations/emr/helptext.py | 36 ++++++++- .../examples/emr/create-cluster-examples.rst | 35 +++++++- tests/unit/customizations/emr/__init__.py | 3 +- ...t_ec2_attributes_with_security_groups.json | 6 ++ .../emr/test_add_instance_groups.py | 3 +- .../customizations/emr/test_create_cluster.py | 79 +++++++++++++++++++ 10 files changed, 194 insertions(+), 11 deletions(-) create mode 100644 tests/unit/customizations/emr/input_ec2_attributes_with_security_groups.json diff --git a/awscli/customizations/emr/applicationutils.py b/awscli/customizations/emr/applicationutils.py index 9b12ce881efc..f4bf9840b4dc 100644 --- a/awscli/customizations/emr/applicationutils.py +++ b/awscli/customizations/emr/applicationutils.py @@ -22,7 +22,7 @@ def build_applications(session, step_list = [] ba_list = [] region = parsed_globals.region if parsed_globals.region \ - else session.get_config_variable('region') + else session.get_config_variable('region') for app_config in parsed_applications: app_name = app_config['Name'].lower() diff --git a/awscli/customizations/emr/argumentschema.py b/awscli/customizations/emr/argumentschema.py index 3493586373b0..2f470721547f 100644 --- a/awscli/customizations/emr/argumentschema.py +++ b/awscli/customizations/emr/argumentschema.py @@ -87,8 +87,29 @@ EC2_ROLE_NAME + ". In order to use the default" " role, you must have already created it using the " "create-default-roles command. " + }, + "EmrManagedMasterSecurityGroup": { + "type": "string", + "description": helptext.EMR_MANAGED_MASTER_SECURITY_GROUP + }, + "EmrManagedSlaveSecurityGroup": { + "type": "string", + "description": helptext.EMR_MANAGED_SLAVE_SECURITY_GROUP + }, + "AdditionalMasterSecurityGroups": { + "type": "array", + "description": helptext.ADDITIONAL_MASTER_SECURITY_GROUPS, + "items": { + "type": "string" + } + }, + "AdditionalSlaveSecurityGroups": { + "type": "array", + "description": helptext.ADDITIONAL_SLAVE_SECURITY_GROUPS, + "items": { + "type": "string" + } } - } } diff --git a/awscli/customizations/emr/createcluster.py b/awscli/customizations/emr/createcluster.py index 2fe1af9bbaa1..530e7b69ea0c 100644 --- a/awscli/customizations/emr/createcluster.py +++ b/awscli/customizations/emr/createcluster.py @@ -302,6 +302,22 @@ def _build_ec2_attributes(self, cluster, parsed_attrs): src_params=parsed_attrs, src_key='InstanceProfile', dest_params=cluster, dest_key='JobFlowRole') + emrutils.apply_params( + src_params=parsed_attrs, src_key='EmrManagedMasterSecurityGroup', + dest_params=instances, dest_key='EmrManagedMasterSecurityGroup') + + emrutils.apply_params( + src_params=parsed_attrs, src_key='EmrManagedSlaveSecurityGroup', + dest_params=instances, dest_key='EmrManagedSlaveSecurityGroup') + + emrutils.apply_params( + src_params=parsed_attrs, src_key='AdditionalMasterSecurityGroups', + dest_params=instances, dest_key='AdditionalMasterSecurityGroups') + + emrutils.apply_params( + src_params=parsed_attrs, src_key='AdditionalSlaveSecurityGroups', + dest_params=instances, dest_key='AdditionalSlaveSecurityGroups') + emrutils.apply(params=cluster, key='Instances', value=instances) return cluster diff --git a/awscli/customizations/emr/describecluster.py b/awscli/customizations/emr/describecluster.py index ada3e0a45c9b..6c9e4b53c0b8 100644 --- a/awscli/customizations/emr/describecluster.py +++ b/awscli/customizations/emr/describecluster.py @@ -77,7 +77,7 @@ def _call(self, operation_object, parameters, parsed_globals): endpoint_url=parsed_globals.endpoint_url, verify=parsed_globals.verify_ssl) http_response, response_data = operation_object.call(endpoint, - **parameters) + **parameters) return response_data def _get_key_of_result(self, keys): diff --git a/awscli/customizations/emr/helptext.py b/awscli/customizations/emr/helptext.py index d69954e05c48..392a954e9e1c 100644 --- a/awscli/customizations/emr/helptext.py +++ b/awscli/customizations/emr/helptext.py @@ -99,9 +99,12 @@ EC2_ATTRIBUTES = ( '

Specifies the following Amazon EC2 attributes: KeyName,' - ' AvailabilityZone, SubnetId, and InstanceProfile. AvailabilityZone and ' - 'Subnet cannot be specified together. To create the default ' - 'instance profile ' + EC2_ROLE_NAME + ',' + ' AvailabilityZone, SubnetId, InstanceProfile,' + ' EmrManagedMasterSecurityGroup, EmrManagedSlaveSecurityGroup,' + ' AdditionalMasterSecurityGroups and AdditionalSlaveSecurityGroups.' + ' AvailabilityZone and Subnet cannot be specified together.' + ' To create the default instance profile ' + + EC2_ROLE_NAME + ',' ' use aws emr create-default-roles command.

' 'This command will also create the default EMR service role ' '' + EMR_ROLE_NAME + '.' @@ -112,7 +115,16 @@ '
  • SubnetId- Assign the EMR cluster to this Amazon VPC Subnet.
    • ' '
  • InstanceProfile - Provides access to other AWS services such as S3,' ' DynamoDB from EC2 instances that are launched by EMR..
    • ' - ) + '
  • EmrManagedMasterSecurityGroup - The identifier of the Amazon EC2' + ' security group (managed by Amazon Elastic MapReduce)' + ' for the master node.
    • ' + '
  • EmrManagedSlaveSecurityGroup - The identifier of the Amazon EC2' + ' security group (managed by Amazon Elastic MapReduce)' + ' for the slave nodes.
    • ' + '
  • AdditionalMasterSecurityGroups - A list of additional Amazon EC2' + ' security group IDs for the master node
    • ' + '
  • AdditionalSlaveSecurityGroups - A list of additional Amazon EC2' + ' security group IDs for the slave nodes.
    • ') AUTO_TERMINATE = ( '

    Specifies whether the cluster should terminate after' @@ -227,3 +239,19 @@ LIST_CLUSTERS_CREATED_BEFORE = ( '

    The creation date and time end value filter for ' 'listing clusters. For example, 2014-07-15T00:01:30.

    ') + +EMR_MANAGED_MASTER_SECURITY_GROUP = ( + '

    The identifier of the Amazon EC2 security group (managed by Amazon ' + 'Elastic MapReduce) for the master node.

    ') + +EMR_MANAGED_SLAVE_SECURITY_GROUP = ( + '

    The identifier of the Amazon EC2 security group (managed by Amazon ' + 'Elastic MapReduce) for the slave nodes.

    ') + +ADDITIONAL_MASTER_SECURITY_GROUPS = ( + '

    A list of additional Amazon EC2 security group IDs for ' + 'the master node

    ') + +ADDITIONAL_SLAVE_SECURITY_GROUPS = ( + '

    A list of additional Amazon EC2 security group IDs for ' + 'the slave nodes.

    ') diff --git a/awscli/examples/emr/create-cluster-examples.rst b/awscli/examples/emr/create-cluster-examples.rst index f938bc9519aa..45d3bad704d0 100644 --- a/awscli/examples/emr/create-cluster-examples.rst +++ b/awscli/examples/emr/create-cluster-examples.rst @@ -40,7 +40,39 @@ - Create an Amazon EMR cluster in an AvailabilityZone. For example, us-east-1b:: - aws emr create-cluster --ec2-attributes AvailabilityZone=us-east-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge + aws emr create-cluster --ec2-attributes AvailabilityZone=us-east-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge + +- Create an Amazon EMR cluster specifying the Amazon EC2 security groups:: + + aws emr create-cluster --ami-version 3.3.1 --service-role EMR_DefaultRole --ec2-attributes InstanceProfile=myRole,EmrManagedMasterSecurityGroup=sg-master1,EmrManagedSlaveSecurityGroup=sg-slave1,AdditionalMasterSecurityGroups=[sg-addMaster1,sg-addMaster2,sg-addMaster3,sg-addMaster4],AdditionalSlaveSecurityGroups=[sg-addSlave1,sg-addSlave2,sg-addSlave3,sg-addSlave4] --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge + +- Create an Amazon EMR cluster specifying only the EMR managed Amazon EC2 security groups:: + + aws emr create-cluster --ami-version 3.3.1 --service-role EMR_DefaultRole --ec2-attributes InstanceProfile=myRole,EmrManagedMasterSecurityGroup=sg-master1,EmrManagedSlaveSecurityGroup=sg-slave1 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge + +- Create an Amazon EMR cluster specifying only the additional Amazon EC2 security groups:: + + aws emr create-cluster --ami-version 3.3.1 --service-role EMR_DefaultRole --ec2-attributes InstanceProfile=myRole,AdditionalMasterSecurityGroups=[sg-addMaster1,sg-addMaster2,sg-addMaster3,sg-addMaster4],AdditionalSlaveSecurityGroups=[sg-addSlave1,sg-addSlave2,sg-addSlave3,sg-addSlave4] --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge + +- JSON equivalent (contents of ec2_attributes.json):: + + [ + { + "SubnetId": "subnet-xxxxx", + "KeyName": "myKey", + "InstanceProfile":"myRole", + "EmrManagedMasterSecurityGroup": "sg-master1", + "EmrManagedSlaveSecurityGroup": "sg-slave1", + "AdditionalMasterSecurityGroups": ["sg-addMaster1","sg-addMaster2","sg-addMaster3","sg-addMaster4"], + "AdditionalSlaveSecurityGroups": ["sg-addSlave1","sg-addSlave2","sg-addSlave3","sg-addSlave4"] + } + ] + +NOTE: JSON arguments must include options and values as their own items in the list. + +- Command (using ec2_attributes.json):: + + aws emr create-cluster --ami-version 3.3.1 --service-role EMR_DefaultRole --ec2-attributes file://./ec2_attributes.json --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge **7. Enable debugging and specify a Log URI** @@ -211,4 +243,3 @@ NOTE: JSON arguments must include options and values as their own items in the l - Command:: aws emr create-cluster --instance-type m3.xlarge --ami-version 3.2.1 --emrfs SSE=true,Consistent=true,RetryCount=5,RetryPeriod=30,Args=[fs.s3.serverSideEncryptionAlgorithm=AES256] - diff --git a/tests/unit/customizations/emr/__init__.py b/tests/unit/customizations/emr/__init__.py index 8f856d2d84cf..2a62aacea549 100644 --- a/tests/unit/customizations/emr/__init__.py +++ b/tests/unit/customizations/emr/__init__.py @@ -24,4 +24,5 @@ class EMRBaseAWSCommandParamsTest(BaseAWSCommandParamsTest): def setUp(self): super(EMRBaseAWSCommandParamsTest, self).setUp() # We actually will just disable preview mode completely. - self.driver.session.unregister('building-command-table.main', mark_as_preview) + self.driver.session.unregister('building-command-table.main', + mark_as_preview) diff --git a/tests/unit/customizations/emr/input_ec2_attributes_with_security_groups.json b/tests/unit/customizations/emr/input_ec2_attributes_with_security_groups.json new file mode 100644 index 000000000000..41ab65f82138 --- /dev/null +++ b/tests/unit/customizations/emr/input_ec2_attributes_with_security_groups.json @@ -0,0 +1,6 @@ +{ + "EmrManagedMasterSecurityGroup": "sg-master1", + "EmrManagedSlaveSecurityGroup": "sg-slave1", + "AdditionalMasterSecurityGroups": ["sg-addMaster1","sg-addMaster2","sg-addMaster3","sg-addMaster4"], + "AdditionalSlaveSecurityGroups": ["sg-addSlave1","sg-addSlave2","sg-addSlave3","sg-addSlave4"] +} \ No newline at end of file diff --git a/tests/unit/customizations/emr/test_add_instance_groups.py b/tests/unit/customizations/emr/test_add_instance_groups.py index dc15fa7da214..f3c630af0b2c 100644 --- a/tests/unit/customizations/emr/test_add_instance_groups.py +++ b/tests/unit/customizations/emr/test_add_instance_groups.py @@ -61,7 +61,8 @@ def test_instance_groups_missing_instance_group_type_error(self): cmd = self.prefix + ' Name=Task,InstanceType=m1.small,' +\ 'InstanceCount=5' result = self.run_cmd(cmd, 255) - self.assert_error_message_has_field_name(result[1], 'InstanceGroupType') + self.assert_error_message_has_field_name(result[1], + 'InstanceGroupType') def test_instance_groups_missing_instance_type_error(self): cmd = self.prefix + ' Name=Task,InstanceGroupType=Task,' +\ diff --git a/tests/unit/customizations/emr/test_create_cluster.py b/tests/unit/customizations/emr/test_create_cluster.py index 504a3333ee12..afcfbcdb4e2c 100644 --- a/tests/unit/customizations/emr/test_create_cluster.py +++ b/tests/unit/customizations/emr/test_create_cluster.py @@ -322,6 +322,16 @@ 'Tags': [] } +EMR_MANAGED_MASTER_SECURITY_GROUP = 'sg-master1' + +EMR_MANAGED_SLAVE_SECURITY_GROUP = 'sg-slave1' + +ADDITIONAL_MASTER_SECURITY_GROUPS = \ + ['sg-addMaster1', 'sg-addMaster2', 'sg-addMaster3', 'sg-addMaster4'] + +ADDITIONAL_SLAVE_SECURITY_GROUPS = \ + ['sg-addSlave1', 'sg-addSlave2', 'sg-addSlave3', 'sg-addSlave4'] + class TestCreateCluster(BaseAWSCommandParamsTest): prefix = 'emr create-cluster ' @@ -1250,5 +1260,74 @@ def test_emr_fs_config(self): cmd = DEFAULT_CMD + '--emrfs file://' + data_path self.assert_params_for_cmd2(cmd, result) + def test_all_security_groups(self): + cmd = DEFAULT_CMD + ( + '--ec2-attributes EmrManagedMasterSecurityGroup=sg-master1,' + 'EmrManagedSlaveSecurityGroup=sg-slave1,AdditionalMasterSecu' + 'rityGroups=[sg-addMaster1,sg-addMaster2,sg-addMaster3,' + 'sg-addMaster4],AdditionalSlaveSecurityGroups=[sg-addSlave1,' + 'sg-addSlave2,sg-addSlave3,sg-addSlave4]') + + result = copy.deepcopy(DEFAULT_RESULT) + instances = result['Instances'] + instances['EmrManagedMasterSecurityGroup'] = \ + EMR_MANAGED_MASTER_SECURITY_GROUP + instances['EmrManagedSlaveSecurityGroup'] = \ + EMR_MANAGED_SLAVE_SECURITY_GROUP + instances['AdditionalMasterSecurityGroups'] = \ + ADDITIONAL_MASTER_SECURITY_GROUPS + instances['AdditionalSlaveSecurityGroups'] = \ + ADDITIONAL_SLAVE_SECURITY_GROUPS + + self.assert_params_for_cmd2(cmd, result) + + def test_emr_managed_security_groups(self): + cmd = DEFAULT_CMD + ( + '--ec2-attributes EmrManagedMasterSecurityGroup=sg-master1,' + 'EmrManagedSlaveSecurityGroup=sg-slave1') + + result = copy.deepcopy(DEFAULT_RESULT) + instances = result['Instances'] + instances['EmrManagedMasterSecurityGroup'] = \ + EMR_MANAGED_MASTER_SECURITY_GROUP + instances['EmrManagedSlaveSecurityGroup'] = \ + EMR_MANAGED_SLAVE_SECURITY_GROUP + + self.assert_params_for_cmd2(cmd, result) + + def test_additional_security_groups(self): + cmd = DEFAULT_CMD + ( + '--ec2-attributes AdditionalMasterSecurityGroups=[sg-addMaster1' + ',sg-addMaster2,sg-addMaster3,sg-addMaster4],AdditionalSlaveSecu' + 'rityGroups=[sg-addSlave1,sg-addSlave2,sg-addSlave3,sg-addSlave4]') + + result = copy.deepcopy(DEFAULT_RESULT) + instances = result['Instances'] + instances['AdditionalMasterSecurityGroups'] = \ + ADDITIONAL_MASTER_SECURITY_GROUPS + instances['AdditionalSlaveSecurityGroups'] = \ + ADDITIONAL_SLAVE_SECURITY_GROUPS + + self.assert_params_for_cmd2(cmd, result) + + def test_security_groups_from_json_file(self): + data_path = os.path.join( + os.path.dirname(__file__), + 'input_ec2_attributes_with_security_groups.json') + cmd = DEFAULT_CMD + '--ec2-attributes file://' + data_path + + result = copy.deepcopy(DEFAULT_RESULT) + instances = result['Instances'] + instances['EmrManagedMasterSecurityGroup'] = \ + EMR_MANAGED_MASTER_SECURITY_GROUP + instances['EmrManagedSlaveSecurityGroup'] = \ + EMR_MANAGED_SLAVE_SECURITY_GROUP + instances['AdditionalMasterSecurityGroups'] = \ + ADDITIONAL_MASTER_SECURITY_GROUPS + instances['AdditionalSlaveSecurityGroups'] = \ + ADDITIONAL_SLAVE_SECURITY_GROUPS + + self.assert_params_for_cmd2(cmd, result) + if __name__ == "__main__": unittest.main() From b843f3907661b422a8f1cb546ad558605f0301dd Mon Sep 17 00:00:00 2001 From: kyleknap Date: Tue, 20 Jan 2015 14:53:35 -0800 Subject: [PATCH 2/3] Update Changlog --- CHANGELOG.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1fb380e61bfb..9e14b504d26a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,14 @@ CHANGELOG ========= +Next Release (TBD) +================== + +* feature:``aws emr``: Add support for security groups. +* feature:``aws cognitio-identity``: Enhance authentication flow by being able + to save associations of IAM roles with identity pools. + + 1.7.2 ===== From 45ab73f0c365fcfae7464dd8c27314e2136d034e Mon Sep 17 00:00:00 2001 From: AWS Date: Tue, 20 Jan 2015 14:56:26 -0800 Subject: [PATCH 3/3] Bumping version to 1.7.3 --- CHANGELOG.rst | 4 ++-- awscli/__init__.py | 2 +- doc/source/conf.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9e14b504d26a..ca7da716b446 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,8 +2,8 @@ CHANGELOG ========= -Next Release (TBD) -================== +1.7.3 +===== * feature:``aws emr``: Add support for security groups. * feature:``aws cognitio-identity``: Enhance authentication flow by being able diff --git a/awscli/__init__.py b/awscli/__init__.py index 7fec9656df12..654c6c6ccb28 100644 --- a/awscli/__init__.py +++ b/awscli/__init__.py @@ -17,7 +17,7 @@ """ import os -__version__ = '1.7.2' +__version__ = '1.7.3' # # Get our data path to be added to botocore's search path diff --git a/doc/source/conf.py b/doc/source/conf.py index 2fc046cc8f8c..642ec1b1a143 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -52,7 +52,7 @@ # The short X.Y version. version = '1.7' # The full version, including alpha/beta/rc tags. -release = '1.7.2' +release = '1.7.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index ef34a8c13885..753a2a689d12 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import awscli -requires = ['botocore>=0.83.0,<0.84.0', +requires = ['botocore>=0.84.0,<0.85.0', 'bcdoc>=0.12.0,<0.13.0', 'colorama==0.2.5', 'docutils>=0.10',