From 2329a23b937ff0ca27aee390793d8680f4abda5e Mon Sep 17 00:00:00 2001 From: Shuo Diao Date: Mon, 16 Jun 2014 14:38:11 -0700 Subject: [PATCH 1/6] EMR: Bug fix in install-applications and create-cluster 1. Rename --apps to --applications in install-applications 2. Fixed bug in create-cluster --applications Name=MapR where Args is required. --- awscli/customizations/emr/applicationutils.py | 2 +- awscli/customizations/emr/installapplications.py | 6 +++--- .../unit/customizations/emr/test_create_cluster.py | 11 +++++++++++ .../emr/test_install_applications.py | 14 ++++++++------ 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/awscli/customizations/emr/applicationutils.py b/awscli/customizations/emr/applicationutils.py index d6b8b40460ba..1ceb0c06ded3 100644 --- a/awscli/customizations/emr/applicationutils.py +++ b/awscli/customizations/emr/applicationutils.py @@ -27,7 +27,7 @@ def build_applications(parsed_applications, parsed_globals, ami_version=None): if app_name in constants.MAPR_NAMES: app_list.append( build_supported_product( - app_config['Name'], app_config['Args'])) + app_config['Name'], app_config.get('Args'))) elif app_name == constants.HIVE: hive_version = app_config.get('Version') if hive_version is None: diff --git a/awscli/customizations/emr/installapplications.py b/awscli/customizations/emr/installapplications.py index 504895804da7..5eaf53119a91 100644 --- a/awscli/customizations/emr/installapplications.py +++ b/awscli/customizations/emr/installapplications.py @@ -27,7 +27,7 @@ class InstallApplications(BasicCommand): ARG_TABLE = [ {'name': 'cluster-id', 'required': True, 'help_text': helptext.CLUSTER_ID}, - {'name': 'apps', 'required': True, + {'name': 'applications', 'required': True, 'help_text': helptext.INSTALL_APPLICATIONS, 'schema': argumentschema.APPLICATIONS_SCHEMA}, ] @@ -38,9 +38,9 @@ def _run_main(self, parsed_args, parsed_globals): parameters = {'JobFlowId': parsed_args.cluster_id} - self._check_for_supported_apps(parsed_args.apps) + self._check_for_supported_apps(parsed_args.applications) parameters['Steps'] = applicationutils.build_applications( - parsed_args.apps, parsed_globals)[2] + parsed_args.applications, parsed_globals)[2] emrutils.call_and_display_response(self._session, 'AddJobFlowSteps', parameters, parsed_globals) diff --git a/tests/unit/customizations/emr/test_create_cluster.py b/tests/unit/customizations/emr/test_create_cluster.py index 46259ef46116..11899bad9fbc 100644 --- a/tests/unit/customizations/emr/test_create_cluster.py +++ b/tests/unit/customizations/emr/test_create_cluster.py @@ -698,6 +698,17 @@ def test_install_mapr_with_args(self): result['NewSupportedProducts'] = [INSTALL_MAPR_PRODUCT] self.assert_params_for_cmd(cmd, result) + def test_install_mapr_without_args(self): + cmd = DEFAULT_CMD + \ + '--applications Name=mapr' + result = copy.deepcopy(DEFAULT_RESULT) + result['NewSupportedProducts'] = \ + [ + {'Name': 'mapr', + 'Args': []} + ] + self.assert_params_for_cmd(cmd, result) + def test_applications_all_types(self): cmd = DEFAULT_CMD + ( '--applications ' diff --git a/tests/unit/customizations/emr/test_install_applications.py b/tests/unit/customizations/emr/test_install_applications.py index aeb79eb7d1bd..9029627a2f6d 100644 --- a/tests/unit/customizations/emr/test_install_applications.py +++ b/tests/unit/customizations/emr/test_install_applications.py @@ -42,10 +42,10 @@ class TestInstallApplications(BaseAWSCommandParamsTest): - prefix = 'emr install-applications --cluster-id j-ABC123456' + prefix = 'emr install-applications --cluster-id j-ABC123456' def test_intall_hive_with_version(self): - cmdline = self.prefix + ' --apps Name=Hive,Version=0.8.1.8' + cmdline = self.prefix + ' --applications Name=Hive,Version=0.8.1.8' step = copy.deepcopy(INSTALL_HIVE_STEP) step['HadoopJarStep']['Args'][5] = '0.8.1.8' @@ -54,7 +54,7 @@ def test_intall_hive_with_version(self): self.assert_params_for_cmd(cmdline, result) def test_intall_pig_with_version(self): - cmdline = self.prefix + ' --apps Name=Pig,Version=0.9.2.1' + cmdline = self.prefix + ' --applications Name=Pig,Version=0.9.2.1' step = copy.deepcopy(INSTALL_PIG_STEP) step['HadoopJarStep']['Args'][5] = '0.9.2.1' @@ -63,14 +63,15 @@ def test_intall_pig_with_version(self): self.assert_params_for_cmd(cmdline, result) def test_intall_hive_and_pig_without_version(self): - cmdline = self.prefix + ' --cluster-id j-ABC123456 --apps Name=Hive' +\ + cmdline = self.prefix + ' --cluster-id j-ABC123456 --applications Name=Hive' +\ ' Name=Pig' result = {'JobFlowId': 'j-ABC123456', 'Steps': [INSTALL_HIVE_STEP, INSTALL_PIG_STEP]} self.assert_params_for_cmd(cmdline, result) def test_install_impala_error(self): - cmdline = self.prefix + ' --cluster-id j-ABC123456 --apps Name=Impala' + cmdline = self.prefix + \ + ' --cluster-id j-ABC123456 --applications Name=Impala' expected_error_msg = "\naws: error: Impala cannot be installed on" +\ " a running cluster. 'Name' should be one of the following:" +\ @@ -79,7 +80,8 @@ def test_install_impala_error(self): self.assertEqual(result[1], expected_error_msg) def test_install_unknown_app_error(self): - cmdline = self.prefix + ' --cluster-id j-ABC123456 --apps Name=unknown' + cmdline = self.prefix + \ + ' --cluster-id j-ABC123456 --applications Name=unknown' expected_error_msg = "\naws: error: Unknown application: unknown." +\ " 'Name' should be one of the following: HIVE, PIG, HBASE," +\ From c911ef55a51ceeafa5daf836b37538ecc9758044 Mon Sep 17 00:00:00 2001 From: Liang Date: Mon, 16 Jun 2014 14:57:10 -0700 Subject: [PATCH 2/6] EMR: Add unit tests for application --- awscli/customizations/emr/applicationutils.py | 2 +- awscli/customizations/emr/constants.py | 2 +- .../customizations/emr/test_create_cluster.py | 20 +++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/awscli/customizations/emr/applicationutils.py b/awscli/customizations/emr/applicationutils.py index 1ceb0c06ded3..ce5c87ce96d5 100644 --- a/awscli/customizations/emr/applicationutils.py +++ b/awscli/customizations/emr/applicationutils.py @@ -24,7 +24,7 @@ def build_applications(parsed_applications, parsed_globals, ami_version=None): for app_config in parsed_applications: app_name = app_config['Name'].lower() - if app_name in constants.MAPR_NAMES: + if app_name in constants.SUPPORTED_PRODUCTS: app_list.append( build_supported_product( app_config['Name'], app_config.get('Args'))) diff --git a/awscli/customizations/emr/constants.py b/awscli/customizations/emr/constants.py index 4d13239b0761..bb1709fcc8c0 100644 --- a/awscli/customizations/emr/constants.py +++ b/awscli/customizations/emr/constants.py @@ -27,7 +27,7 @@ SCRIPT_RUNNER_PATH = '/libs/script-runner/script-runner.jar' DEBUGGING_PATH = '/libs/state-pusher/0.1/fetch' DEBUGGING_NAME = 'Setup Hadoop Debugging' -MAPR_NAMES = ['mapr', 'mapr-m3', 'mapr-m5', 'mapr-m7'] +SUPPORTED_PRODUCTS = ['mapr', 'mapr-m3', 'mapr-m5', 'mapr-m7', 'hue'] MAX_BOOTSTRAP_ACTION_NUMBER = 16 BOOTSTRAP_ACTION_NAME = 'Bootstrap action' diff --git a/tests/unit/customizations/emr/test_create_cluster.py b/tests/unit/customizations/emr/test_create_cluster.py index 11899bad9fbc..6d836c85c5af 100644 --- a/tests/unit/customizations/emr/test_create_cluster.py +++ b/tests/unit/customizations/emr/test_create_cluster.py @@ -146,6 +146,17 @@ 'Args': ['--edition', 'm5', '--version', '3.0.2'] } +INSTALL_SUPPORTED_PRODUCTS = [ + { + 'Name': 'hue', + 'Args': ['--hue-config', 's3://elasticmapreduce/hue-config'] + }, + { + 'Name': 'mapr', + 'Args': ['--edition', 'm7'] + } +] + CUSTOM_JAR_STEP = { 'Name': 'Custom JAR', 'ActionOnFailure': 'CONTINUE', @@ -709,6 +720,15 @@ def test_install_mapr_without_args(self): ] self.assert_params_for_cmd(cmd, result) + def test_supported_products(self): + cmd = DEFAULT_CMD + ( + '--applications ' + 'Name=hue,Args=--hue-config,s3://elasticmapreduce/hue-config ' + 'Name=mapr,Args=--edition,m7') + result = copy.deepcopy(DEFAULT_RESULT) + result['NewSupportedProducts'] = INSTALL_SUPPORTED_PRODUCTS + self.assert_params_for_cmd(cmd, result) + def test_applications_all_types(self): cmd = DEFAULT_CMD + ( '--applications ' From 91ea6e04bbf88fbb45c5b469618611b8fa66d03f Mon Sep 17 00:00:00 2001 From: Mohan Gandhi Date: Fri, 13 Jun 2014 22:08:24 +0000 Subject: [PATCH 3/6] EMR: Update create-default-roles and Add --service-role & --use-default-roles --- awscli/customizations/emr/createcluster.py | 15 +- .../customizations/emr/createdefaultroles.py | 59 ++++++- awscli/customizations/emr/helptext.py | 19 ++- .../examples/emr/create-cluster-examples.rst | 38 +++-- .../examples/emr/create-cluster-synopsis.rst | 4 +- awscli/examples/emr/create-default-roles.rst | 144 ++++++++++++------ .../customizations/emr/test_create_cluster.py | 58 +++++-- .../emr/test_create_default_role.py | 35 ++++- 8 files changed, 292 insertions(+), 80 deletions(-) diff --git a/awscli/customizations/emr/createcluster.py b/awscli/customizations/emr/createcluster.py index 91f7c3b3b27b..d18564e5101b 100644 --- a/awscli/customizations/emr/createcluster.py +++ b/awscli/customizations/emr/createcluster.py @@ -22,6 +22,8 @@ from awscli.customizations.emr import exceptions from awscli.customizations.emr import applicationutils from awscli.customizations.emr import instancegroupsutils +from awscli.customizations.emr.createdefaultroles import EMR_ROLE_NAME +from awscli.customizations.emr.createdefaultroles import EC2_ROLE_NAME import re @@ -52,6 +54,10 @@ class CreateCluster(BasicCommand): 'help_text': helptext.CLUSTER_NAME}, {'name': 'log-uri', 'help_text': helptext.LOG_URI}, + {'name': 'service-role', + 'help_text': helptext.SERVICE_ROLE}, + {'name': 'use-default-roles', 'action': 'store_true', + 'help_text': helptext.USE_DEFAULT_ROLES}, {'name': 'ec2-attributes', 'help_text': helptext.EC2_ATTRIBUTES, 'schema': argumentschema.EC2_ATTRIBUTES_SCHEMA}, @@ -104,6 +110,13 @@ def _run_main(self, parsed_args, parsed_globals): emrutils.apply_dict( params, 'AdditionalInfo', parsed_args.additional_info) emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri) + if parsed_args.use_default_roles is True: + parsed_args.service_role = EMR_ROLE_NAME + if parsed_args.ec2_attributes is None: + parsed_args.ec2_attributes = {} + parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME + + emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role) instances_config = {} instances_config['InstanceGroups'] = \ instancegroupsutils.build_instance_groups( @@ -337,4 +350,4 @@ def _get_missing_applications_for_steps(self, specified_apps, parsed_args): if step_type in allowed_app_steps and \ step_type not in specified_apps: missing_apps.add(step['Type'].title()) - return missing_apps \ No newline at end of file + return missing_apps diff --git a/awscli/customizations/emr/createdefaultroles.py b/awscli/customizations/emr/createdefaultroles.py index c675de4a1034..e9f3c7049021 100644 --- a/awscli/customizations/emr/createdefaultroles.py +++ b/awscli/customizations/emr/createdefaultroles.py @@ -27,6 +27,7 @@ EC2_ROLE_NAME = "EMR_EC2_DefaultRole" +EMR_ROLE_NAME = "EMR_DefaultRole" EC2_ROLE_POLICY = { "Statement": [ @@ -49,6 +50,40 @@ } +EMR_ROLE_POLICY = { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CancelSpotInstanceRequests", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:Describe*", + "ec2:DeleteTags", + "ec2:ModifyImageAttribute", + "ec2:ModifyInstanceAttribute", + "ec2:RequestSpotInstances", + "ec2:RunInstances", + "ec2:TerminateInstances", + "iam:PassRole", + "iam:ListRolePolicies", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:ListInstanceProfiles", + "s3:Get*", + "s3:List*", + "s3:CreateBucket", + "sdb:BatchPutAttributes", + "sdb:Select" + ], + "Effect": "Allow", + "Resource": "*" + } + ] +} + + def assume_role_policy(serviceprincipal): return { "Version": "2008-10-17", @@ -98,7 +133,8 @@ def _get_regex_match_from_endpoint_host(endpoint_host): class CreateDefaultRoles(BasicCommand): NAME = "create-default-roles" DESCRIPTION = ('Creates the default IAM role ' + - EC2_ROLE_NAME + ' which can be used when' + EC2_ROLE_NAME + ' and ' + + EMR_ROLE_NAME + ' which can be used when' ' creating the cluster using the create-cluster command.') ARG_TABLE = [ {'name': 'iam-endpoint', @@ -132,7 +168,7 @@ def _run_main(self, parsed_args, parsed_globals): LOG.debug('Role ' + role_name + ' exists.') else: LOG.debug('Role ' + role_name + ' does not exist.' - ' Creating default role ' + role_name) + ' Creating default role for EC2: ' + role_name) ec2_result = self._create_role_with_role_policy( role_name, role_name, constants.EC2, emrutils.dict_to_string(EC2_ROLE_POLICY), @@ -151,10 +187,22 @@ def _run_main(self, parsed_args, parsed_globals): instance_profile_name, parsed_globals) + # Check if the default EMR Role exists. + role_name = EMR_ROLE_NAME + if self._check_if_role_exists(role_name, parsed_globals): + LOG.debug('Role ' + role_name + ' exists.') + else: + LOG.debug('Role ' + role_name + ' does not exist.' + ' Creating default role for EMR: ' + role_name) + emr_result = self._create_role_with_role_policy( + role_name, role_name, constants.EMR, + emrutils.dict_to_string(EMR_ROLE_POLICY), + parsed_globals) + emrutils.display_response( self._session, self._session.get_service('iam').get_operation('CreateRole'), - self._construct_result(ec2_result), + self._construct_result(ec2_result, emr_result), parsed_globals) return 0 @@ -166,11 +214,12 @@ def _check_for_iam_endpoint(self, region, iam_endpoint): if iam_endpoint is None: raise exceptions.UnknownIamEndpointError(region=region) - def _construct_result(self, ec2_response): + def _construct_result(self, ec2_response, emr_response): result = [] self._construct_role_and_role_policy_structure( result, ec2_response, EC2_ROLE_POLICY) - + self._construct_role_and_role_policy_structure( + result, emr_response, EMR_ROLE_POLICY) return result def _construct_role_and_role_policy_structure( diff --git a/awscli/customizations/emr/helptext.py b/awscli/customizations/emr/helptext.py index db6c1229e5db..12b6ad5d4f32 100644 --- a/awscli/customizations/emr/helptext.py +++ b/awscli/customizations/emr/helptext.py @@ -11,6 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +from awscli.customizations.emr.createdefaultroles import EMR_ROLE_NAME from awscli.customizations.emr.createdefaultroles import EC2_ROLE_NAME TERMINATE_CLUSTERS = ( @@ -49,6 +50,19 @@ 'of the cluster. If a value is not provided, ' 'logs are not created.

') +SERVICE_ROLE = ( + '

Allows EMR to call other AWS Services such as EC2 on your behalf.

' + 'To create the default Service Role ' + EMR_ROLE_NAME + ',' + ' use aws emr create-default-roles command.

' + 'This command will also create the default EC2 instance profile ' + '' + EC2_ROLE_NAME + '.') + +USE_DEFAULT_ROLES = ( + '

Uses --service-role=' + EMR_ROLE_NAME + ', and ' + '--ec2-attributes

InstanceProfile=' + EC2_ROLE_NAME + '' + 'To create the default service role and instance profile' + ' use aws emr create-default-roles command.

') + AMI_VERSION = ( '

The version number of the Amazon Machine Image (AMI) ' 'to use for Amazon EC2 instances in the cluster. ' @@ -75,12 +89,15 @@ 'Subnet cannot be specified together. To create the default ' 'instance profile ' + EC2_ROLE_NAME + ',' ' use aws emr create-default-roles command.

' + 'This command will also create the default EMR service role ' + '' + EMR_ROLE_NAME + '.' '
  • KeyName - the name of the AWS EC2 key pair you are using ' 'to launch the cluster.
    • ' '
  • AvailabilityZone - An isolated resource ' 'location within a region.
    • ' '
  • SubnetId- Assign the EMR cluster to this Amazon VPC Subnet.
    • ' - '
  • InstanceProfile - A container for the EC2 IAM Role.
    • ' + '
  • InstanceProfile - Provides access to other AWS services such as S3,' + ' DynamoDB from EC2 instances that are launched by EMR..
    • ' ) AUTO_TERMINATE = ( diff --git a/awscli/examples/emr/create-cluster-examples.rst b/awscli/examples/emr/create-cluster-examples.rst index c27ace380e8a..d1f5339bd1bf 100644 --- a/awscli/examples/emr/create-cluster-examples.rst +++ b/awscli/examples/emr/create-cluster-examples.rst @@ -4,13 +4,25 @@ aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate -**2. Create an Amazon EMR cluster with MASTER, CORE, and TASK instance groups** +**2. Create an Amazon EMR cluster with ServiceRole and InstanceProfile + +- Command:: + + aws emr create-cluster --ami-version 3.1.0 --service-role EMR_DefaultRole --ec2-attributes InstanceProfiles=EC2_EMR_DefaultRoles --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate + +**3. Create an Amazon EMR cluster with default roles + +- Command:: + + aws emr create-cluster --ami-version 3.1.0 --use-default-roles --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate + +**4. Create an Amazon EMR cluster with MASTER, CORE, and TASK instance groups** - Command:: aws emr create-cluster --ami-version 3.1.0 --auto-terminate --instance-groups Name=Master,InstanceGroupType=MASTER,InstanceType=m3.xlarge,InstanceCount=1 Name=Core,InstanceGroupType=CORE,InstanceType=m3.xlarge,InstanceCount=2 Name=Task,InstanceGroupType=TASK,InstanceType=m3.xlarge,InstanceCount=2 -**3. Specify whether the cluster should terminate after completing all the steps** +**5. Specify whether the cluster should terminate after completing all the steps** - Create an Amazon EMR cluster that will terminate after completing all the steps:: @@ -20,7 +32,7 @@ aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --no-auto-terminate -**4. Specify EC2 Attributes** +**6. Specify EC2 Attributes** - Create an Amazon EMR cluster with Amazon EC2 Key Pair "myKey" and instance profile "myProfile":: @@ -34,13 +46,13 @@ aws emr create-cluster --ec2-attributes AvailabilityZone=us-west-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate -**5. Enable debugging and specify a Log URI** +**7. Enable debugging and specify a Log URI** - Command:: aws emr create-cluster --enable-debugging --log-uri s3://myBucket/myLog --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate -**6. Add tags when creating an Amazon EMR cluster** +**8. Add tags when creating an Amazon EMR cluster** - Add a list of tags:: @@ -50,7 +62,7 @@ aws emr describe-cluster --cluster-id j-XXXXXXYY --query Cluster.Tags -**7. Add a list of bootstrap actions when creating an Amazon EMR Cluster** +**9. Add a list of bootstrap actions when creating an Amazon EMR Cluster** - Command:: @@ -60,7 +72,7 @@ aws emr create-cluster --bootstrap-actions Path=s3://elasticmapreduce/bootstrap-actions/configure-hadoop,Name="Change the maximum number of map tasks",Args=[-M,s3://myawsbucket/config.xml,-m,mapred.tasktracker.map.tasks.maximum=2] Path=s3://elasticmapreduce/bootstrap-actions/configure-daemons,Name="Set the NameNode heap size",Args=[--namenode-heap-size=2048,--namenode-opts=-XX:GCTimeRatio=19] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate -**8. Create an Amazon EMR cluster with applications** +**10. Create an Amazon EMR cluster with applications** - Create an Amazon EMR cluster with Hive, Pig, HBase, Ganglia, and Impala installed:: @@ -74,13 +86,13 @@ aws emr create-cluster --applications Name=MapR,Args=--edition,m7,--version,3.0.2 --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate -**9. Restore HBase data from backup when creating an Amazon EMR cluster** +**11. Restore HBase data from backup when creating an Amazon EMR cluster** -Command:: aws emr create-cluster --applications Name=HBase --restore-from-hbase-backup Dir=s3://myBucket/myBackup,BackupVersion=myBackupVersion --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate -**10. To add Custom JAR steps to a cluster when creating an Amazon EMR cluster** +**12. To add Custom JAR steps to a cluster when creating an Amazon EMR cluster** - Command:: @@ -94,7 +106,7 @@ Type, Name, ActionOnFailure, Args -**11. To add Streaming steps when creating an Amazon EMR cluster** +**13. To add Streaming steps when creating an Amazon EMR cluster** - Command:: @@ -108,7 +120,7 @@ Name, ActionOnFailure -**12. To add Hive steps when creating an Amazon EMR cluster** +**14. To add Hive steps when creating an Amazon EMR cluster** - Command:: @@ -122,7 +134,7 @@ Name, ActionOnFailure, Version -**13. To add Pig steps when creating an Amazon EMR cluster** +**15. To add Pig steps when creating an Amazon EMR cluster** - Command:: @@ -136,7 +148,7 @@ Name, ActionOnFailure, Version -**14. To add Impala steps when creating an Amazon EMR cluster** +**16. To add Impala steps when creating an Amazon EMR cluster** - Command:: diff --git a/awscli/examples/emr/create-cluster-synopsis.rst b/awscli/examples/emr/create-cluster-synopsis.rst index f8a22c77e7b2..b37981df62f4 100644 --- a/awscli/examples/emr/create-cluster-synopsis.rst +++ b/awscli/examples/emr/create-cluster-synopsis.rst @@ -2,10 +2,12 @@ --ami-version --instance-groups --auto-terminate | --no-auto-terminate + [--use-default-role] + [--service-role ] [--name ] [--log-uri ] [--additional-info ] - [--ec2-attributes ] + [--ec2-attributes ] [--termination-protected | --no-termination-protected] [--visible-to-all-users | --no-visible-to-all-users] [--enable-debugging | --no-enable-debugging] diff --git a/awscli/examples/emr/create-default-roles.rst b/awscli/examples/emr/create-default-roles.rst index 4600f828825a..77b73ea2da1e 100644 --- a/awscli/examples/emr/create-default-roles.rst +++ b/awscli/examples/emr/create-default-roles.rst @@ -10,48 +10,102 @@ If the role does not exist then the output will be: - [ - { - "RolePolicy": { - "Statement": [ - { - "Action": [ - "cloudwatch:*", - "dynamodb:*", - "ec2:Describe*", - "elasticmapreduce:Describe*", - "rds:Describe*", - "s3:*", - "sdb:*", - "sns:*", - "sqs:*" - ], - "Resource": [ - "*" - ], - "Effect": "Allow" - } - ] - }, - "Role": { - "AssumeRolePolicyDocument": { - "Version": "2008-10-17", - "Statement": [ - { - "Action": "sts:AssumeRole", - "Sid": "", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - } - } - ] - }, - "RoleId": "AROLORKUOAL65X57SBWFK", - "CreateDate": "2014-05-07T00:02:06.154Z", - "RoleName": "EMR_EC2_DefaultRole", - "Path": "/", - "Arn": "arn:aws:iam::176430881729:role/EMR_EC2_DefaultRole" - } - } - ] \ No newline at end of file + [ + { + "RolePolicy": { + "Statement": [ + { + "Action": [ + "cloudwatch:*", + "dynamodb:*", + "ec2:Describe*", + "elasticmapreduce:Describe*", + "rds:Describe*", + "s3:*", + "sdb:*", + "sns:*", + "sqs:*" + ], + "Resource": [ + "*" + ], + "Effect": "Allow" + } + ] + }, + "Role": { + "AssumeRolePolicyDocument": { + "Version": "2008-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ] + }, + "RoleId": "AROAJEATMSYEQGRVYQDX4", + "CreateDate": "2014-06-14T01:05:15.356Z", + "RoleName": "EMR_EC2_DefaultRole", + "Path": "/", + "Arn": "arn:aws:iam::176430881729:role/EMR_EC2_DefaultRole" + } + }, + { + "RolePolicy": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CancelSpotInstanceRequests", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:Describe*", + "ec2:DeleteTags", + "ec2:ModifyImageAttribute", + "ec2:ModifyInstanceAttribute", + "ec2:RequestSpotInstances", + "ec2:RunInstances", + "ec2:TerminateInstances", + "iam:PassRole", + "iam:ListRolePolicies", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:ListInstanceProfiles", + "s3:Get*", + "s3:List*", + "s3:CreateBucket", + "sdb:BatchPutAttributes", + "sdb:Select" + ], + "Resource": "*", + "Effect": "Allow" + } + ] + }, + "Role": { + "AssumeRolePolicyDocument": { + "Version": "2008-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": "elasticmapreduce.amazonaws.com" + } + } + ] + }, + "RoleId": "AROAJRHC33G6KRX5D5QF2", + "CreateDate": "2014-06-14T01:05:17.464Z", + "RoleName": "EMR_DefaultRole", + "Path": "/", + "Arn": "arn:aws:iam::176430881729:role/EMR_DefaultRole" + } + } + ] diff --git a/tests/unit/customizations/emr/test_create_cluster.py b/tests/unit/customizations/emr/test_create_cluster.py index 6d836c85c5af..3c678258404c 100644 --- a/tests/unit/customizations/emr/test_create_cluster.py +++ b/tests/unit/customizations/emr/test_create_cluster.py @@ -50,7 +50,7 @@ 'InstanceGroupType=TASK,Name=TASK,' 'InstanceCount=1,InstanceType=m1.large ') -DEFAULT_CMD = 'emr create-cluster --auto-terminate --ami-version 3.0.4 --instance-groups ' + \ +DEFAULT_CMD = 'emr create-cluster --auto-terminate --ami-version 3.0.4 --use-default-roles --instance-groups ' + \ DEFAULT_INSTANCE_GROUPS_ARG DEFAULT_INSTANCES = {'KeepJobFlowAliveWhenNoSteps': False, @@ -58,12 +58,17 @@ 'InstanceGroups': DEFAULT_INSTANCE_GROUPS } +EC2_ROLE_NAME = "EMR_EC2_DefaultRole" +EMR_ROLE_NAME = "EMR_DefaultRole" + DEFAULT_RESULT = \ { 'Name': DEFAULT_CLUSTER_NAME, 'Instances': DEFAULT_INSTANCES, 'AmiVersion': '3.0.4', 'VisibleToAllUsers': False, + 'JobFlowRole': EC2_ROLE_NAME, + 'ServiceRole': EMR_ROLE_NAME, 'Tags': [] } @@ -286,6 +291,29 @@ class TestCreateCluster(BaseAWSCommandParamsTest): def test_default_cmd(self): self.assert_params_for_cmd(DEFAULT_CMD, DEFAULT_RESULT) + def test_cluster_with_out_service_role_and_instance_profile(self): + cmd = ('emr create-cluster --auto-terminate --ami-version 3.0.4' + ' --instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG) + result = copy.deepcopy(DEFAULT_RESULT) + del result['JobFlowRole'] + del result['ServiceRole'] + self.assert_params_for_cmd(cmd, result) + + def test_cluster_with_service_role_and_instance_profile(self): + cmd = ('emr create-cluster --auto-terminate --ami-version 3.0.4' + ' --instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG + + ' --service-role ServiceRole --ec2-attributes ' + 'InstanceProfile=Ec2_InstanceProfile') + result = copy.deepcopy(DEFAULT_RESULT) + result['JobFlowRole'] = 'Ec2_InstanceProfile' + result['ServiceRole'] = 'ServiceRole' + self.assert_params_for_cmd(cmd, result) + + def test_cluster_default_roles_overrides(self): + cmd = (DEFAULT_CMD + '--service-role ServiceRole ' + '--ec2-attributes InstanceProfile=Ec2_InstanceProfile') + self.assert_params_for_cmd(cmd, DEFAULT_RESULT) + def test_cluster_name_no_space(self): cmd = DEFAULT_CMD + '--name MyCluster' result = copy.deepcopy(DEFAULT_RESULT) @@ -319,7 +347,8 @@ def test_additional_info(self): self.assert_params_for_cmd(cmd, result) def test_no_auto_terminte(self): - cmd = ('emr create-cluster --ami-version 3.0.4 --no-auto-terminate' + + cmd = ('emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--no-auto-terminate' + ' --instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG) result = copy.deepcopy(DEFAULT_RESULT) instances = copy.deepcopy(DEFAULT_INSTANCES) @@ -337,7 +366,8 @@ def test_auto_terminate_and_no_auto_terminate(self): self.assertEquals(expected_error_msg, result[1]) def test_missing_auto_terminate_or_no_auto_terminate(self): - cmd = (self.prefix + '--ami-version 3.0.4 --instance-groups ' + + cmd = (self.prefix + '--use-default-roles --ami-version 3.0.4 ' + '--instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG) expected_error_msg = ( '\naws: error: Must specify one of the following boolean options:' @@ -430,7 +460,8 @@ def test_enable_debugging_and_no_enable_debugging(self): def test_instance_groups_default_name_market(self): cmd = ( - 'emr create-cluster --ami-version 3.0.4 --auto-terminate ' + 'emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--auto-terminate ' '--instance-groups ' 'InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m1.large ' 'InstanceGroupType=CORE,InstanceCount=1,InstanceType=m1.large ' @@ -439,7 +470,8 @@ def test_instance_groups_default_name_market(self): def test_instance_groups_instance_group_type_mismatch_cases(self): cmd = ( - 'emr create-cluster --ami-version 3.0.4 --auto-terminate ' + 'emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--auto-terminate ' '--instance-groups ' 'Name=MASTER,InstanceGroupType=MaSter,InstanceCount=1,' 'InstanceType=m1.large Name=CORE,InstanceGroupType=cORE,' @@ -449,7 +481,8 @@ def test_instance_groups_instance_group_type_mismatch_cases(self): def test_instance_groups_missing_instance_group_type_error(self): cmd = ( - 'emr create-cluster --ami-version 3.0.4 --auto-terminate ' + 'emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--auto-terminate ' '--instance-groups ' 'Name=Master,InstanceCount=1,InstanceType=m1.small') expect_error_msg = ( @@ -460,7 +493,8 @@ def test_instance_groups_missing_instance_group_type_error(self): def test_instance_groups_missing_instance_type_error(self): cmd = ( - 'emr create-cluster --ami-version 3.0.4 --auto-terminate ' + 'emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--auto-terminate ' '--instance-groups ' 'Name=Master,InstanceGroupType=MASTER,InstanceCount=1') expect_error_msg = ( @@ -471,7 +505,8 @@ def test_instance_groups_missing_instance_type_error(self): def test_instance_groups_missing_instance_count_error(self): cmd = ( - 'emr create-cluster --ami-version 3.0.4 --auto-terminate ' + 'emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--auto-terminate ' '--instance-groups ' 'Name=Master,InstanceGroupType=MASTER,InstanceType=m1.xlarge') expect_error_msg = ( @@ -483,7 +518,8 @@ def test_instance_groups_missing_instance_count_error(self): def test_instance_groups_from_json_file(self): data_path = os.path.join( os.path.dirname(__file__), 'input_instance_groups.json') - cmd = ('emr create-cluster --ami-version 3.0.4 --auto-terminate ' + cmd = ('emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--auto-terminate ' '--instance-groups file://' + data_path) result = copy.deepcopy(DEFAULT_RESULT) result['Instances']['InstanceGroups'] = \ @@ -513,11 +549,11 @@ def test_instance_groups_from_json_file(self): def test_ec2_attributes_no_az(self): cmd = DEFAULT_CMD + ( '--ec2-attributes KeyName=testkey,SubnetId=subnet-123456,' - 'InstanceProfile=aws-emr-ec2-role') + 'InstanceProfile=EMR_EC2_DefaultRole') result = copy.deepcopy(DEFAULT_RESULT) result['Instances']['Ec2KeyName'] = 'testkey' result['Instances']['Ec2SubnetId'] = 'subnet-123456' - result['JobFlowRole'] = 'aws-emr-ec2-role' + result['JobFlowRole'] = 'EMR_EC2_DefaultRole' self.assert_params_for_cmd(cmd, result) def test_ec2_attributes_az(self): diff --git a/tests/unit/customizations/emr/test_create_default_role.py b/tests/unit/customizations/emr/test_create_default_role.py index a17d83a18f4b..0b210be14e60 100644 --- a/tests/unit/customizations/emr/test_create_default_role.py +++ b/tests/unit/customizations/emr/test_create_default_role.py @@ -20,6 +20,7 @@ EC2_ROLE_NAME = "EMR_EC2_DefaultRole" +EMR_ROLE_NAME = "EMR_DefaultRole" EC2_ROLE_POLICY = { "Statement": [ @@ -90,11 +91,23 @@ class TestCreateDefaultRole(BaseAWSCommandParamsTest): ] } + emr_role_policy_document = { + "Version": "2008-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": {"Service": "elasticmapreduce.amazonaws.com.cn"}, + "Action": "sts:AssumeRole" + } + ] + } + def test_default_roles_exist(self): cmdline = self.prefix self.run_cmd(cmdline, expected_rc=0) - self.assertEqual(len(self.operations_called), 2) + self.assertEqual(len(self.operations_called), 3) self.assertEqual(self.operations_called[0][0].name, 'GetRole') self.assertEqual(self.operations_called[0][1]['RoleName'], @@ -104,6 +117,9 @@ def test_default_roles_exist(self): 'GetInstanceProfile') self.assertEqual(self.operations_called[1][1]['InstanceProfileName'], EC2_ROLE_NAME) + self.assertEqual(self.operations_called[2][0].name, 'GetRole') + self.assertEqual(self.operations_called[2][1]['RoleName'], + EMR_ROLE_NAME) @mock.patch('awscli.customizations.emr.emr.' 'CreateDefaultRoles._construct_result') @@ -121,9 +137,9 @@ def test_default_roles_not_exist(self, role_exists_patch, cmdline = self.prefix + ' --region cn-north-1' self.run_cmd(cmdline, expected_rc=0) - # Only 4 operations will be called as we are mocking + # Only 6 operations will be called as we are mocking # _check_if_role_exists and _check_if_instance_profile_exists methods. - self.assertEqual(len(self.operations_called), 4) + self.assertEqual(len(self.operations_called), 6) self.assertEqual(self.operations_called[0][0].name, 'CreateRole') self.assertEqual(self.operations_called[0][1]['RoleName'], @@ -153,6 +169,19 @@ def test_default_roles_not_exist(self, role_exists_patch, self.assertEqual(self.operations_called[3][1]['RoleName'], EC2_ROLE_NAME) + self.assertEqual(self.operations_called[4][0].name, 'CreateRole') + self.assertEqual(self.operations_called[4][1]['RoleName'], + EMR_ROLE_NAME) + self.assertEqual( + self.operations_called[4][1]['AssumeRolePolicyDocument'], + emrutils.dict_to_string(self.emr_role_policy_document)) + + self.assertEqual(self.operations_called[5][0].name, 'PutRolePolicy') + self.assertEqual(self.operations_called[5][1]['PolicyName'], + EMR_ROLE_NAME) + self.assertEqual(self.operations_called[5][1]['RoleName'], + EMR_ROLE_NAME) + @mock.patch('awscli.customizations.emr.emr.' 'CreateDefaultRoles._construct_result') @mock.patch('awscli.customizations.emr.createdefaultroles' From dee928b5f51f9039c2c850d2236a40faf1b05ec4 Mon Sep 17 00:00:00 2001 From: Mohan Gandhi Date: Fri, 13 Jun 2014 22:08:24 +0000 Subject: [PATCH 4/6] Test failure fix --- awscli/customizations/emr/createdefaultroles.py | 1 + 1 file changed, 1 insertion(+) diff --git a/awscli/customizations/emr/createdefaultroles.py b/awscli/customizations/emr/createdefaultroles.py index e9f3c7049021..2c8aa69c0c86 100644 --- a/awscli/customizations/emr/createdefaultroles.py +++ b/awscli/customizations/emr/createdefaultroles.py @@ -148,6 +148,7 @@ class CreateDefaultRoles(BasicCommand): def _run_main(self, parsed_args, parsed_globals): ec2_result = None + emr_result = None self.iam = self._session.get_service('iam') self.iam_endpoint_url = parsed_args.iam_endpoint region = self._get_region(parsed_globals) From cae12c9e612f598b7a164337d15388c1b8e546ba Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Tue, 24 Jun 2014 14:11:57 -0700 Subject: [PATCH 5/6] Update changelog with latest changes --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1b46ccebc41b..032f59b15fa0 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,7 @@ CHANGELOG Next Release (TBD) ================== +* feature:``aws ses``: Add support for delivery notifications * bugfix:Region Config: Fix issue for ``cn-north-1`` region (`issue botocore 314 `__) * bugfix:Amazon EC2 Credential File: Fix regression for parsing From b02d152f4e1ecb701c1d041bf91eed79078338f3 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Tue, 24 Jun 2014 16:20:42 -0700 Subject: [PATCH 6/6] Bumping version to 1.3.19 --- CHANGELOG.rst | 4 ++-- awscli/__init__.py | 2 +- doc/source/conf.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 032f59b15fa0..59b990d8bdfa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,8 +2,8 @@ CHANGELOG ========= -Next Release (TBD) -================== +1.3.19 +====== * feature:``aws ses``: Add support for delivery notifications * bugfix:Region Config: Fix issue for ``cn-north-1`` region diff --git a/awscli/__init__.py b/awscli/__init__.py index 921c0b600bbe..b32446081050 100644 --- a/awscli/__init__.py +++ b/awscli/__init__.py @@ -17,7 +17,7 @@ """ import os -__version__ = '1.3.18' +__version__ = '1.3.19' # # Get our data path to be added to botocore's search path diff --git a/doc/source/conf.py b/doc/source/conf.py index e3a7e64a75c7..34d4161f139d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -52,7 +52,7 @@ # The short X.Y version. version = '1.3.' # The full version, including alpha/beta/rc tags. -release = '1.3.18' +release = '1.3.19' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 3d291eaac8ac..242054c42a62 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import awscli -requires = ['botocore>=0.52.0,<0.53.0', +requires = ['botocore>=0.53.0,<0.54.0', 'bcdoc>=0.12.0,<0.13.0', 'six>=1.1.0', 'colorama==0.2.5',