Merge branch 'release-1.3.19' into develop

* release-1.3.19:
  Bumping version to 1.3.19
  Update changelog with latest changes
  Test failure fix
  EMR: Update create-default-roles and Add --service-role & --use-default-roles
  EMR: Add unit tests for application
  EMR: Bug fix in install-applications and create-cluster

CHANGELOG.rst

@@ -2,9 +2,10 @@
 CHANGELOG
 =========
 
-Next Release (TBD)
-==================
+1.3.19
+======
 
+* feature:``aws ses``: Add support for delivery notifications
 * bugfix:Region Config: Fix issue for ``cn-north-1`` region
   (`issue botocore 314 <https://github.com/boto/botocore/pull/314>`__)
 * bugfix:Amazon EC2 Credential File: Fix regression for parsing

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.3.18'
+__version__ = '1.3.19'
 
 #
 # Get our data path to be added to botocore's search path

awscli/customizations/emr/applicationutils.py

@@ -24,10 +24,10 @@ def build_applications(parsed_applications, parsed_globals, ami_version=None):
     for app_config in parsed_applications:
         app_name = app_config['Name'].lower()
 
-        if app_name in constants.MAPR_NAMES:
+        if app_name in constants.SUPPORTED_PRODUCTS:
             app_list.append(
                 build_supported_product(
-                    app_config['Name'], app_config['Args']))
+                    app_config['Name'], app_config.get('Args')))
         elif app_name == constants.HIVE:
             hive_version = app_config.get('Version')
             if hive_version is None:

awscli/customizations/emr/constants.py

@@ -27,7 +27,7 @@
 SCRIPT_RUNNER_PATH = '/libs/script-runner/script-runner.jar'
 DEBUGGING_PATH = '/libs/state-pusher/0.1/fetch'
 DEBUGGING_NAME = 'Setup Hadoop Debugging'
-MAPR_NAMES = ['mapr', 'mapr-m3', 'mapr-m5', 'mapr-m7']
+SUPPORTED_PRODUCTS = ['mapr', 'mapr-m3', 'mapr-m5', 'mapr-m7', 'hue']
 
 MAX_BOOTSTRAP_ACTION_NUMBER = 16
 BOOTSTRAP_ACTION_NAME = 'Bootstrap action'

awscli/customizations/emr/createcluster.py

@@ -22,6 +22,8 @@
 from awscli.customizations.emr import exceptions
 from awscli.customizations.emr import applicationutils
 from awscli.customizations.emr import instancegroupsutils
+from awscli.customizations.emr.createdefaultroles import EMR_ROLE_NAME
+from awscli.customizations.emr.createdefaultroles import EC2_ROLE_NAME
 import re
 
 
@@ -52,6 +54,10 @@ class CreateCluster(BasicCommand):
          'help_text': helptext.CLUSTER_NAME},
         {'name': 'log-uri',
          'help_text': helptext.LOG_URI},
+        {'name': 'service-role',
+         'help_text': helptext.SERVICE_ROLE},
+        {'name': 'use-default-roles', 'action': 'store_true',
+         'help_text': helptext.USE_DEFAULT_ROLES},
         {'name': 'ec2-attributes',
          'help_text': helptext.EC2_ATTRIBUTES,
          'schema': argumentschema.EC2_ATTRIBUTES_SCHEMA},
@@ -104,6 +110,13 @@ def _run_main(self, parsed_args, parsed_globals):
         emrutils.apply_dict(
             params, 'AdditionalInfo', parsed_args.additional_info)
         emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)
+        if parsed_args.use_default_roles is True:
+            parsed_args.service_role = EMR_ROLE_NAME
+            if parsed_args.ec2_attributes is None:
+                parsed_args.ec2_attributes = {}
+            parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME
+
+        emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)
         instances_config = {}
         instances_config['InstanceGroups'] = \
             instancegroupsutils.build_instance_groups(
@@ -337,4 +350,4 @@ def _get_missing_applications_for_steps(self, specified_apps, parsed_args):
                     if step_type in allowed_app_steps and \
                             step_type not in specified_apps:
                         missing_apps.add(step['Type'].title())
-        return missing_apps
+        return missing_apps

awscli/customizations/emr/createdefaultroles.py

@@ -27,6 +27,7 @@
 
 
 EC2_ROLE_NAME = "EMR_EC2_DefaultRole"
+EMR_ROLE_NAME = "EMR_DefaultRole"
 
 EC2_ROLE_POLICY = {
     "Statement": [
@@ -49,6 +50,40 @@
 }
 
 
+EMR_ROLE_POLICY = {
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "ec2:AuthorizeSecurityGroupIngress",
+                "ec2:CancelSpotInstanceRequests",
+                "ec2:CreateSecurityGroup",
+                "ec2:CreateTags",
+                "ec2:Describe*",
+                "ec2:DeleteTags",
+                "ec2:ModifyImageAttribute",
+                "ec2:ModifyInstanceAttribute",
+                "ec2:RequestSpotInstances",
+                "ec2:RunInstances",
+                "ec2:TerminateInstances",
+                "iam:PassRole",
+                "iam:ListRolePolicies",
+                "iam:GetRole",
+                "iam:GetRolePolicy",
+                "iam:ListInstanceProfiles",
+                "s3:Get*",
+                "s3:List*",
+                "s3:CreateBucket",
+                "sdb:BatchPutAttributes",
+                "sdb:Select"
+            ],
+            "Effect": "Allow",
+            "Resource": "*"
+        }
+    ]
+}
+
+
 def assume_role_policy(serviceprincipal):
     return {
         "Version": "2008-10-17",
@@ -98,7 +133,8 @@ def _get_regex_match_from_endpoint_host(endpoint_host):
 class CreateDefaultRoles(BasicCommand):
     NAME = "create-default-roles"
     DESCRIPTION = ('Creates the default IAM role ' +
-                   EC2_ROLE_NAME + ' which can be used when'
+                   EC2_ROLE_NAME + ' and ' +
+                   EMR_ROLE_NAME + ' which can be used when'
                    ' creating the cluster using the create-cluster command.')
     ARG_TABLE = [
         {'name': 'iam-endpoint',
@@ -112,6 +148,7 @@ class CreateDefaultRoles(BasicCommand):
 
     def _run_main(self, parsed_args, parsed_globals):
         ec2_result = None
+        emr_result = None
         self.iam = self._session.get_service('iam')
         self.iam_endpoint_url = parsed_args.iam_endpoint
         region = self._get_region(parsed_globals)
@@ -132,7 +169,7 @@ def _run_main(self, parsed_args, parsed_globals):
             LOG.debug('Role ' + role_name + ' exists.')
         else:
             LOG.debug('Role ' + role_name + ' does not exist.'
-                      ' Creating default role ' + role_name)
+                      ' Creating default role for EC2: ' + role_name)
             ec2_result = self._create_role_with_role_policy(
                 role_name, role_name, constants.EC2,
                 emrutils.dict_to_string(EC2_ROLE_POLICY),
@@ -151,10 +188,22 @@ def _run_main(self, parsed_args, parsed_globals):
                                                     instance_profile_name,
                                                     parsed_globals)
 
+        # Check if the default EMR Role exists.
+        role_name = EMR_ROLE_NAME
+        if self._check_if_role_exists(role_name, parsed_globals):
+            LOG.debug('Role ' + role_name + ' exists.')
+        else:
+            LOG.debug('Role ' + role_name + ' does not exist.'
+                      ' Creating default role for EMR: ' + role_name)
+            emr_result = self._create_role_with_role_policy(
+                role_name, role_name, constants.EMR,
+                emrutils.dict_to_string(EMR_ROLE_POLICY),
+                parsed_globals)
+
         emrutils.display_response(
             self._session,
             self._session.get_service('iam').get_operation('CreateRole'),
-            self._construct_result(ec2_result),
+            self._construct_result(ec2_result, emr_result),
             parsed_globals)
 
         return 0
@@ -166,11 +215,12 @@ def _check_for_iam_endpoint(self, region, iam_endpoint):
             if iam_endpoint is None:
                 raise exceptions.UnknownIamEndpointError(region=region)
 
-    def _construct_result(self, ec2_response):
+    def _construct_result(self, ec2_response, emr_response):
         result = []
         self._construct_role_and_role_policy_structure(
             result, ec2_response, EC2_ROLE_POLICY)
-
+        self._construct_role_and_role_policy_structure(
+            result, emr_response, EMR_ROLE_POLICY)
         return result
 
     def _construct_role_and_role_policy_structure(

awscli/customizations/emr/helptext.py

@@ -11,6 +11,7 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 
+from awscli.customizations.emr.createdefaultroles import EMR_ROLE_NAME
 from awscli.customizations.emr.createdefaultroles import EC2_ROLE_NAME
 
 TERMINATE_CLUSTERS = (
@@ -49,6 +50,19 @@
     'of the cluster. If a value is not provided, '
     'logs are not created.</p>')
 
+SERVICE_ROLE = (
+    '<p>Allows EMR to call other AWS Services such as EC2 on your behalf.</p>'
+    'To create the default Service Role <code>' + EMR_ROLE_NAME + '</code>,'
+    ' use <code>aws emr create-default-roles</code> command. </p>'
+    'This command will also create the default EC2 instance profile '
+    '<code>' + EC2_ROLE_NAME + '</code>.')
+
+USE_DEFAULT_ROLES = (
+    '<p>Uses --service-role=<code>' + EMR_ROLE_NAME + '</code>, and '
+    '--ec2-attributes </p> InstanceProfile=<code>' + EC2_ROLE_NAME + '</code>'
+    'To create the default service role and instance profile'
+    ' use <code>aws emr create-default-roles</code> command. </p>')
+
 AMI_VERSION = (
     '<p>The version number of the Amazon Machine Image (AMI) '
     'to use for Amazon EC2 instances in the cluster. '
@@ -75,12 +89,15 @@
     'Subnet cannot be specified together. To create the default '
     'instance profile <code>' + EC2_ROLE_NAME + '</code>,'
     ' use <code>aws emr create-default-roles</code> command. </p>'
+    'This command will also create the default EMR service role '
+    '<code>' + EMR_ROLE_NAME + '</code>.'
     '<li>KeyName - the name of the AWS EC2 key pair you are using '
     'to launch the cluster.</li>'
     '<li>AvailabilityZone - An isolated resource '
     'location within a region.</li>'
     '<li>SubnetId- Assign the EMR cluster to this Amazon VPC Subnet. </li>'
-    '<li>InstanceProfile - A container for the EC2 IAM Role. </li>'
+    '<li>InstanceProfile - Provides access to other AWS services such as S3,'
+    ' DynamoDB from EC2 instances that are launched by EMR.. </li>'
     )
 
 AUTO_TERMINATE = (

awscli/customizations/emr/installapplications.py

@@ -27,7 +27,7 @@ class InstallApplications(BasicCommand):
     ARG_TABLE = [
         {'name': 'cluster-id', 'required': True,
          'help_text': helptext.CLUSTER_ID},
-        {'name': 'apps', 'required': True,
+        {'name': 'applications', 'required': True,
          'help_text': helptext.INSTALL_APPLICATIONS,
          'schema': argumentschema.APPLICATIONS_SCHEMA},
     ]
@@ -38,9 +38,9 @@ def _run_main(self, parsed_args, parsed_globals):
 
         parameters = {'JobFlowId': parsed_args.cluster_id}
 
-        self._check_for_supported_apps(parsed_args.apps)
+        self._check_for_supported_apps(parsed_args.applications)
         parameters['Steps'] = applicationutils.build_applications(
-            parsed_args.apps, parsed_globals)[2]
+            parsed_args.applications, parsed_globals)[2]
 
         emrutils.call_and_display_response(self._session, 'AddJobFlowSteps',
                                            parameters, parsed_globals)

awscli/examples/emr/create-cluster-examples.rst

@@ -4,13 +4,25 @@
 
     aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
 
-**2. Create an Amazon EMR cluster with MASTER, CORE, and TASK instance groups**
+**2. Create an Amazon EMR cluster with ServiceRole and InstanceProfile
+
+- Command::
+
+    aws emr create-cluster --ami-version 3.1.0 --service-role EMR_DefaultRole --ec2-attributes InstanceProfiles=EC2_EMR_DefaultRoles --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
+    
+**3. Create an Amazon EMR cluster with default roles
+
+- Command::
+
+    aws emr create-cluster --ami-version 3.1.0 --use-default-roles --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
+
+**4. Create an Amazon EMR cluster with MASTER, CORE, and TASK instance groups**
 
 - Command::
 
 	aws emr create-cluster --ami-version 3.1.0 --auto-terminate --instance-groups Name=Master,InstanceGroupType=MASTER,InstanceType=m3.xlarge,InstanceCount=1 Name=Core,InstanceGroupType=CORE,InstanceType=m3.xlarge,InstanceCount=2 Name=Task,InstanceGroupType=TASK,InstanceType=m3.xlarge,InstanceCount=2
 
-**3. Specify whether the cluster should terminate after completing all the steps**
+**5. Specify whether the cluster should terminate after completing all the steps**
 
 - Create an Amazon EMR cluster that will terminate after completing all the steps::
 
@@ -20,7 +32,7 @@
 
 	aws emr create-cluster --ami-version 3.1.0  --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge  InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --no-auto-terminate
 
-**4. Specify EC2 Attributes**
+**6. Specify EC2 Attributes**
 
 - Create an Amazon EMR cluster with Amazon EC2 Key Pair "myKey" and instance profile "myProfile"::
 
@@ -34,13 +46,13 @@
 
 	aws emr create-cluster --ec2-attributes AvailabilityZone=us-west-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
 
-**5. Enable debugging and specify a Log URI**
+**7. Enable debugging and specify a Log URI**
 
 - Command::
 
     aws emr create-cluster --enable-debugging --log-uri s3://myBucket/myLog  --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
 
-**6. Add tags when creating an Amazon EMR cluster**
+**8. Add tags when creating an Amazon EMR cluster**
 
 - Add a list of tags::
 
@@ -50,7 +62,7 @@
 
     aws emr describe-cluster --cluster-id j-XXXXXXYY --query Cluster.Tags
 
-**7. Add a list of bootstrap actions when creating an Amazon EMR Cluster**
+**9. Add a list of bootstrap actions when creating an Amazon EMR Cluster**
 
 - Command::
 
@@ -60,7 +72,7 @@
 
     aws emr create-cluster --bootstrap-actions Path=s3://elasticmapreduce/bootstrap-actions/configure-hadoop,Name="Change the maximum number of map tasks",Args=[-M,s3://myawsbucket/config.xml,-m,mapred.tasktracker.map.tasks.maximum=2] Path=s3://elasticmapreduce/bootstrap-actions/configure-daemons,Name="Set the NameNode heap size",Args=[--namenode-heap-size=2048,--namenode-opts=-XX:GCTimeRatio=19] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
 
-**8. Create an Amazon EMR cluster with applications**
+**10. Create an Amazon EMR cluster with applications**
 
 - Create an Amazon EMR cluster with Hive, Pig, HBase, Ganglia, and Impala installed::
 
@@ -74,13 +86,13 @@
 
     aws emr create-cluster --applications Name=MapR,Args=--edition,m7,--version,3.0.2 --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
 
-**9. Restore HBase data from backup when creating an Amazon EMR cluster**
+**11. Restore HBase data from backup when creating an Amazon EMR cluster**
 
 -Command::
 
     aws emr create-cluster --applications Name=HBase --restore-from-hbase-backup Dir=s3://myBucket/myBackup,BackupVersion=myBackupVersion --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
 
-**10. To add Custom JAR steps to a cluster when creating an Amazon EMR cluster**
+**12. To add Custom JAR steps to a cluster when creating an Amazon EMR cluster**
 
 - Command::
 
@@ -94,7 +106,7 @@
 
     Type, Name, ActionOnFailure, Args
 
-**11. To add Streaming steps when creating an Amazon EMR cluster**
+**13. To add Streaming steps when creating an Amazon EMR cluster**
 
 - Command::
  
@@ -108,7 +120,7 @@
 
     Name, ActionOnFailure
 
-**12. To add Hive steps when creating an Amazon EMR cluster**
+**14. To add Hive steps when creating an Amazon EMR cluster**
 
 - Command::
 
@@ -122,7 +134,7 @@
 
     Name, ActionOnFailure, Version
 
-**13. To add Pig steps when creating an Amazon EMR cluster**
+**15. To add Pig steps when creating an Amazon EMR cluster**
 
 - Command::
 
@@ -136,7 +148,7 @@
 
     Name, ActionOnFailure, Version
 
-**14. To add Impala steps when creating an Amazon EMR cluster**
+**16. To add Impala steps when creating an Amazon EMR cluster**
 
 - Command::