Merge branch 'release-1.3.19'

* release-1.3.19:
  Bumping version to 1.3.19
  Update changelog with latest changes
  Test failure fix
  EMR: Update create-default-roles and Add --service-role & --use-default-roles
  EMR: Add unit tests for application
  EMR: Bug fix in install-applications and create-cluster
  Update Changelog with latest bugfixes
  Updates to SES topics and some fixes for autoscaling and rds.
  removed tab chars from a topic, and removed a bad link from another

CHANGELOG.rst

@@ -2,6 +2,20 @@
 CHANGELOG
 =========
 
+1.3.19
+======
+
+* feature:``aws ses``: Add support for delivery notifications
+* bugfix:Region Config: Fix issue for ``cn-north-1`` region
+  (`issue botocore 314 <https://github.com/boto/botocore/pull/314>`__)
+* bugfix:Amazon EC2 Credential File: Fix regression for parsing
+  EC2 credential file
+  (`issue botocore 315 <https://github.com/boto/botocore/pull/315>`__)
+* bugfix:Signature Version 2: Fix timestamp format when calculating
+  signature version 2 signatures
+  (`issue botocore 308 <https://github.com/boto/botocore/pull/308>`__)
+
+
 1.3.18
 ======
 

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.3.18'
+__version__ = '1.3.19'
 
 #
 # Get our data path to be added to botocore's search path

awscli/customizations/emr/applicationutils.py

@@ -24,10 +24,10 @@ def build_applications(parsed_applications, parsed_globals, ami_version=None):
     for app_config in parsed_applications:
         app_name = app_config['Name'].lower()
 
-        if app_name in constants.MAPR_NAMES:
+        if app_name in constants.SUPPORTED_PRODUCTS:
             app_list.append(
                 build_supported_product(
-                    app_config['Name'], app_config['Args']))
+                    app_config['Name'], app_config.get('Args')))
         elif app_name == constants.HIVE:
             hive_version = app_config.get('Version')
             if hive_version is None:

awscli/customizations/emr/constants.py

@@ -27,7 +27,7 @@
 SCRIPT_RUNNER_PATH = '/libs/script-runner/script-runner.jar'
 DEBUGGING_PATH = '/libs/state-pusher/0.1/fetch'
 DEBUGGING_NAME = 'Setup Hadoop Debugging'
-MAPR_NAMES = ['mapr', 'mapr-m3', 'mapr-m5', 'mapr-m7']
+SUPPORTED_PRODUCTS = ['mapr', 'mapr-m3', 'mapr-m5', 'mapr-m7', 'hue']
 
 MAX_BOOTSTRAP_ACTION_NUMBER = 16
 BOOTSTRAP_ACTION_NAME = 'Bootstrap action'

awscli/customizations/emr/createcluster.py

@@ -22,6 +22,8 @@
 from awscli.customizations.emr import exceptions
 from awscli.customizations.emr import applicationutils
 from awscli.customizations.emr import instancegroupsutils
+from awscli.customizations.emr.createdefaultroles import EMR_ROLE_NAME
+from awscli.customizations.emr.createdefaultroles import EC2_ROLE_NAME
 import re
 
 
@@ -52,6 +54,10 @@ class CreateCluster(BasicCommand):
          'help_text': helptext.CLUSTER_NAME},
         {'name': 'log-uri',
          'help_text': helptext.LOG_URI},
+        {'name': 'service-role',
+         'help_text': helptext.SERVICE_ROLE},
+        {'name': 'use-default-roles', 'action': 'store_true',
+         'help_text': helptext.USE_DEFAULT_ROLES},
         {'name': 'ec2-attributes',
          'help_text': helptext.EC2_ATTRIBUTES,
          'schema': argumentschema.EC2_ATTRIBUTES_SCHEMA},
@@ -104,6 +110,13 @@ def _run_main(self, parsed_args, parsed_globals):
         emrutils.apply_dict(
             params, 'AdditionalInfo', parsed_args.additional_info)
         emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)
+        if parsed_args.use_default_roles is True:
+            parsed_args.service_role = EMR_ROLE_NAME
+            if parsed_args.ec2_attributes is None:
+                parsed_args.ec2_attributes = {}
+            parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME
+
+        emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)
         instances_config = {}
         instances_config['InstanceGroups'] = \
             instancegroupsutils.build_instance_groups(
@@ -337,4 +350,4 @@ def _get_missing_applications_for_steps(self, specified_apps, parsed_args):
                     if step_type in allowed_app_steps and \
                             step_type not in specified_apps:
                         missing_apps.add(step['Type'].title())
-        return missing_apps
+        return missing_apps

awscli/customizations/emr/createdefaultroles.py

@@ -27,6 +27,7 @@
 
 
 EC2_ROLE_NAME = "EMR_EC2_DefaultRole"
+EMR_ROLE_NAME = "EMR_DefaultRole"
 
 EC2_ROLE_POLICY = {
     "Statement": [
@@ -49,6 +50,40 @@
 }
 
 
+EMR_ROLE_POLICY = {
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Action": [
+                "ec2:AuthorizeSecurityGroupIngress",
+                "ec2:CancelSpotInstanceRequests",
+                "ec2:CreateSecurityGroup",
+                "ec2:CreateTags",
+                "ec2:Describe*",
+                "ec2:DeleteTags",
+                "ec2:ModifyImageAttribute",
+                "ec2:ModifyInstanceAttribute",
+                "ec2:RequestSpotInstances",
+                "ec2:RunInstances",
+                "ec2:TerminateInstances",
+                "iam:PassRole",
+                "iam:ListRolePolicies",
+                "iam:GetRole",
+                "iam:GetRolePolicy",
+                "iam:ListInstanceProfiles",
+                "s3:Get*",
+                "s3:List*",
+                "s3:CreateBucket",
+                "sdb:BatchPutAttributes",
+                "sdb:Select"
+            ],
+            "Effect": "Allow",
+            "Resource": "*"
+        }
+    ]
+}
+
+
 def assume_role_policy(serviceprincipal):
     return {
         "Version": "2008-10-17",
@@ -98,7 +133,8 @@ def _get_regex_match_from_endpoint_host(endpoint_host):
 class CreateDefaultRoles(BasicCommand):
     NAME = "create-default-roles"
     DESCRIPTION = ('Creates the default IAM role ' +
-                   EC2_ROLE_NAME + ' which can be used when'
+                   EC2_ROLE_NAME + ' and ' +
+                   EMR_ROLE_NAME + ' which can be used when'
                    ' creating the cluster using the create-cluster command.')
     ARG_TABLE = [
         {'name': 'iam-endpoint',
@@ -112,6 +148,7 @@ class CreateDefaultRoles(BasicCommand):
 
     def _run_main(self, parsed_args, parsed_globals):
         ec2_result = None
+        emr_result = None
         self.iam = self._session.get_service('iam')
         self.iam_endpoint_url = parsed_args.iam_endpoint
         region = self._get_region(parsed_globals)
@@ -132,7 +169,7 @@ def _run_main(self, parsed_args, parsed_globals):
             LOG.debug('Role ' + role_name + ' exists.')
         else:
             LOG.debug('Role ' + role_name + ' does not exist.'
-                      ' Creating default role ' + role_name)
+                      ' Creating default role for EC2: ' + role_name)
             ec2_result = self._create_role_with_role_policy(
                 role_name, role_name, constants.EC2,
                 emrutils.dict_to_string(EC2_ROLE_POLICY),
@@ -151,10 +188,22 @@ def _run_main(self, parsed_args, parsed_globals):
                                                     instance_profile_name,
                                                     parsed_globals)
 
+        # Check if the default EMR Role exists.
+        role_name = EMR_ROLE_NAME
+        if self._check_if_role_exists(role_name, parsed_globals):
+            LOG.debug('Role ' + role_name + ' exists.')
+        else:
+            LOG.debug('Role ' + role_name + ' does not exist.'
+                      ' Creating default role for EMR: ' + role_name)
+            emr_result = self._create_role_with_role_policy(
+                role_name, role_name, constants.EMR,
+                emrutils.dict_to_string(EMR_ROLE_POLICY),
+                parsed_globals)
+
         emrutils.display_response(
             self._session,
             self._session.get_service('iam').get_operation('CreateRole'),
-            self._construct_result(ec2_result),
+            self._construct_result(ec2_result, emr_result),
             parsed_globals)
 
         return 0
@@ -166,11 +215,12 @@ def _check_for_iam_endpoint(self, region, iam_endpoint):
             if iam_endpoint is None:
                 raise exceptions.UnknownIamEndpointError(region=region)
 
-    def _construct_result(self, ec2_response):
+    def _construct_result(self, ec2_response, emr_response):
         result = []
         self._construct_role_and_role_policy_structure(
             result, ec2_response, EC2_ROLE_POLICY)
-
+        self._construct_role_and_role_policy_structure(
+            result, emr_response, EMR_ROLE_POLICY)
         return result
 
     def _construct_role_and_role_policy_structure(

awscli/customizations/emr/helptext.py

@@ -11,6 +11,7 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 
+from awscli.customizations.emr.createdefaultroles import EMR_ROLE_NAME
 from awscli.customizations.emr.createdefaultroles import EC2_ROLE_NAME
 
 TERMINATE_CLUSTERS = (
@@ -49,6 +50,19 @@
     'of the cluster. If a value is not provided, '
     'logs are not created.</p>')
 
+SERVICE_ROLE = (
+    '<p>Allows EMR to call other AWS Services such as EC2 on your behalf.</p>'
+    'To create the default Service Role <code>' + EMR_ROLE_NAME + '</code>,'
+    ' use <code>aws emr create-default-roles</code> command. </p>'
+    'This command will also create the default EC2 instance profile '
+    '<code>' + EC2_ROLE_NAME + '</code>.')
+
+USE_DEFAULT_ROLES = (
+    '<p>Uses --service-role=<code>' + EMR_ROLE_NAME + '</code>, and '
+    '--ec2-attributes </p> InstanceProfile=<code>' + EC2_ROLE_NAME + '</code>'
+    'To create the default service role and instance profile'
+    ' use <code>aws emr create-default-roles</code> command. </p>')
+
 AMI_VERSION = (
     '<p>The version number of the Amazon Machine Image (AMI) '
     'to use for Amazon EC2 instances in the cluster. '
@@ -75,12 +89,15 @@
     'Subnet cannot be specified together. To create the default '
     'instance profile <code>' + EC2_ROLE_NAME + '</code>,'
     ' use <code>aws emr create-default-roles</code> command. </p>'
+    'This command will also create the default EMR service role '
+    '<code>' + EMR_ROLE_NAME + '</code>.'
     '<li>KeyName - the name of the AWS EC2 key pair you are using '
     'to launch the cluster.</li>'
     '<li>AvailabilityZone - An isolated resource '
     'location within a region.</li>'
     '<li>SubnetId- Assign the EMR cluster to this Amazon VPC Subnet. </li>'
-    '<li>InstanceProfile - A container for the EC2 IAM Role. </li>'
+    '<li>InstanceProfile - Provides access to other AWS services such as S3,'
+    ' DynamoDB from EC2 instances that are launched by EMR.. </li>'
     )
 
 AUTO_TERMINATE = (

awscli/customizations/emr/installapplications.py

@@ -27,7 +27,7 @@ class InstallApplications(BasicCommand):
     ARG_TABLE = [
         {'name': 'cluster-id', 'required': True,
          'help_text': helptext.CLUSTER_ID},
-        {'name': 'apps', 'required': True,
+        {'name': 'applications', 'required': True,
          'help_text': helptext.INSTALL_APPLICATIONS,
          'schema': argumentschema.APPLICATIONS_SCHEMA},
     ]
@@ -38,9 +38,9 @@ def _run_main(self, parsed_args, parsed_globals):
 
         parameters = {'JobFlowId': parsed_args.cluster_id}
 
-        self._check_for_supported_apps(parsed_args.apps)
+        self._check_for_supported_apps(parsed_args.applications)
         parameters['Steps'] = applicationutils.build_applications(
-            parsed_args.apps, parsed_globals)[2]
+            parsed_args.applications, parsed_globals)[2]
 
         emrutils.call_and_display_response(self._session, 'AddJobFlowSteps',
                                            parameters, parsed_globals)

awscli/examples/autoscaling/put-scheduled-update-group-action.rst

@@ -2,11 +2,11 @@
 
 The following ``put-scheduled-update-group-action`` command adds a scheduled action to an Auto Scaling group::
 
-	aws autoscaling put-scheduled-update-group-action --auto-scaling-group-name basic-auto-scaling-group --scheduled-action-name sample-scheduled-action --start-time "2014-05-12T08:00:00Z" --end-time "2014-05-12T08:00:00Z" --min-size 2 --max-size 6 --desired-capacity 4
+    aws autoscaling put-scheduled-update-group-action --auto-scaling-group-name basic-auto-scaling-group --scheduled-action-name sample-scheduled-action --start-time "2014-05-12T08:00:00Z" --end-time "2014-05-12T08:00:00Z" --min-size 2 --max-size 6 --desired-capacity 4
 
 The following example creates a scheduled action to scale on a recurring schedule that is scheduled to execute at 00:30 hours on the first of January, June, and December every year::
 
-	aws autoscaling put-scheduled-update-group-action --auto-scaling-group-name basic-auto-scaling-group --scheduled-action-name sample-scheduled-action --recurrence "30 0 1 1,6,12 0" --min-size 2 --max-size 6 --desired-capacity 4
+    aws autoscaling put-scheduled-update-group-action --auto-scaling-group-name basic-auto-scaling-group --scheduled-action-name sample-scheduled-action --recurrence "30 0 1 1,6,12 0" --min-size 2 --max-size 6 --desired-capacity 4
 
 For more information, see `Scheduled Scaling`_ in the *Auto Scaling Developer Guide*.