diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 038034336466..59b990d8bdfa 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,20 @@ CHANGELOG ========= +1.3.19 +====== + +* feature:``aws ses``: Add support for delivery notifications +* bugfix:Region Config: Fix issue for ``cn-north-1`` region + (`issue botocore 314 `__) +* bugfix:Amazon EC2 Credential File: Fix regression for parsing + EC2 credential file + (`issue botocore 315 `__) +* bugfix:Signature Version 2: Fix timestamp format when calculating + signature version 2 signatures + (`issue botocore 308 `__) + + 1.3.18 ====== diff --git a/awscli/__init__.py b/awscli/__init__.py index 921c0b600bbe..b32446081050 100644 --- a/awscli/__init__.py +++ b/awscli/__init__.py @@ -17,7 +17,7 @@ """ import os -__version__ = '1.3.18' +__version__ = '1.3.19' # # Get our data path to be added to botocore's search path diff --git a/awscli/customizations/emr/applicationutils.py b/awscli/customizations/emr/applicationutils.py index d6b8b40460ba..ce5c87ce96d5 100644 --- a/awscli/customizations/emr/applicationutils.py +++ b/awscli/customizations/emr/applicationutils.py @@ -24,10 +24,10 @@ def build_applications(parsed_applications, parsed_globals, ami_version=None): for app_config in parsed_applications: app_name = app_config['Name'].lower() - if app_name in constants.MAPR_NAMES: + if app_name in constants.SUPPORTED_PRODUCTS: app_list.append( build_supported_product( - app_config['Name'], app_config['Args'])) + app_config['Name'], app_config.get('Args'))) elif app_name == constants.HIVE: hive_version = app_config.get('Version') if hive_version is None: diff --git a/awscli/customizations/emr/constants.py b/awscli/customizations/emr/constants.py index 4d13239b0761..bb1709fcc8c0 100644 --- a/awscli/customizations/emr/constants.py +++ b/awscli/customizations/emr/constants.py @@ -27,7 +27,7 @@ SCRIPT_RUNNER_PATH = '/libs/script-runner/script-runner.jar' DEBUGGING_PATH = '/libs/state-pusher/0.1/fetch' DEBUGGING_NAME = 'Setup Hadoop Debugging' -MAPR_NAMES = ['mapr', 'mapr-m3', 'mapr-m5', 'mapr-m7'] +SUPPORTED_PRODUCTS = ['mapr', 'mapr-m3', 'mapr-m5', 'mapr-m7', 'hue'] MAX_BOOTSTRAP_ACTION_NUMBER = 16 BOOTSTRAP_ACTION_NAME = 'Bootstrap action' diff --git a/awscli/customizations/emr/createcluster.py b/awscli/customizations/emr/createcluster.py index 91f7c3b3b27b..d18564e5101b 100644 --- a/awscli/customizations/emr/createcluster.py +++ b/awscli/customizations/emr/createcluster.py @@ -22,6 +22,8 @@ from awscli.customizations.emr import exceptions from awscli.customizations.emr import applicationutils from awscli.customizations.emr import instancegroupsutils +from awscli.customizations.emr.createdefaultroles import EMR_ROLE_NAME +from awscli.customizations.emr.createdefaultroles import EC2_ROLE_NAME import re @@ -52,6 +54,10 @@ class CreateCluster(BasicCommand): 'help_text': helptext.CLUSTER_NAME}, {'name': 'log-uri', 'help_text': helptext.LOG_URI}, + {'name': 'service-role', + 'help_text': helptext.SERVICE_ROLE}, + {'name': 'use-default-roles', 'action': 'store_true', + 'help_text': helptext.USE_DEFAULT_ROLES}, {'name': 'ec2-attributes', 'help_text': helptext.EC2_ATTRIBUTES, 'schema': argumentschema.EC2_ATTRIBUTES_SCHEMA}, @@ -104,6 +110,13 @@ def _run_main(self, parsed_args, parsed_globals): emrutils.apply_dict( params, 'AdditionalInfo', parsed_args.additional_info) emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri) + if parsed_args.use_default_roles is True: + parsed_args.service_role = EMR_ROLE_NAME + if parsed_args.ec2_attributes is None: + parsed_args.ec2_attributes = {} + parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME + + emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role) instances_config = {} instances_config['InstanceGroups'] = \ instancegroupsutils.build_instance_groups( @@ -337,4 +350,4 @@ def _get_missing_applications_for_steps(self, specified_apps, parsed_args): if step_type in allowed_app_steps and \ step_type not in specified_apps: missing_apps.add(step['Type'].title()) - return missing_apps \ No newline at end of file + return missing_apps diff --git a/awscli/customizations/emr/createdefaultroles.py b/awscli/customizations/emr/createdefaultroles.py index c675de4a1034..2c8aa69c0c86 100644 --- a/awscli/customizations/emr/createdefaultroles.py +++ b/awscli/customizations/emr/createdefaultroles.py @@ -27,6 +27,7 @@ EC2_ROLE_NAME = "EMR_EC2_DefaultRole" +EMR_ROLE_NAME = "EMR_DefaultRole" EC2_ROLE_POLICY = { "Statement": [ @@ -49,6 +50,40 @@ } +EMR_ROLE_POLICY = { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CancelSpotInstanceRequests", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:Describe*", + "ec2:DeleteTags", + "ec2:ModifyImageAttribute", + "ec2:ModifyInstanceAttribute", + "ec2:RequestSpotInstances", + "ec2:RunInstances", + "ec2:TerminateInstances", + "iam:PassRole", + "iam:ListRolePolicies", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:ListInstanceProfiles", + "s3:Get*", + "s3:List*", + "s3:CreateBucket", + "sdb:BatchPutAttributes", + "sdb:Select" + ], + "Effect": "Allow", + "Resource": "*" + } + ] +} + + def assume_role_policy(serviceprincipal): return { "Version": "2008-10-17", @@ -98,7 +133,8 @@ def _get_regex_match_from_endpoint_host(endpoint_host): class CreateDefaultRoles(BasicCommand): NAME = "create-default-roles" DESCRIPTION = ('Creates the default IAM role ' + - EC2_ROLE_NAME + ' which can be used when' + EC2_ROLE_NAME + ' and ' + + EMR_ROLE_NAME + ' which can be used when' ' creating the cluster using the create-cluster command.') ARG_TABLE = [ {'name': 'iam-endpoint', @@ -112,6 +148,7 @@ class CreateDefaultRoles(BasicCommand): def _run_main(self, parsed_args, parsed_globals): ec2_result = None + emr_result = None self.iam = self._session.get_service('iam') self.iam_endpoint_url = parsed_args.iam_endpoint region = self._get_region(parsed_globals) @@ -132,7 +169,7 @@ def _run_main(self, parsed_args, parsed_globals): LOG.debug('Role ' + role_name + ' exists.') else: LOG.debug('Role ' + role_name + ' does not exist.' - ' Creating default role ' + role_name) + ' Creating default role for EC2: ' + role_name) ec2_result = self._create_role_with_role_policy( role_name, role_name, constants.EC2, emrutils.dict_to_string(EC2_ROLE_POLICY), @@ -151,10 +188,22 @@ def _run_main(self, parsed_args, parsed_globals): instance_profile_name, parsed_globals) + # Check if the default EMR Role exists. + role_name = EMR_ROLE_NAME + if self._check_if_role_exists(role_name, parsed_globals): + LOG.debug('Role ' + role_name + ' exists.') + else: + LOG.debug('Role ' + role_name + ' does not exist.' + ' Creating default role for EMR: ' + role_name) + emr_result = self._create_role_with_role_policy( + role_name, role_name, constants.EMR, + emrutils.dict_to_string(EMR_ROLE_POLICY), + parsed_globals) + emrutils.display_response( self._session, self._session.get_service('iam').get_operation('CreateRole'), - self._construct_result(ec2_result), + self._construct_result(ec2_result, emr_result), parsed_globals) return 0 @@ -166,11 +215,12 @@ def _check_for_iam_endpoint(self, region, iam_endpoint): if iam_endpoint is None: raise exceptions.UnknownIamEndpointError(region=region) - def _construct_result(self, ec2_response): + def _construct_result(self, ec2_response, emr_response): result = [] self._construct_role_and_role_policy_structure( result, ec2_response, EC2_ROLE_POLICY) - + self._construct_role_and_role_policy_structure( + result, emr_response, EMR_ROLE_POLICY) return result def _construct_role_and_role_policy_structure( diff --git a/awscli/customizations/emr/helptext.py b/awscli/customizations/emr/helptext.py index db6c1229e5db..12b6ad5d4f32 100644 --- a/awscli/customizations/emr/helptext.py +++ b/awscli/customizations/emr/helptext.py @@ -11,6 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. +from awscli.customizations.emr.createdefaultroles import EMR_ROLE_NAME from awscli.customizations.emr.createdefaultroles import EC2_ROLE_NAME TERMINATE_CLUSTERS = ( @@ -49,6 +50,19 @@ 'of the cluster. If a value is not provided, ' 'logs are not created.

') +SERVICE_ROLE = ( + '

Allows EMR to call other AWS Services such as EC2 on your behalf.

' + 'To create the default Service Role ' + EMR_ROLE_NAME + ',' + ' use aws emr create-default-roles command.

' + 'This command will also create the default EC2 instance profile ' + '' + EC2_ROLE_NAME + '.') + +USE_DEFAULT_ROLES = ( + '

Uses --service-role=' + EMR_ROLE_NAME + ', and ' + '--ec2-attributes

InstanceProfile=' + EC2_ROLE_NAME + '' + 'To create the default service role and instance profile' + ' use aws emr create-default-roles command.

') + AMI_VERSION = ( '

The version number of the Amazon Machine Image (AMI) ' 'to use for Amazon EC2 instances in the cluster. ' @@ -75,12 +89,15 @@ 'Subnet cannot be specified together. To create the default ' 'instance profile ' + EC2_ROLE_NAME + ',' ' use aws emr create-default-roles command.

' + 'This command will also create the default EMR service role ' + '' + EMR_ROLE_NAME + '.' '
  • KeyName - the name of the AWS EC2 key pair you are using ' 'to launch the cluster.
    • ' '
  • AvailabilityZone - An isolated resource ' 'location within a region.
    • ' '
  • SubnetId- Assign the EMR cluster to this Amazon VPC Subnet.
    • ' - '
  • InstanceProfile - A container for the EC2 IAM Role.
    • ' + '
  • InstanceProfile - Provides access to other AWS services such as S3,' + ' DynamoDB from EC2 instances that are launched by EMR..
    • ' ) AUTO_TERMINATE = ( diff --git a/awscli/customizations/emr/installapplications.py b/awscli/customizations/emr/installapplications.py index 504895804da7..5eaf53119a91 100644 --- a/awscli/customizations/emr/installapplications.py +++ b/awscli/customizations/emr/installapplications.py @@ -27,7 +27,7 @@ class InstallApplications(BasicCommand): ARG_TABLE = [ {'name': 'cluster-id', 'required': True, 'help_text': helptext.CLUSTER_ID}, - {'name': 'apps', 'required': True, + {'name': 'applications', 'required': True, 'help_text': helptext.INSTALL_APPLICATIONS, 'schema': argumentschema.APPLICATIONS_SCHEMA}, ] @@ -38,9 +38,9 @@ def _run_main(self, parsed_args, parsed_globals): parameters = {'JobFlowId': parsed_args.cluster_id} - self._check_for_supported_apps(parsed_args.apps) + self._check_for_supported_apps(parsed_args.applications) parameters['Steps'] = applicationutils.build_applications( - parsed_args.apps, parsed_globals)[2] + parsed_args.applications, parsed_globals)[2] emrutils.call_and_display_response(self._session, 'AddJobFlowSteps', parameters, parsed_globals) diff --git a/awscli/examples/autoscaling/put-scheduled-update-group-action.rst b/awscli/examples/autoscaling/put-scheduled-update-group-action.rst index fea18260856a..81e1c0f0949c 100644 --- a/awscli/examples/autoscaling/put-scheduled-update-group-action.rst +++ b/awscli/examples/autoscaling/put-scheduled-update-group-action.rst @@ -2,11 +2,11 @@ The following ``put-scheduled-update-group-action`` command adds a scheduled action to an Auto Scaling group:: - aws autoscaling put-scheduled-update-group-action --auto-scaling-group-name basic-auto-scaling-group --scheduled-action-name sample-scheduled-action --start-time "2014-05-12T08:00:00Z" --end-time "2014-05-12T08:00:00Z" --min-size 2 --max-size 6 --desired-capacity 4 + aws autoscaling put-scheduled-update-group-action --auto-scaling-group-name basic-auto-scaling-group --scheduled-action-name sample-scheduled-action --start-time "2014-05-12T08:00:00Z" --end-time "2014-05-12T08:00:00Z" --min-size 2 --max-size 6 --desired-capacity 4 The following example creates a scheduled action to scale on a recurring schedule that is scheduled to execute at 00:30 hours on the first of January, June, and December every year:: - aws autoscaling put-scheduled-update-group-action --auto-scaling-group-name basic-auto-scaling-group --scheduled-action-name sample-scheduled-action --recurrence "30 0 1 1,6,12 0" --min-size 2 --max-size 6 --desired-capacity 4 + aws autoscaling put-scheduled-update-group-action --auto-scaling-group-name basic-auto-scaling-group --scheduled-action-name sample-scheduled-action --recurrence "30 0 1 1,6,12 0" --min-size 2 --max-size 6 --desired-capacity 4 For more information, see `Scheduled Scaling`_ in the *Auto Scaling Developer Guide*. diff --git a/awscli/examples/emr/create-cluster-examples.rst b/awscli/examples/emr/create-cluster-examples.rst index c27ace380e8a..d1f5339bd1bf 100644 --- a/awscli/examples/emr/create-cluster-examples.rst +++ b/awscli/examples/emr/create-cluster-examples.rst @@ -4,13 +4,25 @@ aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate -**2. Create an Amazon EMR cluster with MASTER, CORE, and TASK instance groups** +**2. Create an Amazon EMR cluster with ServiceRole and InstanceProfile + +- Command:: + + aws emr create-cluster --ami-version 3.1.0 --service-role EMR_DefaultRole --ec2-attributes InstanceProfiles=EC2_EMR_DefaultRoles --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate + +**3. Create an Amazon EMR cluster with default roles + +- Command:: + + aws emr create-cluster --ami-version 3.1.0 --use-default-roles --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate + +**4. Create an Amazon EMR cluster with MASTER, CORE, and TASK instance groups** - Command:: aws emr create-cluster --ami-version 3.1.0 --auto-terminate --instance-groups Name=Master,InstanceGroupType=MASTER,InstanceType=m3.xlarge,InstanceCount=1 Name=Core,InstanceGroupType=CORE,InstanceType=m3.xlarge,InstanceCount=2 Name=Task,InstanceGroupType=TASK,InstanceType=m3.xlarge,InstanceCount=2 -**3. Specify whether the cluster should terminate after completing all the steps** +**5. Specify whether the cluster should terminate after completing all the steps** - Create an Amazon EMR cluster that will terminate after completing all the steps:: @@ -20,7 +32,7 @@ aws emr create-cluster --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --no-auto-terminate -**4. Specify EC2 Attributes** +**6. Specify EC2 Attributes** - Create an Amazon EMR cluster with Amazon EC2 Key Pair "myKey" and instance profile "myProfile":: @@ -34,13 +46,13 @@ aws emr create-cluster --ec2-attributes AvailabilityZone=us-west-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate -**5. Enable debugging and specify a Log URI** +**7. Enable debugging and specify a Log URI** - Command:: aws emr create-cluster --enable-debugging --log-uri s3://myBucket/myLog --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate -**6. Add tags when creating an Amazon EMR cluster** +**8. Add tags when creating an Amazon EMR cluster** - Add a list of tags:: @@ -50,7 +62,7 @@ aws emr describe-cluster --cluster-id j-XXXXXXYY --query Cluster.Tags -**7. Add a list of bootstrap actions when creating an Amazon EMR Cluster** +**9. Add a list of bootstrap actions when creating an Amazon EMR Cluster** - Command:: @@ -60,7 +72,7 @@ aws emr create-cluster --bootstrap-actions Path=s3://elasticmapreduce/bootstrap-actions/configure-hadoop,Name="Change the maximum number of map tasks",Args=[-M,s3://myawsbucket/config.xml,-m,mapred.tasktracker.map.tasks.maximum=2] Path=s3://elasticmapreduce/bootstrap-actions/configure-daemons,Name="Set the NameNode heap size",Args=[--namenode-heap-size=2048,--namenode-opts=-XX:GCTimeRatio=19] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate -**8. Create an Amazon EMR cluster with applications** +**10. Create an Amazon EMR cluster with applications** - Create an Amazon EMR cluster with Hive, Pig, HBase, Ganglia, and Impala installed:: @@ -74,13 +86,13 @@ aws emr create-cluster --applications Name=MapR,Args=--edition,m7,--version,3.0.2 --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate -**9. Restore HBase data from backup when creating an Amazon EMR cluster** +**11. Restore HBase data from backup when creating an Amazon EMR cluster** -Command:: aws emr create-cluster --applications Name=HBase --restore-from-hbase-backup Dir=s3://myBucket/myBackup,BackupVersion=myBackupVersion --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate -**10. To add Custom JAR steps to a cluster when creating an Amazon EMR cluster** +**12. To add Custom JAR steps to a cluster when creating an Amazon EMR cluster** - Command:: @@ -94,7 +106,7 @@ Type, Name, ActionOnFailure, Args -**11. To add Streaming steps when creating an Amazon EMR cluster** +**13. To add Streaming steps when creating an Amazon EMR cluster** - Command:: @@ -108,7 +120,7 @@ Name, ActionOnFailure -**12. To add Hive steps when creating an Amazon EMR cluster** +**14. To add Hive steps when creating an Amazon EMR cluster** - Command:: @@ -122,7 +134,7 @@ Name, ActionOnFailure, Version -**13. To add Pig steps when creating an Amazon EMR cluster** +**15. To add Pig steps when creating an Amazon EMR cluster** - Command:: @@ -136,7 +148,7 @@ Name, ActionOnFailure, Version -**14. To add Impala steps when creating an Amazon EMR cluster** +**16. To add Impala steps when creating an Amazon EMR cluster** - Command:: diff --git a/awscli/examples/emr/create-cluster-synopsis.rst b/awscli/examples/emr/create-cluster-synopsis.rst index f8a22c77e7b2..b37981df62f4 100644 --- a/awscli/examples/emr/create-cluster-synopsis.rst +++ b/awscli/examples/emr/create-cluster-synopsis.rst @@ -2,10 +2,12 @@ --ami-version --instance-groups --auto-terminate | --no-auto-terminate + [--use-default-role] + [--service-role ] [--name ] [--log-uri ] [--additional-info ] - [--ec2-attributes ] + [--ec2-attributes ] [--termination-protected | --no-termination-protected] [--visible-to-all-users | --no-visible-to-all-users] [--enable-debugging | --no-enable-debugging] diff --git a/awscli/examples/emr/create-default-roles.rst b/awscli/examples/emr/create-default-roles.rst index 4600f828825a..77b73ea2da1e 100644 --- a/awscli/examples/emr/create-default-roles.rst +++ b/awscli/examples/emr/create-default-roles.rst @@ -10,48 +10,102 @@ If the role does not exist then the output will be: - [ - { - "RolePolicy": { - "Statement": [ - { - "Action": [ - "cloudwatch:*", - "dynamodb:*", - "ec2:Describe*", - "elasticmapreduce:Describe*", - "rds:Describe*", - "s3:*", - "sdb:*", - "sns:*", - "sqs:*" - ], - "Resource": [ - "*" - ], - "Effect": "Allow" - } - ] - }, - "Role": { - "AssumeRolePolicyDocument": { - "Version": "2008-10-17", - "Statement": [ - { - "Action": "sts:AssumeRole", - "Sid": "", - "Effect": "Allow", - "Principal": { - "Service": "ec2.amazonaws.com" - } - } - ] - }, - "RoleId": "AROLORKUOAL65X57SBWFK", - "CreateDate": "2014-05-07T00:02:06.154Z", - "RoleName": "EMR_EC2_DefaultRole", - "Path": "/", - "Arn": "arn:aws:iam::176430881729:role/EMR_EC2_DefaultRole" - } - } - ] \ No newline at end of file + [ + { + "RolePolicy": { + "Statement": [ + { + "Action": [ + "cloudwatch:*", + "dynamodb:*", + "ec2:Describe*", + "elasticmapreduce:Describe*", + "rds:Describe*", + "s3:*", + "sdb:*", + "sns:*", + "sqs:*" + ], + "Resource": [ + "*" + ], + "Effect": "Allow" + } + ] + }, + "Role": { + "AssumeRolePolicyDocument": { + "Version": "2008-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + } + } + ] + }, + "RoleId": "AROAJEATMSYEQGRVYQDX4", + "CreateDate": "2014-06-14T01:05:15.356Z", + "RoleName": "EMR_EC2_DefaultRole", + "Path": "/", + "Arn": "arn:aws:iam::176430881729:role/EMR_EC2_DefaultRole" + } + }, + { + "RolePolicy": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CancelSpotInstanceRequests", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:Describe*", + "ec2:DeleteTags", + "ec2:ModifyImageAttribute", + "ec2:ModifyInstanceAttribute", + "ec2:RequestSpotInstances", + "ec2:RunInstances", + "ec2:TerminateInstances", + "iam:PassRole", + "iam:ListRolePolicies", + "iam:GetRole", + "iam:GetRolePolicy", + "iam:ListInstanceProfiles", + "s3:Get*", + "s3:List*", + "s3:CreateBucket", + "sdb:BatchPutAttributes", + "sdb:Select" + ], + "Resource": "*", + "Effect": "Allow" + } + ] + }, + "Role": { + "AssumeRolePolicyDocument": { + "Version": "2008-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Sid": "", + "Effect": "Allow", + "Principal": { + "Service": "elasticmapreduce.amazonaws.com" + } + } + ] + }, + "RoleId": "AROAJRHC33G6KRX5D5QF2", + "CreateDate": "2014-06-14T01:05:17.464Z", + "RoleName": "EMR_DefaultRole", + "Path": "/", + "Arn": "arn:aws:iam::176430881729:role/EMR_DefaultRole" + } + } + ] diff --git a/awscli/examples/rds/create-db-instance.rst b/awscli/examples/rds/create-db-instance.rst index ecac070e5483..966cb4fc1a10 100644 --- a/awscli/examples/rds/create-db-instance.rst +++ b/awscli/examples/rds/create-db-instance.rst @@ -53,7 +53,3 @@ This command outputs a JSON block that indicates that the DB instance was create } } -For more information, see `Create an Amazon RDS DB Instance`_ in the *AWS Command Line Interface User Guide*. - -.. _`Create an Amazon RDS DB Instance`: http://docs.aws.amazon.com/cli/latest/userguide/cli-rds-create-instance.html - diff --git a/awscli/examples/ses/get-identity-notification-attributes.rst b/awscli/examples/ses/get-identity-notification-attributes.rst index 0d2921ca2993..5d06778da8cc 100644 --- a/awscli/examples/ses/get-identity-notification-attributes.rst +++ b/awscli/examples/ses/get-identity-notification-attributes.rst @@ -11,7 +11,8 @@ Output:: "user1@example.com": { "ForwardingEnabled": false, "ComplaintTopic": "arn:aws:sns:us-east-1:EXAMPLE65304:MyTopic", - "BounceTopic": "arn:aws:sns:us-east-1:EXAMPLE65304:MyTopic" + "BounceTopic": "arn:aws:sns:us-east-1:EXAMPLE65304:MyTopic", + "DeliveryTopic": "arn:aws:sns:us-east-1:EXAMPLE65304:MyTopic" }, "user2@example.com": { "ForwardingEnabled": true @@ -19,10 +20,10 @@ Output:: } } -If email feedback forwarding is disabled, then this command returns the Amazon Resource Names (ARNs) of the SNS topics that bounce and complaint notifications are sent to. +This command returns the status of email feedback forwarding and, if applicable, the Amazon Resource Names (ARNs) of the Amazon SNS topics that bounce, complaint, and delivery notifications are sent to. If you call this command with an identity that you have never submitted for verification, that identity won't appear in the output. -For more information about bounce and complaint notifications, see `Bounce and Complaint Notifications in Amazon SES`_ in the *Amazon Simple Email Service Developer Guide*. +For more information about notifications, see `Using Notifications With Amazon SES`_ in the *Amazon Simple Email Service Developer Guide*. -.. _`Bounce and Complaint Notifications in Amazon SES`: http://docs.aws.amazon.com/ses/latest/DeveloperGuide/bounce-complaint-notifications.html +.. _`Using Notifications With Amazon SES`: http://docs.aws.amazon.com/ses/latest/DeveloperGuide/notifications.html diff --git a/awscli/examples/ses/set-identity-feedback-forwarding-enabled.rst b/awscli/examples/ses/set-identity-feedback-forwarding-enabled.rst index 0e21e0416eb7..c3000fdf3cd8 100644 --- a/awscli/examples/ses/set-identity-feedback-forwarding-enabled.rst +++ b/awscli/examples/ses/set-identity-feedback-forwarding-enabled.rst @@ -1,10 +1,12 @@ -**To enable or disable email feedback forwarding for an Amazon SES verified identity** +**To enable or disable bounce and complaint email feedback forwarding for an Amazon SES verified identity** -The following example uses the ``set-identity-feedback-forwarding-enabled`` command to enable a verified email address to receive feedback notifications by email:: +The following example uses the ``set-identity-feedback-forwarding-enabled`` command to enable a verified email address to receive bounce and complaint notifications by email:: aws ses set-identity-feedback-forwarding-enabled --identity user@example.com --forwarding-enabled -For more information about feedback notifications, see `Bounce and Complaint Notifications in Amazon SES`_ in the *Amazon Simple Email Service Developer Guide*. +You are required to receive bounce and complaint notifications via either Amazon SNS or email feedback forwarding, so you can only disable email feedback forwarding if you select an Amazon SNS topic for both bounce and complaint notifications. -.. _`Bounce and Complaint Notifications in Amazon SES`: http://docs.aws.amazon.com/ses/latest/DeveloperGuide/bounce-complaint-notifications.html +For more information about notifications, see `Using Notifications With Amazon SES`_ in the *Amazon Simple Email Service Developer Guide*. + +.. _`Using Notifications With Amazon SES`: http://docs.aws.amazon.com/ses/latest/DeveloperGuide/notifications.html diff --git a/awscli/examples/ses/set-identity-notification-topic.rst b/awscli/examples/ses/set-identity-notification-topic.rst index 41e2889b233f..2e5c8bacb786 100644 --- a/awscli/examples/ses/set-identity-notification-topic.rst +++ b/awscli/examples/ses/set-identity-notification-topic.rst @@ -1,10 +1,10 @@ -**To set the Amazon SNS topic to which Amazon SES will publish bounce and complaint notifications for a verified identity** +**To set the Amazon SNS topic to which Amazon SES will publish bounce, complaint, and/or delivery notifications for a verified identity** The following example uses the ``set-identity-notification-topic`` command to specify the Amazon SNS topic to which a verified email address will receive bounce notifications:: aws ses set-identity-notification-topic --identity user@example.com --notification-type Bounce --sns-topic arn:aws:sns:us-east-1:EXAMPLE65304:MyTopic -For more information about feedback notifications, see `Bounce and Complaint Notifications in Amazon SES`_ in the *Amazon Simple Email Service Developer Guide*. +For more information about notifications, see `Using Notifications With Amazon SES`_ in the *Amazon Simple Email Service Developer Guide*. -.. _`Bounce and Complaint Notifications in Amazon SES`: http://docs.aws.amazon.com/ses/latest/DeveloperGuide/bounce-complaint-notifications.html +.. _`Using Notifications With Amazon SES`: http://docs.aws.amazon.com/ses/latest/DeveloperGuide/notifications.html diff --git a/doc/source/conf.py b/doc/source/conf.py index e3a7e64a75c7..34d4161f139d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -52,7 +52,7 @@ # The short X.Y version. version = '1.3.' # The full version, including alpha/beta/rc tags. -release = '1.3.18' +release = '1.3.19' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 3d291eaac8ac..242054c42a62 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import awscli -requires = ['botocore>=0.52.0,<0.53.0', +requires = ['botocore>=0.53.0,<0.54.0', 'bcdoc>=0.12.0,<0.13.0', 'six>=1.1.0', 'colorama==0.2.5', diff --git a/tests/unit/customizations/emr/test_create_cluster.py b/tests/unit/customizations/emr/test_create_cluster.py index 46259ef46116..3c678258404c 100644 --- a/tests/unit/customizations/emr/test_create_cluster.py +++ b/tests/unit/customizations/emr/test_create_cluster.py @@ -50,7 +50,7 @@ 'InstanceGroupType=TASK,Name=TASK,' 'InstanceCount=1,InstanceType=m1.large ') -DEFAULT_CMD = 'emr create-cluster --auto-terminate --ami-version 3.0.4 --instance-groups ' + \ +DEFAULT_CMD = 'emr create-cluster --auto-terminate --ami-version 3.0.4 --use-default-roles --instance-groups ' + \ DEFAULT_INSTANCE_GROUPS_ARG DEFAULT_INSTANCES = {'KeepJobFlowAliveWhenNoSteps': False, @@ -58,12 +58,17 @@ 'InstanceGroups': DEFAULT_INSTANCE_GROUPS } +EC2_ROLE_NAME = "EMR_EC2_DefaultRole" +EMR_ROLE_NAME = "EMR_DefaultRole" + DEFAULT_RESULT = \ { 'Name': DEFAULT_CLUSTER_NAME, 'Instances': DEFAULT_INSTANCES, 'AmiVersion': '3.0.4', 'VisibleToAllUsers': False, + 'JobFlowRole': EC2_ROLE_NAME, + 'ServiceRole': EMR_ROLE_NAME, 'Tags': [] } @@ -146,6 +151,17 @@ 'Args': ['--edition', 'm5', '--version', '3.0.2'] } +INSTALL_SUPPORTED_PRODUCTS = [ + { + 'Name': 'hue', + 'Args': ['--hue-config', 's3://elasticmapreduce/hue-config'] + }, + { + 'Name': 'mapr', + 'Args': ['--edition', 'm7'] + } +] + CUSTOM_JAR_STEP = { 'Name': 'Custom JAR', 'ActionOnFailure': 'CONTINUE', @@ -275,6 +291,29 @@ class TestCreateCluster(BaseAWSCommandParamsTest): def test_default_cmd(self): self.assert_params_for_cmd(DEFAULT_CMD, DEFAULT_RESULT) + def test_cluster_with_out_service_role_and_instance_profile(self): + cmd = ('emr create-cluster --auto-terminate --ami-version 3.0.4' + ' --instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG) + result = copy.deepcopy(DEFAULT_RESULT) + del result['JobFlowRole'] + del result['ServiceRole'] + self.assert_params_for_cmd(cmd, result) + + def test_cluster_with_service_role_and_instance_profile(self): + cmd = ('emr create-cluster --auto-terminate --ami-version 3.0.4' + ' --instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG + + ' --service-role ServiceRole --ec2-attributes ' + 'InstanceProfile=Ec2_InstanceProfile') + result = copy.deepcopy(DEFAULT_RESULT) + result['JobFlowRole'] = 'Ec2_InstanceProfile' + result['ServiceRole'] = 'ServiceRole' + self.assert_params_for_cmd(cmd, result) + + def test_cluster_default_roles_overrides(self): + cmd = (DEFAULT_CMD + '--service-role ServiceRole ' + '--ec2-attributes InstanceProfile=Ec2_InstanceProfile') + self.assert_params_for_cmd(cmd, DEFAULT_RESULT) + def test_cluster_name_no_space(self): cmd = DEFAULT_CMD + '--name MyCluster' result = copy.deepcopy(DEFAULT_RESULT) @@ -308,7 +347,8 @@ def test_additional_info(self): self.assert_params_for_cmd(cmd, result) def test_no_auto_terminte(self): - cmd = ('emr create-cluster --ami-version 3.0.4 --no-auto-terminate' + + cmd = ('emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--no-auto-terminate' + ' --instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG) result = copy.deepcopy(DEFAULT_RESULT) instances = copy.deepcopy(DEFAULT_INSTANCES) @@ -326,7 +366,8 @@ def test_auto_terminate_and_no_auto_terminate(self): self.assertEquals(expected_error_msg, result[1]) def test_missing_auto_terminate_or_no_auto_terminate(self): - cmd = (self.prefix + '--ami-version 3.0.4 --instance-groups ' + + cmd = (self.prefix + '--use-default-roles --ami-version 3.0.4 ' + '--instance-groups ' + DEFAULT_INSTANCE_GROUPS_ARG) expected_error_msg = ( '\naws: error: Must specify one of the following boolean options:' @@ -419,7 +460,8 @@ def test_enable_debugging_and_no_enable_debugging(self): def test_instance_groups_default_name_market(self): cmd = ( - 'emr create-cluster --ami-version 3.0.4 --auto-terminate ' + 'emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--auto-terminate ' '--instance-groups ' 'InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m1.large ' 'InstanceGroupType=CORE,InstanceCount=1,InstanceType=m1.large ' @@ -428,7 +470,8 @@ def test_instance_groups_default_name_market(self): def test_instance_groups_instance_group_type_mismatch_cases(self): cmd = ( - 'emr create-cluster --ami-version 3.0.4 --auto-terminate ' + 'emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--auto-terminate ' '--instance-groups ' 'Name=MASTER,InstanceGroupType=MaSter,InstanceCount=1,' 'InstanceType=m1.large Name=CORE,InstanceGroupType=cORE,' @@ -438,7 +481,8 @@ def test_instance_groups_instance_group_type_mismatch_cases(self): def test_instance_groups_missing_instance_group_type_error(self): cmd = ( - 'emr create-cluster --ami-version 3.0.4 --auto-terminate ' + 'emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--auto-terminate ' '--instance-groups ' 'Name=Master,InstanceCount=1,InstanceType=m1.small') expect_error_msg = ( @@ -449,7 +493,8 @@ def test_instance_groups_missing_instance_group_type_error(self): def test_instance_groups_missing_instance_type_error(self): cmd = ( - 'emr create-cluster --ami-version 3.0.4 --auto-terminate ' + 'emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--auto-terminate ' '--instance-groups ' 'Name=Master,InstanceGroupType=MASTER,InstanceCount=1') expect_error_msg = ( @@ -460,7 +505,8 @@ def test_instance_groups_missing_instance_type_error(self): def test_instance_groups_missing_instance_count_error(self): cmd = ( - 'emr create-cluster --ami-version 3.0.4 --auto-terminate ' + 'emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--auto-terminate ' '--instance-groups ' 'Name=Master,InstanceGroupType=MASTER,InstanceType=m1.xlarge') expect_error_msg = ( @@ -472,7 +518,8 @@ def test_instance_groups_missing_instance_count_error(self): def test_instance_groups_from_json_file(self): data_path = os.path.join( os.path.dirname(__file__), 'input_instance_groups.json') - cmd = ('emr create-cluster --ami-version 3.0.4 --auto-terminate ' + cmd = ('emr create-cluster --use-default-roles --ami-version 3.0.4 ' + '--auto-terminate ' '--instance-groups file://' + data_path) result = copy.deepcopy(DEFAULT_RESULT) result['Instances']['InstanceGroups'] = \ @@ -502,11 +549,11 @@ def test_instance_groups_from_json_file(self): def test_ec2_attributes_no_az(self): cmd = DEFAULT_CMD + ( '--ec2-attributes KeyName=testkey,SubnetId=subnet-123456,' - 'InstanceProfile=aws-emr-ec2-role') + 'InstanceProfile=EMR_EC2_DefaultRole') result = copy.deepcopy(DEFAULT_RESULT) result['Instances']['Ec2KeyName'] = 'testkey' result['Instances']['Ec2SubnetId'] = 'subnet-123456' - result['JobFlowRole'] = 'aws-emr-ec2-role' + result['JobFlowRole'] = 'EMR_EC2_DefaultRole' self.assert_params_for_cmd(cmd, result) def test_ec2_attributes_az(self): @@ -698,6 +745,26 @@ def test_install_mapr_with_args(self): result['NewSupportedProducts'] = [INSTALL_MAPR_PRODUCT] self.assert_params_for_cmd(cmd, result) + def test_install_mapr_without_args(self): + cmd = DEFAULT_CMD + \ + '--applications Name=mapr' + result = copy.deepcopy(DEFAULT_RESULT) + result['NewSupportedProducts'] = \ + [ + {'Name': 'mapr', + 'Args': []} + ] + self.assert_params_for_cmd(cmd, result) + + def test_supported_products(self): + cmd = DEFAULT_CMD + ( + '--applications ' + 'Name=hue,Args=--hue-config,s3://elasticmapreduce/hue-config ' + 'Name=mapr,Args=--edition,m7') + result = copy.deepcopy(DEFAULT_RESULT) + result['NewSupportedProducts'] = INSTALL_SUPPORTED_PRODUCTS + self.assert_params_for_cmd(cmd, result) + def test_applications_all_types(self): cmd = DEFAULT_CMD + ( '--applications ' diff --git a/tests/unit/customizations/emr/test_create_default_role.py b/tests/unit/customizations/emr/test_create_default_role.py index a17d83a18f4b..0b210be14e60 100644 --- a/tests/unit/customizations/emr/test_create_default_role.py +++ b/tests/unit/customizations/emr/test_create_default_role.py @@ -20,6 +20,7 @@ EC2_ROLE_NAME = "EMR_EC2_DefaultRole" +EMR_ROLE_NAME = "EMR_DefaultRole" EC2_ROLE_POLICY = { "Statement": [ @@ -90,11 +91,23 @@ class TestCreateDefaultRole(BaseAWSCommandParamsTest): ] } + emr_role_policy_document = { + "Version": "2008-10-17", + "Statement": [ + { + "Sid": "", + "Effect": "Allow", + "Principal": {"Service": "elasticmapreduce.amazonaws.com.cn"}, + "Action": "sts:AssumeRole" + } + ] + } + def test_default_roles_exist(self): cmdline = self.prefix self.run_cmd(cmdline, expected_rc=0) - self.assertEqual(len(self.operations_called), 2) + self.assertEqual(len(self.operations_called), 3) self.assertEqual(self.operations_called[0][0].name, 'GetRole') self.assertEqual(self.operations_called[0][1]['RoleName'], @@ -104,6 +117,9 @@ def test_default_roles_exist(self): 'GetInstanceProfile') self.assertEqual(self.operations_called[1][1]['InstanceProfileName'], EC2_ROLE_NAME) + self.assertEqual(self.operations_called[2][0].name, 'GetRole') + self.assertEqual(self.operations_called[2][1]['RoleName'], + EMR_ROLE_NAME) @mock.patch('awscli.customizations.emr.emr.' 'CreateDefaultRoles._construct_result') @@ -121,9 +137,9 @@ def test_default_roles_not_exist(self, role_exists_patch, cmdline = self.prefix + ' --region cn-north-1' self.run_cmd(cmdline, expected_rc=0) - # Only 4 operations will be called as we are mocking + # Only 6 operations will be called as we are mocking # _check_if_role_exists and _check_if_instance_profile_exists methods. - self.assertEqual(len(self.operations_called), 4) + self.assertEqual(len(self.operations_called), 6) self.assertEqual(self.operations_called[0][0].name, 'CreateRole') self.assertEqual(self.operations_called[0][1]['RoleName'], @@ -153,6 +169,19 @@ def test_default_roles_not_exist(self, role_exists_patch, self.assertEqual(self.operations_called[3][1]['RoleName'], EC2_ROLE_NAME) + self.assertEqual(self.operations_called[4][0].name, 'CreateRole') + self.assertEqual(self.operations_called[4][1]['RoleName'], + EMR_ROLE_NAME) + self.assertEqual( + self.operations_called[4][1]['AssumeRolePolicyDocument'], + emrutils.dict_to_string(self.emr_role_policy_document)) + + self.assertEqual(self.operations_called[5][0].name, 'PutRolePolicy') + self.assertEqual(self.operations_called[5][1]['PolicyName'], + EMR_ROLE_NAME) + self.assertEqual(self.operations_called[5][1]['RoleName'], + EMR_ROLE_NAME) + @mock.patch('awscli.customizations.emr.emr.' 'CreateDefaultRoles._construct_result') @mock.patch('awscli.customizations.emr.createdefaultroles' diff --git a/tests/unit/customizations/emr/test_install_applications.py b/tests/unit/customizations/emr/test_install_applications.py index aeb79eb7d1bd..9029627a2f6d 100644 --- a/tests/unit/customizations/emr/test_install_applications.py +++ b/tests/unit/customizations/emr/test_install_applications.py @@ -42,10 +42,10 @@ class TestInstallApplications(BaseAWSCommandParamsTest): - prefix = 'emr install-applications --cluster-id j-ABC123456' + prefix = 'emr install-applications --cluster-id j-ABC123456' def test_intall_hive_with_version(self): - cmdline = self.prefix + ' --apps Name=Hive,Version=0.8.1.8' + cmdline = self.prefix + ' --applications Name=Hive,Version=0.8.1.8' step = copy.deepcopy(INSTALL_HIVE_STEP) step['HadoopJarStep']['Args'][5] = '0.8.1.8' @@ -54,7 +54,7 @@ def test_intall_hive_with_version(self): self.assert_params_for_cmd(cmdline, result) def test_intall_pig_with_version(self): - cmdline = self.prefix + ' --apps Name=Pig,Version=0.9.2.1' + cmdline = self.prefix + ' --applications Name=Pig,Version=0.9.2.1' step = copy.deepcopy(INSTALL_PIG_STEP) step['HadoopJarStep']['Args'][5] = '0.9.2.1' @@ -63,14 +63,15 @@ def test_intall_pig_with_version(self): self.assert_params_for_cmd(cmdline, result) def test_intall_hive_and_pig_without_version(self): - cmdline = self.prefix + ' --cluster-id j-ABC123456 --apps Name=Hive' +\ + cmdline = self.prefix + ' --cluster-id j-ABC123456 --applications Name=Hive' +\ ' Name=Pig' result = {'JobFlowId': 'j-ABC123456', 'Steps': [INSTALL_HIVE_STEP, INSTALL_PIG_STEP]} self.assert_params_for_cmd(cmdline, result) def test_install_impala_error(self): - cmdline = self.prefix + ' --cluster-id j-ABC123456 --apps Name=Impala' + cmdline = self.prefix + \ + ' --cluster-id j-ABC123456 --applications Name=Impala' expected_error_msg = "\naws: error: Impala cannot be installed on" +\ " a running cluster. 'Name' should be one of the following:" +\ @@ -79,7 +80,8 @@ def test_install_impala_error(self): self.assertEqual(result[1], expected_error_msg) def test_install_unknown_app_error(self): - cmdline = self.prefix + ' --cluster-id j-ABC123456 --apps Name=unknown' + cmdline = self.prefix + \ + ' --cluster-id j-ABC123456 --applications Name=unknown' expected_error_msg = "\naws: error: Unknown application: unknown." +\ " 'Name' should be one of the following: HIVE, PIG, HBASE," +\