Merge branch 'release-1.4.3' into develop

* release-1.4.3:
  Bumping version to 1.4.3
  Updated the changelog for new features.
  EMR: Remove UnknownApplicationError
  EMR: Update modify-cluster-attributes error message.
  EMR: Update help text and examples. 1. Add helptext description, which says leaving instance-count creates one node cluster; 2. Sanitizing add-tags example 3. Edit/change example text for create-cluster and add-tags

CHANGELOG.rst

@@ -2,9 +2,16 @@
 CHANGELOG
 =========
 
-Next Release (TBD)
-==================
+1.4.3
+=====
 
+* feature:``aws iam``: Update ``aws iam`` command to latest version.
+* feature:``aws cognito-sync``: Update ``aws cognito-sync`` command 
+  to latest version.
+* feature:``aws opsworks``: Update ``aws opsworks`` command to latest
+  version.
+* feature:``aws elasticbeanstalk``: Add support for bundling logs.
+* feature:``aws kinesis``: Add suport for tagging.
 * feature:Page Size: Add a ``--page-size`` option, that
   controls page size when perfoming an operation that
   uses pagination.

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.4.2'
+__version__ = '1.4.3'
 
 #
 # Get our data path to be added to botocore's search path

awscli/customizations/emr/applicationutils.py

@@ -24,11 +24,7 @@ def build_applications(parsed_applications, parsed_globals, ami_version=None):
     for app_config in parsed_applications:
         app_name = app_config['Name'].lower()
 
-        if app_name in constants.SUPPORTED_PRODUCTS:
-            app_list.append(
-                build_supported_product(
-                    app_config['Name'], app_config.get('Args')))
-        elif app_name == constants.HIVE:
+        if app_name == constants.HIVE:
             hive_version = constants.LATEST
             step_list.append(
                 _build_install_hive_step(region=parsed_globals.region))
@@ -71,12 +67,14 @@ def build_applications(parsed_applications, parsed_globals, ami_version=None):
                     region=parsed_globals.region,
                     args=app_config.get('Args')))
         else:
-            raise exceptions.UnknownApplicationError(app_name=app_name)
+            app_list.append(
+                _build_supported_product(
+                    app_config['Name'], app_config.get('Args')))
 
     return app_list, ba_list, step_list
 
 
-def build_supported_product(name, args):
+def _build_supported_product(name, args):
     if args is None:
         args = []
     config = {'Name': name.lower(), 'Args': args}

awscli/customizations/emr/constants.py

@@ -27,7 +27,6 @@
 SCRIPT_RUNNER_PATH = '/libs/script-runner/script-runner.jar'
 DEBUGGING_PATH = '/libs/state-pusher/0.1/fetch'
 DEBUGGING_NAME = 'Setup Hadoop Debugging'
-SUPPORTED_PRODUCTS = ['mapr', 'mapr-m3', 'mapr-m5', 'mapr-m7', 'hue']
 
 MAX_BOOTSTRAP_ACTION_NUMBER = 16
 BOOTSTRAP_ACTION_NAME = 'Bootstrap action'

awscli/customizations/emr/exceptions.py

@@ -97,17 +97,6 @@ class UnknownStepTypeError(EmrError):
     fmt = ('aws: error: The step type {step_type} is not supported.')
 
 
-class UnknownApplicationError(EmrError):
-    """
-    The provided application name is unknown.
-
-    :ivar application_name: the application name provided.
-    """
-    fmt = ('aws: error: The application name {app_name} is not supported. '
-           '"Name" should be one of the following: ' +
-           (', '.join(constants.APPLICATIONS))) + '.'
-
-
 class UnknownIamEndpointError(EmrError):
     """
     The IAM endpoint is not known for the specified region.
@@ -187,6 +176,17 @@ class SubnetAndAzValidationError(EmrError):
            'tyZone (placement) because ec2SubnetId implies a placement.')
 
 
+class MutualExclusiveOptionError(EmrError):
+    """
+    The provided option1 and option2 are mutually exclusive.
+
+    :ivar option1
+    :ivar option2
+    """
+    fmt = ('aws: error: You cannot specify both {option1} and {option2} '
+           'options together.')
+
+
 class MissingApplicationsError(EmrError):
     """
     The application required for a step is not installed when creating a
@@ -219,3 +219,15 @@ class ClusterStatesFilterValidationError(EmrError):
     """
     fmt = ('aws: error: You can specify only one of the cluster state '
            'filters: --cluster-states, --active, --terminated, --failed.')
+
+
+class MissingClusterAttributesError(EmrError):
+    """
+    In the modify-cluster-attributes command, customers need to provide
+    at least one of the following cluster attributes: --visible-to-all-users,
+    --no-visible-to-all-users, --termination-protected
+    and --no-termination-protected
+    """
+    fmt = ('aws: error: Must specify one of the following boolean options: '
+           '--visible-to-all-users|--no-visible-to-all-users, '
+           '--termination-protected|--no-termination-protected.')

awscli/customizations/emr/modifyclusterattributes.py

@@ -14,6 +14,7 @@
 from awscli.customizations.commands import BasicCommand
 from awscli.customizations.emr import helptext
 from awscli.customizations.emr import emrutils
+from awscli.customizations.emr import exceptions
 
 
 class ModifyClusterAttr(BasicCommand):
@@ -31,26 +32,25 @@ class ModifyClusterAttr(BasicCommand):
             'help_text': 'Change cluster visibility for IAM users'},
         {'name': 'termination-protected', 'required': False, 'action':
             'store_true', 'group_name': 'terminate',
-            'help_text': 'Set termination protected on or off'},
+            'help_text': 'Set termination protection on or off'},
         {'name': 'no-termination-protected', 'required': False, 'action':
             'store_true', 'group_name': 'terminate',
-            'help_text': 'Set termination protected on or off'},
+            'help_text': 'Set termination protection on or off'},
     ]
 
     def _run_main(self, args, parsed_globals):
 
         if (args.visible_to_all_users and args.no_visible_to_all_users):
-            raise ValueError(
-                'aws: error: Cannot use both options --visible-to-all-users '
-                'and --no-visible-to-all-users together.')
+            raise exceptions.MutualExclusiveOptionError(
+                option1='--visible-to-all-users',
+                option2='--no-visible-to-all-users')
         if (args.termination_protected and args.no_termination_protected):
-            raise ValueError(
-                'aws: error: Cannot use both options --termination-protected '
-                'and --no-termination-protected together.')
+            raise exceptions.MutualExclusiveOptionError(
+                option1='--termination-protected',
+                option2='--no-termination-protected')
         if not(args.termination_protected or args.no_termination_protected
                or args.visible_to_all_users or args.no_visible_to_all_users):
-            raise ValueError('aws: error: You need to specify atleast one of '
-                             'the options.')
+            raise exceptions.MissingClusterAttributesError()
 
         if (args.visible_to_all_users or args.no_visible_to_all_users):
             visible = (args.visible_to_all_users and
@@ -69,4 +69,4 @@ def _run_main(self, args, parsed_globals):
             emrutils.call_and_display_response(self._session,
                                                'SetTerminationProtection',
                                                parameters, parsed_globals)
-        return 0
+        return 0

awscli/examples/emr/add-steps.rst

@@ -49,7 +49,7 @@
 
 - Command::
 
-    aws emr add-steps --cluster-id j-XXXXXXXX --steps Type=HIVE,Name='Hive program',ActionOnFailure=CONTINUE,Version=latest,Args=[-f,s3://mybuckey/myhivescript.q,-d,INPUT=s3://mybucket/myhiveinput,-d,OUTPUT=s3://mybucket/myhiveoutput,arg1,arg2] Type=HIVE,Name='Hive steps',ActionOnFailure=TERMINATE_CLUSTER,Version=latest,Args=[-f,s3://elasticmapreduce/samples/hive-ads/libs/model-build.q,-d,INPUT=s3://elasticmapreduce/samples/hive-ads/tables,-d,OUTPUT=s3://mybucket/hive-ads/output/2014-04-18/11-07-32,-d,LIBS=s3://elasticmapreduce/samples/hive-ads/libs]
+    aws emr add-steps --cluster-id j-XXXXXXXX --steps Type=HIVE,Name='Hive program',ActionOnFailure=CONTINUE,Args=[-f,s3://mybuckey/myhivescript.q,-d,INPUT=s3://mybucket/myhiveinput,-d,OUTPUT=s3://mybucket/myhiveoutput,arg1,arg2] Type=HIVE,Name='Hive steps',ActionOnFailure=TERMINATE_CLUSTER,Args=[-f,s3://elasticmapreduce/samples/hive-ads/libs/model-build.q,-d,INPUT=s3://elasticmapreduce/samples/hive-ads/tables,-d,OUTPUT=s3://mybucket/hive-ads/output/2014-04-18/11-07-32,-d,LIBS=s3://elasticmapreduce/samples/hive-ads/libs]
 
 
 - Required parameters::
@@ -74,7 +74,7 @@
 
 - Command::
 
-    aws emr add-steps --cluster-id j-XXXXXXXX --steps Type=PIG,Name='Pig program',ActionOnFailure=CONTINUE,Version=latest,Args=[-f,s3://mybuckey/mypigscript.pig,-p,INPUT=s3://mybucket/mypiginput,-p,OUTPUT=s3://mybucket/mypigoutput,arg1,arg2] Type=PIG,Name='Pig program',Version=latest,Args=[-f,s3://elasticmapreduce/samples/pig-apache/do-reports2.pig,-p,INPUT=s3://elasticmapreduce/samples/pig-apache/input,-p,OUTPUT=s3://mybucket/pig-apache/output,arg1,arg2]
+    aws emr add-steps --cluster-id j-XXXXXXXX --steps Type=PIG,Name='Pig program',ActionOnFailure=CONTINUE,Args=[-f,s3://mybuckey/mypigscript.pig,-p,INPUT=s3://mybucket/mypiginput,-p,OUTPUT=s3://mybucket/mypigoutput,arg1,arg2] Type=PIG,Name='Pig program',Args=[-f,s3://elasticmapreduce/samples/pig-apache/do-reports2.pig,-p,INPUT=s3://elasticmapreduce/samples/pig-apache/input,-p,OUTPUT=s3://mybucket/pig-apache/output,arg1,arg2]
 
 
 - Required parameters::

awscli/examples/emr/create-cluster-examples.rst

@@ -8,7 +8,7 @@
 
 - Command::
 
-    aws emr create-cluster --ami-version 3.1.0 --service-role EMR_DefaultRole --ec2-attributes InstanceProfiles=EC2_EMR_DefaultRoles --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
+    aws emr create-cluster --ami-version 3.1.0 --service-role EMR_DefaultRole --ec2-attributes InstanceProfile=EMR_EC2_DefaultRole --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge 
 
 **3. Create an Amazon EMR cluster with default roles**
 
@@ -28,10 +28,6 @@
 
     aws emr create-cluster --ami-version 3.1.0  --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge  InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
 
-- Create an Amazon EMR cluster that will NOT terminate after completing all the steps::
-
-    aws emr create-cluster --ami-version 3.1.0  --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge  InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --no-auto-terminate
-
 **6. Specify EC2 Attributes**
 
 - Create an Amazon EMR cluster with Amazon EC2 Key Pair "myKey" and instance profile "myProfile"::
@@ -42,9 +38,9 @@
 
     aws emr create-cluster --ec2-attributes SubnetId=subnet-xxxxx --ami-version 3.1.0  --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
 
-- Create an Amazon EMR cluster in an AvailabilityZone. For example, us-west-1b::
+- Create an Amazon EMR cluster in an AvailabilityZone. For example, us-east-1b::
 
-    aws emr create-cluster --ec2-attributes AvailabilityZone=us-west-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
+    aws emr create-cluster --ec2-attributes AvailabilityZone=us-east-1b --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge 
 
 **7. Enable debugging and specify a Log URI**
 
@@ -70,7 +66,7 @@
 
 - The following example changes the maximum number of map tasks and sets the NameNode heap size::
 
-    aws emr create-cluster --bootstrap-actions Path=s3://elasticmapreduce/bootstrap-actions/configure-hadoop,Name="Change the maximum number of map tasks",Args=[-M,s3://myawsbucket/config.xml,-m,mapred.tasktracker.map.tasks.maximum=2] Path=s3://elasticmapreduce/bootstrap-actions/configure-daemons,Name="Set the NameNode heap size",Args=[--namenode-heap-size=2048,--namenode-opts=-XX:GCTimeRatio=19] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
+    aws emr create-cluster --bootstrap-actions Path=s3://elasticmapreduce/bootstrap-actions/configure-hadoop,Name="Change the maximum number of map tasks",Args=[--yarn-key-value,mapred.tasktracker.map.tasks.maximum=2] Path=s3://elasticmapreduce/bootstrap-actions/configure-daemons,Name="Set the NameNode heap size",Args=[--namenode-heap-size=2048,--namenode-opts=-XX:GCTimeRatio=19] --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge
 
 **10. Create an Amazon EMR cluster with applications**
 
@@ -124,7 +120,7 @@
 
 - Command::
 
-    aws emr create-cluster --steps Type=HIVE,Name='Hive program',ActionOnFailure=CONTINUE,Args=[-f,s3://mybuckey/myhivescript.q,-d,INPUT=s3://mybucket/myhiveinput,-d,OUTPUT=s3://mybucket/myhiveoutput,arg1,arg2] Type=HIVE,Name='Hive steps',ActionOnFailure=TERMINATE_CLUSTER,Args=[-f,s3://elasticmapreduce/samples/hive-ads/libs/model-build.q,-d,INPUT=s3://elasticmapreduce/samples/hive-ads/tables,-d,OUTPUT=s3://mybucket/hive-ads/output/2014-04-18/11-07-32,-d,LIBS=s3://elasticmapreduce/samples/hive-ads/libs] --applications Name=Hive --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
+    aws emr create-cluster --steps Type=HIVE,Name='Hive program',ActionOnFailure=CONTINUE,ActionOnFailure=TERMINATE_CLUSTER,Args=[-f,s3://elasticmapreduce/samples/hive-ads/libs/model-build.q,-d,INPUT=s3://elasticmapreduce/samples/hive-ads/tables,-d,OUTPUT=s3://mybucket/hive-ads/output/2014-04-18/11-07-32,-d,LIBS=s3://elasticmapreduce/samples/hive-ads/libs] --applications Name=Hive --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge
 
 - Hive steps required parameters::
 
@@ -138,7 +134,7 @@
 
 - Command::
 
-    aws emr create-cluster --steps Type=PIG,Name='Pig program',ActionOnFailure=CONTINUE,Args=[-f,s3://mybuckey/mypigscript.pig,-p,INPUT=s3://mybucket/mypiginput,-p,OUTPUT=s3://mybucket/mypigoutput,arg1,arg2] Type=PIG,Name='Pig program',Args=[-f,s3://elasticmapreduce/samples/pig-apache/do-reports2.pig,-p,INPUT=s3://elasticmapreduce/samples/pig-apache/input,-p,OUTPUT=s3://mybucket/pig-apache/output,arg1,arg2] --applications Name=Pig --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
+    aws emr create-cluster --steps Type=PIG,Name='Pig program',ActionOnFailure=CONTINUE,Args=[-f,s3://elasticmapreduce/samples/pig-apache/do-reports2.pig,-p,INPUT=s3://elasticmapreduce/samples/pig-apache/input,-p,OUTPUT=s3://mybucket/pig-apache/output] --applications Name=Pig --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge
 
 - Pig steps required parameters::
 
@@ -152,7 +148,7 @@
 
 - Command::
 
-    aws emr create-cluster --steps Type=IMPALA,Name='Impala program',ActionOnFailure=CONTINUE,Args=-f,--impala-script,s3://myimpala/input,--console-output-path,s3://myimpala/output Type=IMPALA,Name='Impala program',ActionOnFailure=CONTINUE,Args=-f,--impala-script,s3://myimpala/input,--console-output-path,s3://myimpala/output --applications Name=Impala --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge --auto-terminate
+    aws emr create-cluster --steps Type=CUSTOM_JAR,Name='Wikipedia Impala program',ActionOnFailure=CONTINUE,Jar=s3://elasticmapreduce/libs/script-runner/script-runner.jar,Args="/home/hadoop/impala/examples/wikipedia/wikipedia-with-s3distcp.sh" Type=IMPALA,Name='Impala program',ActionOnFailure=CONTINUE,Args=-f,--impala-script,s3://myimpala/input,--console-output-path,s3://myimpala/output --applications Name=Impala --ami-version 3.1.0 --instance-groups InstanceGroupType=MASTER,InstanceCount=1,InstanceType=m3.xlarge InstanceGroupType=CORE,InstanceCount=2,InstanceType=m3.xlarge 
 
 - Impala steps required parameters::
 

doc/source/conf.py

@@ -52,7 +52,7 @@
 # The short X.Y version.
 version = '1.4'
 # The full version, including alpha/beta/rc tags.
-release = '1.4.2'
+release = '1.4.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

setup.py

@@ -6,7 +6,7 @@
 import awscli
 
 
-requires = ['botocore>=0.62.0,<0.63.0',
+requires = ['botocore>=0.63.0,<0.64.0',
             'bcdoc>=0.12.0,<0.13.0',
             'six>=1.1.0',
             'colorama==0.2.5',

tests/unit/customizations/emr/test_create_cluster.py

@@ -172,6 +172,10 @@
     {
         'Name': 'mapr',
         'Args': ['--edition', 'm7']
+    },
+    {
+        'Name': 'unknown',
+        'Args': ['arg1', 'k1=v1']
     }
 ]
 
@@ -790,15 +794,6 @@ def test_bootstrap_actions_from_json_file(self):
         self.assert_params_for_cmd(cmd, result)
 
     # Applications test cases
-    def test_wrong_application_type_error(self):
-        cmd = DEFAULT_CMD + '--applications Name=unknown'
-        expected_error_msg = (
-            '\naws: error: The application name unknown is not supported. '
-            '"Name" should be one of the following: HIVE, PIG, HBASE, '
-            'GANGLIA, IMPALA, MAPR, MAPR_M3, MAPR_M5, MAPR_M7.\n')
-        result = self.run_cmd(cmd, 255)
-        self.assertEquals(expected_error_msg, result[1])
-
     def test_install_hive_with_defaults(self):
         cmd = DEFAULT_CMD + '--applications Name=Hive'
         result = copy.deepcopy(DEFAULT_RESULT)
@@ -877,7 +872,8 @@ def test_supported_products(self):
         cmd = DEFAULT_CMD + (
             '--applications '
             'Name=hue,Args=--hue-config,s3://elasticmapreduce/hue-config '
-            'Name=mapr,Args=--edition,m7')
+            'Name=mapr,Args=--edition,m7 '
+            'Name=unknown,Args=[arg1,k1=v1]')
         result = copy.deepcopy(DEFAULT_RESULT)
         result['NewSupportedProducts'] = INSTALL_SUPPORTED_PRODUCTS
         result.pop('Steps')

tests/unit/customizations/emr/test_modify_cluster_attributes.py

@@ -43,19 +43,25 @@ def test_no_termination_protected(self):
         result = {'JobFlowIds': ['j-ABC123456'], 'TerminationProtected': False}
         self.assert_params_for_cmd(cmdline, result)
 
-    @raises(Exception)
     def test_visible_to_all_and_no_visible_to_all(self):
         args = ' --cluster-id j-ABC123456 --no-visible-to-all-users'\
                ' --visible-to-all-users'
         cmdline = self.prefix + args
-        self.assert_params_for_cmd(cmdline)
+        expected_error_msg = (
+            '\naws: error: You cannot specify both --visible-to-all-users '
+            'and --no-visible-to-all-users options together.\n')
+        result = self.run_cmd(cmdline, 255)
+        self.assertEquals(expected_error_msg, result[1])
 
-    @raises(Exception)
     def test_temination_protected_and_no_termination_protected(self):
         args = ' --cluster-id j-ABC123456 --no-termination-protected'\
                ' --termination-protected'
         cmdline = self.prefix + args
-        self.assert_params_for_cmd(cmdline)
+        expected_error_msg = (
+            '\naws: error: You cannot specify both --termination-protected '
+            'and --no-termination-protected options together.\n')
+        result = self.run_cmd(cmdline, 255)
+        self.assertEquals(expected_error_msg, result[1])
 
     def test_termination_protected_and_visible_to_all(self):
         args = ' --cluster-id j-ABC123456 --termination-protected'\
@@ -85,12 +91,15 @@ def test_termination_protected_and_no_visible_to_all(self):
         self.assertDictEqual(
             self.operations_called[1][1], result_set_termination_protection)
 
-    @raises(Exception)
-    def test_atleast_one_option(self):
+    def test_at_least_one_option(self):
         args = ' --cluster-id j-ABC123456'
         cmdline = self.prefix + args
-        self.assert_params_for_cmd(cmdline)
-
+        expected_error_msg = (
+            '\naws: error: Must specify one of the following boolean options: '
+            '--visible-to-all-users|--no-visible-to-all-users, '
+            '--termination-protected|--no-termination-protected.\n')
+        result = self.run_cmd(cmdline, 255)
+        self.assertEquals(expected_error_msg, result[1])
 
 if __name__ == "__main__":
     unittest.main()