Merge branch 'release-1.5.2'

* release-1.5.2:
  Bumping version to 1.5.2
  Updated Changelog with new features.
  EMR: Updated test for describe-cluster
  EMR: Fix describe-cluster
  EMR: Add validation logic in steputils and clean up create-cluster and add-steps examples.
  Update tests based on feedback
  Renamed sync strategy class names.
  Remove unused imports and vars in handlers
  Update handlers with new event kwargs
  Fix cloudsearch define-index-field for type latlon
  Add issue #937 to the changelog
  Updated the changelog with bugfix.
  Delete large files for cross region ``mv``
  Cleaned up the custom sync strategy code
  Made seperate directory for  sync strategies
  Add more customizations to sync strategies
  Added custom sync strategy

CHANGELOG.rst

@@ -2,6 +2,21 @@
 CHANGELOG
 =========
 
+1.5.2
+=====
+
+* feature:``aws cloudsearch``: Add support for advance Japanese language
+  processing.
+* feature:``aws rds``: Add support for gp2 which provides faster
+  access than disk-based storage.
+* bugfix:``aws s3 mv``: Delete multi-part objects when transferring objects
+  across regions using ``--source-region``
+  (`issue 938 <https://github.com/aws/aws-cli/pull/938>`__)
+* bugfix:``aws emr ssh``: Fix issue with waiter configuration not
+  being found
+  (`issue 937 <https://github.com/aws/aws-cli/issues/937>`__)
+
+
 1.5.1
 =====
 

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.5.1'
+__version__ = '1.5.2'
 
 #
 # Get our data path to be added to botocore's search path

awscli/customizations/cloudsearch.py

@@ -49,6 +49,12 @@ def index_hydrate(params, container, cli_type, key, value):
     _type = params['index_field']['IndexFieldType']
     _type = ''.join([i.capitalize() for i in _type.split('-')])
 
+    # ``index_field`` of type ``latlon`` is mapped to ``Latlon``.
+    # However, it is defined as ``LatLon`` in the model so it needs to
+    # be changed.
+    if _type == 'Latlon':
+        _type = 'LatLon'
+
     # Transform string value to the correct type?
     if key.split(SEP)[-1] == 'DefaultValue':
         value = DEFAULT_VALUE_TYPE_MAP.get(_type, lambda x: x)(value)

awscli/customizations/commands.py

@@ -120,14 +120,16 @@ class BasicCommand(CLICommand):
 
     def __init__(self, session):
         self._session = session
+        self._arg_table = None
+        self._subcommand_table = None
 
     def __call__(self, args, parsed_globals):
         # args is the remaining unparsed args.
         # We might be able to parse these args so we need to create
         # an arg parser and parse them.
-        subcommand_table = self._build_subcommand_table()
-        arg_table = self.arg_table
-        parser = ArgTableArgParser(arg_table, subcommand_table)
+        self._subcommand_table = self._build_subcommand_table()
+        self._arg_table = self._build_arg_table()
+        parser = ArgTableArgParser(self.arg_table, self.subcommand_table)
         parsed_args, remaining = parser.parse_known_args(args)
 
         # Unpack arguments
@@ -138,8 +140,8 @@ def __call__(self, args, parsed_globals):
             # as these are how the parameters are stored in the
             # `arg_table`.
             xformed = key.replace('_', '-')
-            if xformed in arg_table:
-                cli_argument = arg_table[xformed]
+            if xformed in self.arg_table:
+                cli_argument = self.arg_table[xformed]
 
             value = unpack_argument(
                 self._session,
@@ -178,8 +180,8 @@ def __call__(self, args, parsed_globals):
                 raise ValueError("Unknown options: %s" % ','.join(remaining))
             return self._run_main(parsed_args, parsed_globals)
         else:
-            return subcommand_table[parsed_args.subcommand](remaining,
-                                                            parsed_globals)
+            return self.subcommand_table[parsed_args.subcommand](remaining,
+                                                                 parsed_globals)
 
     def _validate_value_against_schema(self, model, value):
         validate_parameters(value, model)
@@ -233,9 +235,10 @@ def create_help_command_table(self):
             commands[command['name']] = command['command_class'](self._session)
         return commands
 
-    @property
-    def arg_table(self):
+    def _build_arg_table(self):
         arg_table = OrderedDict()
+        self._session.emit('building-arg-table.%s' % self.NAME,
+                           arg_table=self.ARG_TABLE)
         for arg_data in self.ARG_TABLE:
 
             # If a custom schema was passed in, create the argument_model
@@ -249,6 +252,18 @@ def arg_table(self):
             arg_table[arg_data['name']] = custom_argument
         return arg_table
 
+    @property
+    def arg_table(self):
+        if self._arg_table is None:
+            self._arg_table = self._build_arg_table()
+        return self._arg_table
+
+    @property
+    def subcommand_table(self):
+        if self._subcommand_table is None:
+            self._subcommand_table = self._build_subcommand_table()
+        return self._subcommand_table
+
     @classmethod
     def add_command(cls, command_table, session, **kwargs):
         command_table[cls.NAME] = cls(session)

awscli/customizations/datapipeline/__init__.py

@@ -3,7 +3,6 @@
 from datetime import datetime, timedelta
 
 from awscli.arguments import CustomArgument
-from awscli.argprocess import uri_param
 from awscli.customizations.commands import BasicCommand
 from awscli.customizations.datapipeline import translator
 
@@ -72,7 +71,7 @@ def add_pipeline_definition(argument_table, **kwargs):
     del argument_table['pipeline-objects']
 
 
-def translate_definition(operation, http_response, parsed, **kwargs):
+def translate_definition(parsed, **kwargs):
     api_objects = parsed.pop('pipelineObjects', None)
     if api_objects is None:
         return

awscli/customizations/ec2bundleinstance.py

@@ -138,11 +138,10 @@ def _generate_signature(params):
         del params['_SAK']
 
 
-def _check_params(**kwargs):
+def _check_params(params, **kwargs):
     # Called just before call but prior to building the params.
     # Adds information not supplied by the user.
-    logger.debug(kwargs)
-    storage = kwargs['params']['storage']['S3']
+    storage = params['storage']['S3']
     if 'UploadPolicy' not in storage:
         _generate_policy(storage)
     if 'UploadPolicySignature' not in storage:

awscli/customizations/ec2decryptpassword.py

@@ -85,14 +85,13 @@ def add_to_params(self, parameters, value):
                        'the instance.')
                 raise ValueError(msg)
 
-    def _decrypt_password_data(self, http_response, parsed, **kwargs):
+    def _decrypt_password_data(self, parsed, **kwargs):
         """
-        This handler gets called after the GetPasswordData command has
-        been executed.  It is called with the ``http_response`` and
-        the ``parsed`` data.  It checks to see if a private launch
-        key was specified on the command.  If it was, it tries to use
-        that private key to decrypt the password data and replace it
-        in the returned data dictionary.
+        This handler gets called after the GetPasswordData command has been
+        executed.  It is called with the and the ``parsed`` data.  It checks to
+        see if a private launch key was specified on the command.  If it was,
+        it tries to use that private key to decrypt the password data and
+        replace it in the returned data dictionary.
         """
         if self._key_path is not None:
             logger.debug("Decrypting password data using: %s", self._key_path)

awscli/customizations/ec2protocolarg.py

@@ -16,7 +16,7 @@
 parameter of the operation accepts only integer protocol numbers.
 """
 
-def _fix_args(operation, endpoint, params, **kwargs):
+def _fix_args(params, **kwargs):
     key_name = 'Protocol'
     if key_name in params:
         if params[key_name] == 'tcp':

awscli/customizations/ec2runinstances.py

@@ -79,7 +79,7 @@ def _check_args(parsed_args, **kwargs):
                 raise ValueError(msg)
 
 
-def _fix_args(operation, endpoint, params, **kwargs):
+def _fix_args(params, **kwargs):
     # The RunInstances request provides some parameters
     # such as --subnet-id and --security-group-id that can be specified
     # as separate options only if the request DOES NOT include a

awscli/customizations/emr/describecluster.py

@@ -76,26 +76,9 @@ def _call(self, operation_object, parameters, parsed_globals):
             region_name=parsed_globals.region,
             endpoint_url=parsed_globals.endpoint_url,
             verify=parsed_globals.verify_ssl)
-        if operation_object.can_paginate and parsed_globals.paginate:
-            pages = operation_object.paginate(endpoint, **parameters)
-            key = None
-            for page in pages:
-                http_response = page[0]
-                if http_response.status_code == 200:
-                    response_data = page[1]
-                    keys = response_data.keys()
-                    key = self._get_key_of_result(keys)
-                    if key is not None:
-                        result += response_data.get(key)
-
-            if key is not None:
-                return {key: result}
-            else:
-                return None
-        else:
-            http_response, response_data = operation_object.call(endpoint,
+        http_response, response_data = operation_object.call(endpoint,
                                                                  **parameters)
-            return response_data
+        return response_data
 
     def _get_key_of_result(self, keys):
         # Return the first key that is not "Marker"

awscli/customizations/emr/emrutils.py

@@ -143,11 +143,16 @@ def get_script_runner(region='us-east-1'):
 
 
 def check_required_field(structure, name, value):
-    if value is None:
+    if not value:
         raise exceptions.MissingParametersError(
             object_name=structure, missing=name)
 
 
+def check_empty_string_list(name, value):
+    if not value or (len(value) == 1 and value[0].strip() == ""):
+        raise exceptions.EmptyListError(param=name)
+
+
 def call(session, operation_object, parameters, region_name=None,
          endpoint_url=None, verify=None):
         # We could get an error from get_endpoint() about not having

awscli/customizations/emr/exceptions.py

@@ -43,6 +43,15 @@ class MissingParametersError(EmrError):
            '{object_name}: {missing}.')
 
 
+class EmptyListError(EmrError):
+    """
+    The provided list is empty.
+
+    :ivar param: The provided list parameter
+    """
+    fmt = ('aws: error: The prameter {param} cannot be an empty list.')
+
+
 class MissingRequiredInstanceGroupsError(EmrError):
     """
     In create-cluster command, none of --instance-group,

awscli/customizations/emr/steputils.py

@@ -72,14 +72,15 @@ def build_streaming_step(parsed_step):
         arg=parsed_step.get('ActionOnFailure'),
         value=constants.DEFAULT_FAILURE_ACTION)
 
-    args_list = parsed_step.get('Args')
+    args = parsed_step.get('Args')
     emrutils.check_required_field(
         structure=constants.STREAMING_STEP_CONFIG,
         name='Args',
-        value=args_list)
+        value=args)
+    emrutils.check_empty_string_list(name='Args', value=args)
     return emrutils.build_step(
         jar=constants.STREAMING_JAR_PATH,
-        args=args_list,
+        args=args,
         name=name,
         action_on_failure=action_on_failure)
 
@@ -88,6 +89,7 @@ def build_hive_step(parsed_step, region=None):
     args = parsed_step.get('Args')
     emrutils.check_required_field(
         structure=constants.HIVE_STEP_CONFIG, name='Args', value=args)
+    emrutils.check_empty_string_list(name='Args', value=args)
     name = _apply_default_value(
         arg=parsed_step.get('Name'),
         value=constants.DEFAULT_HIVE_STEP_NAME)
@@ -115,7 +117,7 @@ def build_pig_step(parsed_step, region=None):
     args = parsed_step.get('Args')
     emrutils.check_required_field(
         structure=constants.PIG_STEP_CONFIG, name='Args', value=args)
-
+    emrutils.check_empty_string_list(name='Args', value=args)
     name = _apply_default_value(
         arg=parsed_step.get('Name'),
         value=constants.DEFAULT_PIG_STEP_NAME)

awscli/customizations/iamvirtmfa.py

@@ -82,7 +82,7 @@ def _add_options(self, argument_table, operation, **kwargs):
         argument_table['outfile'] = self._outfile
         argument_table['bootstrap-method'] = self._method
 
-    def _save_file(self, http_response, parsed, **kwargs):
+    def _save_file(self, parsed, **kwargs):
         method = self._method.value
         outfile = self._outfile.value
         if method in parsed['VirtualMFADevice']: