Merge branch 'release-1.3.7'

* release-1.3.7: (28 commits)
  Bumping version to 1.3.7
  Add #742 to changelog
  Add a comment about why get_stdout_text_writer is needed.
  Code review feedback
  Py3 integ test fixes
  Update changelog with #749
  Add compat layer for text based stream writers
  Fix S3 sync issue with keys containing urlencode values
  Add issue to changelog
  Remove print statement in test
  Fix issue with scalar/non-scalar lists
  Fix doc example for s3api put-object
  Refactor load-cli-arg common event code
  Add 750 to the changelog
  Update paramfile custom argument to use events
  Aggregate dupe keys into a list in datapipeline translation
  Add issue to CHANGELOG
  Do not auto parse JSON based on filename
  Update tests to not mock __builtin__.open
  Allow custom param values to be read from files/urls
  ...

CHANGELOG.rst

@@ -2,6 +2,30 @@
 CHANGELOG
 =========
 
+1.3.7
+=====
+
+* bugfix:Output Format: Fix issue with encoding errors when
+  using text and table output and redirecting to a pipe or file
+  (`issue 742 <https://github.com/aws/aws-cli/issues/742>`__)
+* bugfix:``aws s3``: Fix issue with sync re-uploading certain
+  files
+  (`issue 749 <https://github.com/aws/aws-cli/issues/749>`__)
+* bugfix:Text Output: Fix issue with inconsistent text output
+  based on order
+  (`issue 751 <https://github.com/aws/aws-cli/issues/751>`__)
+* bugfix:``aws datapipeline``: Fix issue for aggregating keys into
+  a list when calling ``aws datapipeline get-pipeline-definition``
+  (`issue 750 <https://github.com/aws/aws-cli/pull/750>`__)
+* bugfix:``aws s3``: Fix issue when running out of disk
+  space during ``aws s3`` transfers
+  (`issue 739 <https://github.com/aws/aws-cli/issues/739>`__)
+* feature:``aws s3 sync``: Add ``--size-only`` param to the
+  ``aws s3 sync`` command
+  (`issue 472 <https://github.com/aws/aws-cli/issues/473>`__,
+   `issue 719 <https://github.com/aws/aws-cli/pull/719>`__)
+
+
 1.3.6
 =====
 

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.3.6'
+__version__ = '1.3.7'
 
 #
 # Get our data path to be added to botocore's search path

awscli/argprocess.py

@@ -47,6 +47,30 @@ def __init__(self, param, key, valid_keys):
         super(ParamUnknownKeyError, self).__init__(full_message)
 
 
+def unpack_argument(session, service_name, operation_name, param, value):
+    """
+    Unpack an argument's value from the commandline. This is part one of a two
+    step process in handling commandline arguments. Emits the load-cli-arg
+    event with service, operation, and parameter names. Example::
+
+        load-cli-arg.ec2.describe-instances.foo
+
+    """
+    param_name = getattr(param, 'name', 'anonymous')
+
+    value_override = session.emit_first_non_none_response(
+        'load-cli-arg.%s.%s.%s' % (service_name,
+                                   operation_name,
+                                   param_name),
+        param=param, value=value, service_name=service_name,
+        operation_name=operation_name)
+
+    if value_override is not None:
+        value = value_override
+
+    return value
+
+
 def uri_param(param, value, **kwargs):
     """Handler that supports param values from URIs.
     """

awscli/arguments.py

@@ -177,7 +177,8 @@ class CustomArgument(BaseCLIArgument):
 
     def __init__(self, name, help_text='', dest=None, default=None,
                  action=None, required=None, choices=None, nargs=None,
-                 cli_type_name=None, group_name=None, positional_arg=False):
+                 cli_type_name=None, group_name=None, positional_arg=False,
+                 no_paramfile=False):
         self._name = name
         self._help = help_text
         self._dest = dest
@@ -191,6 +192,7 @@ def __init__(self, name, help_text='', dest=None, default=None,
         if choices is None:
             choices = []
         self._choices = choices
+        self.no_paramfile = no_paramfile
         # TODO: We should eliminate this altogether.
         # You should not have to depend on an argument_object
         # as part of the interface.  Currently the argprocess
@@ -366,14 +368,6 @@ def add_to_params(self, parameters, value):
     def _unpack_argument(self, value):
         service_name = self.operation_object.service.endpoint_prefix
         operation_name = xform_name(self.operation_object.name, '-')
-        # This is a two step process.  First we "load" the value.
-        value_override = self._emit_first_response(
-            'load-cli-arg.%s.%s' % (service_name, operation_name),
-            param=self.argument_object, value=value,
-            operation=self.operation_object)
-        if value_override is not None:
-            value = value_override
-        # Then we "process/parse" the argument.
         override = self._emit_first_response('process-cli-arg.%s.%s' % (
             service_name, operation_name), param=self.argument_object,
             value=value,

awscli/clidriver.py

@@ -35,6 +35,7 @@
 from awscli.arguments import BooleanArgument
 from awscli.arguments import CLIArgument
 from awscli.arguments import UnknownArgumentError
+from awscli.argprocess import unpack_argument
 
 
 LOG = logging.getLogger('awscli.clidriver')
@@ -460,9 +461,25 @@ def _build_call_parameters(self, args, arg_table):
         for arg_name, arg_object in arg_table.items():
             py_name = arg_object.py_name
             if py_name in parsed_args:
-                arg_object.add_to_params(service_params, parsed_args[py_name])
+                value = parsed_args[py_name]
+                value = self._unpack_arg(arg_object, value)
+                arg_object.add_to_params(service_params, value)
         return service_params
 
+    def _unpack_arg(self, arg_object, value):
+        # Unpacks a commandline argument into a Python value by firing the
+        # load-cli-arg.service-name.operation-name event.
+        session = self._service_object.session
+        service_name = self._service_object.endpoint_prefix
+        operation_name = xform_name(self._operation_object.name, '-')
+
+        param = arg_object
+        if hasattr(param, 'argument_object') and param.argument_object:
+            param = param.argument_object
+
+        return unpack_argument(session, service_name, operation_name,
+                               param, value)
+
     def _create_argument_table(self):
         argument_table = OrderedDict()
         # Arguments are treated a differently than service and

awscli/compat.py

@@ -0,0 +1,33 @@
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+
+#     http://aws.amazon.com/apache2.0/
+
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import sys
+import six
+
+if six.PY3:
+    def get_stdout_text_writer():
+        return sys.stdout
+else:
+    import codecs
+    import locale
+    def get_stdout_text_writer():
+        # In python3, all the sys.stdout/sys.stderr streams are in text
+        # mode.  This means they expect unicode, and will encode the
+        # unicode automatically before actually writing to stdout/stderr.
+        # In python2, that's not the case.  In order to provide a consistent
+        # interface, we can create a wrapper around sys.stdout that will take
+        # unicode, and automatically encode it to the preferred encoding.
+        # That way consumers can just call get_stdout_text_writer() and write
+        # unicode to the returned stream.  Note that get_stdout_text_writer
+        # just returns sys.stdout in the PY3 section above because python3
+        # handles this.
+        return codecs.getwriter(locale.getpreferredencoding())(sys.stdout)

awscli/customizations/commands.py

@@ -6,6 +6,7 @@
 import awscli
 from awscli.clidocs import CLIDocumentEventHandler
 from awscli.argparser import ArgTableArgParser
+from awscli.argprocess import unpack_argument
 from awscli.clidriver import CLICommand
 from awscli.arguments import CustomArgument
 from awscli.help import HelpCommand
@@ -102,8 +103,24 @@ def __call__(self, args, parsed_globals):
         # We might be able to parse these args so we need to create
         # an arg parser and parse them.
         subcommand_table = self._build_subcommand_table()
-        parser = ArgTableArgParser(self.arg_table, subcommand_table)
+        arg_table = self.arg_table
+        parser = ArgTableArgParser(arg_table, subcommand_table)
         parsed_args, remaining = parser.parse_known_args(args)
+
+        # Unpack arguments
+        for key, value in vars(parsed_args).items():
+            param = None
+            if key in arg_table:
+                param = arg_table[key]
+
+            setattr(parsed_args, key, unpack_argument(
+                self._session,
+                'custom',
+                self.name,
+                param,
+                value
+            ))
+
         if hasattr(parsed_args, 'help'):
             self._display_help(parsed_args, parsed_globals)
         elif getattr(parsed_args, 'subcommand', None) is None:
@@ -155,6 +172,10 @@ def arg_table(self):
     def add_command(cls, command_table, session, **kwargs):
         command_table[cls.NAME] = cls(session)
 
+    @property
+    def name(self):
+        return self.NAME
+
 
 class BasicHelp(HelpCommand):
     event_class = 'command'

awscli/customizations/datapipeline/__init__.py

@@ -170,10 +170,7 @@ class PipelineDefinitionArgument(CustomArgument):
     def add_to_params(self, parameters, value):
         if value is None:
             return
-        new_value = uri_param(self, value)
-        if new_value is not None:
-            value = new_value
-        parsed = json.loads(new_value)
+        parsed = json.loads(value)
         api_objects = translator.definition_to_api(parsed)
         parameters['pipeline_objects'] = api_objects
 

awscli/customizations/datapipeline/translator.py

@@ -80,6 +80,14 @@ def api_to_definition(api_response):
                 value = field['stringValue']
             else:
                 value = {'ref': field['refValue']}
-            current[key] = value
+            if key not in current:
+                current[key] = value
+            elif isinstance(current[key], list):
+                # Dupe keys result in values aggregating
+                # into a list.
+                current[key].append(value)
+            else:
+                converted_list = [current[key], value]
+                current[key] = converted_list
         pipeline_objs.append(current)
     return {'objects': pipeline_objs}

awscli/customizations/s3/comparator.py

@@ -34,6 +34,10 @@ def __init__(self, params=None):
         if 'delete' in params:
             self.delete = params['delete']
 
+        self.compare_on_size_only = False
+        if 'size_only' in params:
+            self.compare_on_size_only = params['size_only']
+
     def call(self, src_files, dest_files):
         """
         This function preforms the actual comparisons.  The parameters it takes
@@ -102,7 +106,12 @@ def call(self, src_files, dest_files):
                     same_size = self.compare_size(src_file, dest_file)
                     same_last_modified_time = self.compare_time(src_file, dest_file)
 
-                    if (not same_size) or (not same_last_modified_time):
+                    if self.compare_on_size_only:
+                        should_sync = not same_size
+                    else:
+                        should_sync = (not same_size) or (not same_last_modified_time)
+
+                    if should_sync:
                         LOG.debug("syncing: %s -> %s, size_changed: %s, "
                                   "last_modified_time_changed: %s",
                                   src_file.src, src_file.dest,

awscli/customizations/s3/fileinfo.py

@@ -2,6 +2,7 @@
 import sys
 import time
 from functools import partial
+import errno
 import hashlib
 
 from dateutil.parser import parse
@@ -13,6 +14,10 @@
         guess_content_type, MD5Error
 
 
+class CreateDirectoryError(Exception):
+    pass
+
+
 def read_file(filename):
     """
     This reads the file into a form that can be sent to S3
@@ -34,8 +39,10 @@ def save_file(filename, response_data, last_update):
     try:
         if not os.path.exists(d):
             os.makedirs(d)
-    except Exception:
-        pass
+    except OSError as e:
+        if not e.errno == errno.EEXIST:
+            raise CreateDirectoryError(
+                "Could not create directory %s: %s" % (d, e))
     md5 = hashlib.md5()
     file_chunks = iter(partial(body.read, 1024 * 1024), b'')
     with open(filename, 'wb') as out_file:

awscli/customizations/s3/s3.py

@@ -796,7 +796,7 @@ def add_verify_ssl(self, parsed_globals):
                                 'sse', 'storage-class', 'content-type',
                                 'cache-control', 'content-disposition',
                                 'content-encoding', 'content-language',
-                                'expires']},
+                                'expires', 'size-only']},
             'ls': {'options': {'nargs': '?', 'default': 's3://'},
                    'params': ['recursive'], 'default': 's3://',
                    'command_class': ListCommand},
@@ -830,7 +830,7 @@ def add_verify_ssl(self, parsed_globals):
                            'dest': 'filters'}},
                'acl': {'options': {'nargs': 1,
                                    'choices': ['private', 'public-read',
-                                               'public-read-write', 
+                                               'public-read-write',
                                                'authenticated-read',
                                                'bucket-owner-read',
                                                'bucket-owner-full-control',
@@ -847,6 +847,9 @@ def add_verify_ssl(self, parsed_globals):
                'content-encoding': {'options': {'nargs': 1}},
                'content-language': {'options': {'nargs': 1}},
                'expires': {'options': {'nargs': 1}},
+               'size-only': {'options': {'action': 'store_true'}, 'documents':
+                   ('Makes the size of each key the only criteria used to '
+                    'decide whether to sync from source to destination.')},
                'index-document': {'options': {}, 'documents':
                    ('A suffix that is appended to a request that is for a '
                     'directory on the website endpoint (e.g. if the suffix '

awscli/customizations/s3/utils.py

@@ -302,14 +302,39 @@ def list_objects(self, bucket, prefix=None):
         kwargs = {'bucket': bucket, 'encoding_type': 'url'}
         if prefix is not None:
             kwargs['prefix'] = prefix
-        pages = self._operation.paginate(self._endpoint, **kwargs)
-        for response, page in pages:
-            contents = page['Contents']
-            for content in contents:
-                source_path = bucket + '/' + unquote_str(content['Key'])
-                size = content['Size']
-                last_update = self._date_parser(content['LastModified'])
-                yield source_path, size, last_update
+        # This event handler is needed because we use encoding_type url and
+        # we're paginating.  The pagination token is the last Key of the
+        # Contents list.  However, botocore does not know that the encoding
+        # type needs to be urldecoded.
+        with ScopedEventHandler(self._operation.session, 'after-call.s3.ListObjects',
+                                self._decode_keys):
+            pages = self._operation.paginate(self._endpoint, **kwargs)
+            for response, page in pages:
+                contents = page['Contents']
+                for content in contents:
+                    source_path = bucket + '/' + content['Key']
+                    size = content['Size']
+                    last_update = self._date_parser(content['LastModified'])
+                    yield source_path, size, last_update
+
+    def _decode_keys(self, parsed, **kwargs):
+        for content in parsed['Contents']:
+            content['Key'] = unquote_str(content['Key'])
+
+
+class ScopedEventHandler(object):
+    """Register an event callback for the duration of a scope."""
+
+    def __init__(self, session, event_name, handler):
+        self._session = session
+        self._event_name = event_name
+        self._handler = handler
+
+    def __enter__(self):
+        self._session.register(self._event_name, self._handler)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        self._session.unregister(self._event_name, self._handler)
 
 
 IORequest = namedtuple('IORequest', ['filename', 'offset', 'data'])