Fix unicode argument processing for py2

In python2, sys.argv is a bytestring of whatever encoding
is used by the terminal.  In python3, sys.argv is a list of unicode
strings.  This causes problems because the rest of the code assumes
unicode.

The fix is to automatically decode to unicode based on sys.stdin
as soon as we parse the args.

This was originally reported in aws#593, and
boto/botocore#218.

I'll need to more investigation to see if this problem applies
to JSON files via file://, this commit only fixes the case where
unicode is specified on the command line.

awscli/argparser.py

@@ -11,6 +11,8 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 import argparse
+import sys
+import six
 from difflib import get_close_matches
 
 
@@ -43,6 +45,14 @@ def _check_value(self, action, value):
                 msg.extend(extra)
             raise argparse.ArgumentError(action, '\n'.join(msg))
 
+    def parse_known_args(self, args, namespace=None):
+        parsed, remaining = super(CLIArgParser, self).parse_known_args(args, namespace)
+        terminal_encoding = getattr(sys.stdin, 'encoding', 'utf-8')
+        for arg, value in vars(parsed).items():
+            if isinstance(value, six.binary_type):
+                setattr(parsed, arg, value.decode(terminal_encoding))
+        return parsed, remaining
+
 
 class MainArgParser(CLIArgParser):
     Formatter = argparse.RawTextHelpFormatter

awscli/argprocess.py

@@ -65,7 +65,7 @@ def _check_for_uri_param(param, value):
     try:
         return get_paramfile(value)
     except ResourceLoadingError as e:
-        raise ParamError(param, str(e))
+        raise ParamError(param, six.text_type(e))
 
 
 def detect_shape_structure(param):
@@ -163,7 +163,8 @@ def get_parse_method_for_param(self, param, value=None):
             check_val = value[0]
         else:
             check_val = value.strip()
-        if isinstance(check_val, str) and check_val.startswith(('[', '{')):
+        if isinstance(check_val, six.string_types) and check_val.startswith(
+                ('[', '{')):
             LOG.debug("Param %s looks like JSON, not considered for "
                       "param shorthand.", param.py_name)
             return
@@ -345,7 +346,7 @@ def _split_on_commas(self, value):
         try:
             return utils.split_on_commas(value)
         except ValueError as e:
-            raise ParamSyntaxError(str(e))
+            raise ParamSyntaxError(six.text_type(e))
 
 
 def unpack_cli_arg(parameter, value):
@@ -370,7 +371,7 @@ def unpack_cli_arg(parameter, value):
     elif parameter.type in COMPLEX_TYPES:
         return unpack_complex_cli_arg(parameter, value)
     else:
-        return str(value)
+        return six.text_type(value)
 
 
 def unpack_complex_cli_arg(parameter, value):
@@ -415,8 +416,8 @@ def unpack_scalar_cli_arg(parameter, value):
             raise ParamError(parameter, msg)
         return open(file_path, 'rb')
     elif parameter.type == 'boolean':
-        if isinstance(value, str) and value.lower() == 'false':
+        if isinstance(value, six.string_types) and value.lower() == 'false':
             return False
         return bool(value)
     else:
-        return str(value)
+        return six.text_type(value)

tests/unit/elasticbeanstalk/test_create_application.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+from tests.unit import BaseAWSCommandParamsTest, unittest
+import sys
+import six
+
+
+class TestUpdateConfigurationTemplate(BaseAWSCommandParamsTest):
+
+    prefix = 'elasticbeanstalk create-application'
+
+    def test_ascii(self):
+        cmdline = self.prefix
+        cmdline += ' --application-name FooBar'
+        result = {'ApplicationName': 'FooBar',}
+        self.assert_params_for_cmd(cmdline, result)
+
+    @unittest.skipIf(
+        six.PY3, 'Unicode cmd line test only is relevant to python2.')
+    def test_py2_bytestring_unicode(self):
+        # In Python2, sys.argv is a list of bytestrings that are encoded
+        # in whatever encoding the terminal uses.  We have an extra step
+        # where we need to decode the bytestring into unicode.  In
+        # python3, sys.argv is a list of unicode objects so this test
+        # doesn't make sense for python3.
+        cmdline = self.prefix
+        app_name = u'\u2713'
+        cmdline += u' --application-name %s' % app_name
+        cmdline = cmdline.encode(getattr(sys.stdin, 'encoding', 'utf-8'))
+        result = {'ApplicationName': u'\u2713',}
+        self.assert_params_for_cmd(cmdline, result)