From 6d7adbfd85ce1992a77a42b8865890db45189ae2 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Fri, 28 Feb 2014 12:25:33 -0800 Subject: [PATCH] Fix unicode argument processing for py2 In python2, sys.argv is a bytestring of whatever encoding is used by the terminal. In python3, sys.argv is a list of unicode strings. This causes problems because the rest of the code assumes unicode. The fix is to automatically decode to unicode based on sys.stdin as soon as we parse the args. This was originally reported in #593, and https://github.com/boto/botocore/pull/218. I'll need to more investigation to see if this problem applies to JSON files via file://, this commit only fixes the case where unicode is specified on the command line. --- awscli/argparser.py | 10 +++++ awscli/argprocess.py | 13 +++--- .../test_create_application.py | 42 +++++++++++++++++++ 3 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 tests/unit/elasticbeanstalk/test_create_application.py diff --git a/awscli/argparser.py b/awscli/argparser.py index 389d0b1d5c16..387ef0d036fc 100644 --- a/awscli/argparser.py +++ b/awscli/argparser.py @@ -11,6 +11,8 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. import argparse +import sys +import six from difflib import get_close_matches @@ -43,6 +45,14 @@ def _check_value(self, action, value): msg.extend(extra) raise argparse.ArgumentError(action, '\n'.join(msg)) + def parse_known_args(self, args, namespace=None): + parsed, remaining = super(CLIArgParser, self).parse_known_args(args, namespace) + terminal_encoding = getattr(sys.stdin, 'encoding', 'utf-8') + for arg, value in vars(parsed).items(): + if isinstance(value, six.binary_type): + setattr(parsed, arg, value.decode(terminal_encoding)) + return parsed, remaining + class MainArgParser(CLIArgParser): Formatter = argparse.RawTextHelpFormatter diff --git a/awscli/argprocess.py b/awscli/argprocess.py index eeef07d1d813..6aa78eecad6d 100644 --- a/awscli/argprocess.py +++ b/awscli/argprocess.py @@ -65,7 +65,7 @@ def _check_for_uri_param(param, value): try: return get_paramfile(value) except ResourceLoadingError as e: - raise ParamError(param, str(e)) + raise ParamError(param, six.text_type(e)) def detect_shape_structure(param): @@ -163,7 +163,8 @@ def get_parse_method_for_param(self, param, value=None): check_val = value[0] else: check_val = value.strip() - if isinstance(check_val, str) and check_val.startswith(('[', '{')): + if isinstance(check_val, six.string_types) and check_val.startswith( + ('[', '{')): LOG.debug("Param %s looks like JSON, not considered for " "param shorthand.", param.py_name) return @@ -345,7 +346,7 @@ def _split_on_commas(self, value): try: return utils.split_on_commas(value) except ValueError as e: - raise ParamSyntaxError(str(e)) + raise ParamSyntaxError(six.text_type(e)) def unpack_cli_arg(parameter, value): @@ -370,7 +371,7 @@ def unpack_cli_arg(parameter, value): elif parameter.type in COMPLEX_TYPES: return unpack_complex_cli_arg(parameter, value) else: - return str(value) + return six.text_type(value) def unpack_complex_cli_arg(parameter, value): @@ -415,8 +416,8 @@ def unpack_scalar_cli_arg(parameter, value): raise ParamError(parameter, msg) return open(file_path, 'rb') elif parameter.type == 'boolean': - if isinstance(value, str) and value.lower() == 'false': + if isinstance(value, six.string_types) and value.lower() == 'false': return False return bool(value) else: - return str(value) + return six.text_type(value) diff --git a/tests/unit/elasticbeanstalk/test_create_application.py b/tests/unit/elasticbeanstalk/test_create_application.py new file mode 100644 index 000000000000..a12d6d415814 --- /dev/null +++ b/tests/unit/elasticbeanstalk/test_create_application.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# Copyright 2012-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +from tests.unit import BaseAWSCommandParamsTest, unittest +import sys +import six + + +class TestUpdateConfigurationTemplate(BaseAWSCommandParamsTest): + + prefix = 'elasticbeanstalk create-application' + + def test_ascii(self): + cmdline = self.prefix + cmdline += ' --application-name FooBar' + result = {'ApplicationName': 'FooBar',} + self.assert_params_for_cmd(cmdline, result) + + @unittest.skipIf( + six.PY3, 'Unicode cmd line test only is relevant to python2.') + def test_py2_bytestring_unicode(self): + # In Python2, sys.argv is a list of bytestrings that are encoded + # in whatever encoding the terminal uses. We have an extra step + # where we need to decode the bytestring into unicode. In + # python3, sys.argv is a list of unicode objects so this test + # doesn't make sense for python3. + cmdline = self.prefix + app_name = u'\u2713' + cmdline += u' --application-name %s' % app_name + cmdline = cmdline.encode(getattr(sys.stdin, 'encoding', 'utf-8')) + result = {'ApplicationName': u'\u2713',} + self.assert_params_for_cmd(cmdline, result)