Fix issue aws#765 with non ascii characters in user-data

Read in the file:// contents as text, which is decoded via
locale.getpreferredencoding().

This introduces a py2/py3 compat_open so that you can open
a file with a encoding argument (not availble in py2's open()).

awscli/argprocess.py

@@ -444,4 +444,4 @@ def unpack_scalar_cli_arg(parameter, value):
             return False
         return bool(value)
     else:
-        return six.text_type(value)
+        return value

awscli/compat.py

@@ -14,11 +14,31 @@
 import six
 
 if six.PY3:
+    import locale
+
     def get_stdout_text_writer():
         return sys.stdout
+
+    def compat_open(filename, mode='r', encoding=None):
+        """Back-port open() that accepts an encoding argument.
+
+        In python3 this uses the built in open() and in python2 this
+        uses the io.open() function.
+
+        If the file is not being opened in binary mode, then we'll
+        use locale.getpreferredencoding() to find the preferred
+        encoding.
+
+        """
+        if 'b' not in mode:
+            encoding = locale.getpreferredencoding()
+        return open(filename, mode, encoding=encoding)
+
 else:
     import codecs
     import locale
+    import io
+
     def get_stdout_text_writer():
         # In python3, all the sys.stdout/sys.stderr streams are in text
         # mode.  This means they expect unicode, and will encode the
@@ -31,3 +51,9 @@ def get_stdout_text_writer():
         # just returns sys.stdout in the PY3 section above because python3
         # handles this.
         return codecs.getwriter(locale.getpreferredencoding())(sys.stdout)
+
+    def compat_open(filename, mode='r', encoding=None):
+        # See docstring for compat_open in the PY3 section above.
+        if 'b' not in mode:
+            encoding = locale.getpreferredencoding()
+        return io.open(filename, mode, encoding=encoding)

awscli/paramfile.py

@@ -18,6 +18,8 @@
 from botocore.vendored import requests
 import six
 
+from awscli.compat import compat_open
+
 
 logger = logging.getLogger(__name__)
 
@@ -50,7 +52,7 @@ def get_file(prefix, path):
     if not os.path.isfile(file_path):
         raise ResourceLoadingError("file does not exist: %s" % file_path)
     try:
-        with open(file_path) as f:
+        with compat_open(file_path, 'r') as f:
             return f.read()
     except (OSError, IOError) as e:
         raise ResourceLoadingError('Unable to load paramfile %s: %s' % (

tests/__init__.py

@@ -21,12 +21,12 @@
 import mock
 from botocore.hooks import HierarchicalEmitter
 from botocore.session import Session
+import botocore.loaders
 
 import awscli.clidriver
 from awscli.plugin import load_plugins
 from awscli.clidriver import CLIDriver
 from awscli import EnvironmentVariables
-import botocore.loaders
 
 
 # The unittest module got a significant overhaul

tests/unit/ec2/test_run_instances.py

@@ -11,6 +11,9 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
+from awscli.compat import compat_open
+
+from tests import temporary_file
 from tests.unit import BaseAWSCommandParamsTest
 
 
@@ -38,6 +41,22 @@ def test_count_scalar(self):
         }
         self.assert_params_for_cmd(args_list, result)
 
+    def test_user_data(self):
+        data = u'\u0039'
+        with temporary_file('r+') as tmp:
+            with compat_open(tmp.name, 'w') as f:
+                f.write(data)
+                f.flush()
+                args = (
+                    self.prefix +
+                    ' --image-id foo --user-data file://%s' % f.name)
+                result = {'ImageId': 'foo',
+                          'MaxCount': '1',
+                          'MinCount': '1',
+                          # base64 encoded content of utf-8 encoding of data.
+                          'UserData': 'OQ=='}
+            self.assert_params_for_cmd(args, result)
+
     def test_count_range(self):
         args = ' --image-id ami-foobar --count 5:10'
         args_list = (self.prefix + args).split()