Merge pull request #1010 from kyleknap/bfile

Add read file in bytes using fileb://

CHANGELOG.rst

@@ -19,13 +19,17 @@ Next Release (TBD)
 * bugfix:Timestamp Input: Fix regression where timestamps without any timezone
   information were not being handled properly
   (`issue 982 <https://github.com/aws/aws-cli/issues/982>`__)
+<<<<<<< HEAD
 * bugfix:Signature Version 4: You can enable Signature Version 4 for Amazon S3
   commands by running ``aws configure set default.s3.signature_version s3v4``
   (`issue 1006 <https://github.com/aws/aws-cli/issues/1006>`__,
   `botocore issue 382 <https://github.com/boto/botocore/pull/382>`__)
 * bugfix:``aws emr``: Fix issue where ``--ssh``, ``--get``, ``--put``
   would not work when the cluster was in a waiting state
   (`issue 1007 <https://github.com/aws/aws-cli/issues/1007>`__)
+* feature:Binary File Input: Add support for reading file contents as binary
+  by prepending the filename with ``fileb://``
+  (`issue 1010 <https://github.com/aws/aws-cli/pull/1010>`__)
 
 
 1.6.2

awscli/arguments.py

@@ -435,7 +435,7 @@ def add_to_params(self, parameters, value):
             # below.  Sometimes this can be more complicated, and subclasses
             # can customize as they need.
             unpacked = self._unpack_argument(value)
-            LOG.debug('Unpacked value of "%s" for parameter "%s": %s', value,
+            LOG.debug('Unpacked value of %r for parameter "%s": %r', value,
                       self.py_name, unpacked)
             parameters[self._serialized_name] = unpacked
 

awscli/paramfile.py

@@ -77,18 +77,19 @@ def get_paramfile(path):
     if isinstance(path, six.string_types):
         for prefix in PrefixMap:
             if path.startswith(prefix):
-                data = PrefixMap[prefix](prefix, path)
+                kwargs = KwargsMap.get(prefix, {})
+                data = PrefixMap[prefix](prefix, path, **kwargs)
     return data
 
 
-def get_file(prefix, path):
+def get_file(prefix, path, mode):
     file_path = path[len(prefix):]
     file_path = os.path.expanduser(file_path)
     file_path = os.path.expandvars(file_path)
     if not os.path.isfile(file_path):
         raise ResourceLoadingError("file does not exist: %s" % file_path)
     try:
-        with compat_open(file_path, 'r') as f:
+        with compat_open(file_path, mode) as f:
             return f.read()
     except (OSError, IOError) as e:
         raise ResourceLoadingError('Unable to load paramfile %s: %s' % (
@@ -109,5 +110,11 @@ def get_uri(prefix, uri):
 
 
 PrefixMap = {'file://': get_file,
+             'fileb://': get_file,
              'http://': get_uri,
              'https://': get_uri}
+
+KwargsMap = {'file://': {'mode': 'r'},
+             'fileb://': {'mode': 'rb'},
+             'http://': {},
+             'https://': {}}

awscli/testutils.py

@@ -357,7 +357,7 @@ def __init__(self):
     def remove_all(self):
         shutil.rmtree(self.rootdir)
 
-    def create_file(self, filename, contents, mtime=None):
+    def create_file(self, filename, contents, mtime=None, mode='w'):
         """Creates a file in a tmpdir
 
         ``filename`` should be a relative path, e.g. "foo/bar/baz.txt"
@@ -367,13 +367,16 @@ def create_file(self, filename, contents, mtime=None):
         mtime will be set to the provided value (must be an epoch time).
         Otherwise the mtime is left untouched.
 
+        ``mode`` is the mode the file should be opened either as ``w`` or
+        `wb``.
+
         Returns the full path to the file.
 
         """
         full_path = os.path.join(self.rootdir, filename)
         if not os.path.isdir(os.path.dirname(full_path)):
             os.makedirs(os.path.dirname(full_path))
-        with open(full_path, 'w') as f:
+        with open(full_path, mode) as f:
             f.write(contents)
         current_time = os.path.getmtime(full_path)
         # Subtract a few years off the last modification date.

tests/unit/s3/test_put_object.py

@@ -15,7 +15,7 @@
 import re
 import copy
 
-from awscli.testutils import BaseAWSCommandParamsTest
+from awscli.testutils import BaseAWSCommandParamsTest, FileCreator
 import six
 
 import awscli.clidriver
@@ -30,15 +30,20 @@
     file_type = io.IOBase
 
 
-class TestGetObject(BaseAWSCommandParamsTest):
+class TestPutObject(BaseAWSCommandParamsTest):
 
     maxDiff = None
     prefix = 's3api put-object'
 
     def setUp(self):
-        super(TestGetObject, self).setUp()
+        super(TestPutObject, self).setUp()
         self.file_path = os.path.join(os.path.dirname(__file__),
                                       'test_put_object_data')
+        self.files = FileCreator()
+
+    def tearDown(self):
+        super(TestPutObject, self).tearDown()
+        self.files.remove_all()
 
     def test_simple(self):
         cmdline = self.prefix
@@ -87,6 +92,24 @@ def test_website_redirect(self):
         }
         self.assert_params_for_cmd2(cmdline, expected)
 
+    def test_sse_key_with_binary_file(self):
+        # Create contents that do not get mapped to ascii
+        contents = b'\xc2'
+        filename = self.files.create_file('key', contents, mode='wb')
+        cmdline = self.prefix
+        cmdline += ' --bucket mybucket'
+        cmdline += ' --key mykey'
+        cmdline += ' --sse-customer-algorithm AES256'
+        cmdline += ' --sse-customer-key fileb://%s' % filename
+        expected = {
+            'Bucket': 'mybucket',
+            'Key': 'mykey',
+            'SSECustomerAlgorithm': 'AES256',
+            'SSECustomerKey': 'wg==',  # Note the key gets base64 encoded.
+            'SSECustomerKeyMD5': 'ZGXa0dMXUr4/MoPo9w/u9w=='
+        }
+        self.assert_params_for_cmd2(cmdline, expected)
+
 
 if __name__ == "__main__":
     unittest.main()

tests/unit/test_paramfile.py

@@ -0,0 +1,40 @@
+# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import six
+
+from awscli.paramfile import get_paramfile
+from awscli.testutils import unittest, FileCreator
+
+
+class TestParamFile(unittest.TestCase):
+    def setUp(self):
+        self.files = FileCreator()
+
+    def tearDown(self):
+        self.files.remove_all()
+
+    def test_text_file(self):
+        contents = 'This is a test'
+        filename = self.files.create_file('foo', contents)
+        prefixed_filename = 'file://' + filename
+        data = get_paramfile(prefixed_filename)
+        self.assertEqual(data, contents)
+        self.assertIsInstance(data, six.string_types)
+
+    def test_binary_file(self):
+        contents = 'This is a test'
+        filename = self.files.create_file('foo', contents)
+        prefixed_filename = 'fileb://' + filename
+        data = get_paramfile(prefixed_filename)
+        self.assertEqual(data, b'This is a test')
+        self.assertIsInstance(data, six.binary_type)