From 49a81a1173553522a6f21da8c5f2a8c5f76a8be4 Mon Sep 17 00:00:00 2001 From: kyleknap Date: Tue, 18 Nov 2014 16:43:27 -0800 Subject: [PATCH 1/3] Add read file in bytes using fileb:// --- awscli/arguments.py | 8 +++++-- awscli/paramfile.py | 13 ++++++++--- tests/unit/s3/test_put_object.py | 28 +++++++++++++++++++--- tests/unit/test_paramfile.py | 40 ++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 tests/unit/test_paramfile.py diff --git a/awscli/arguments.py b/awscli/arguments.py index 2f96cb4369ca..ada21b049f51 100644 --- a/awscli/arguments.py +++ b/awscli/arguments.py @@ -435,8 +435,12 @@ def add_to_params(self, parameters, value): # below. Sometimes this can be more complicated, and subclasses # can customize as they need. unpacked = self._unpack_argument(value) - LOG.debug('Unpacked value of "%s" for parameter "%s": %s', value, - self.py_name, unpacked) + try: + LOG.debug('Unpacked value of "%s" for parameter "%s": %s', + value, self.py_name, unpacked) + except UnicodeDecodeError: + LOG.debug('Unpacked value of "%r" for parameter "%s": %r', + value, self.py_name, unpacked) parameters[self._serialized_name] = unpacked def _unpack_argument(self, value): diff --git a/awscli/paramfile.py b/awscli/paramfile.py index 45eb283a5f69..068c41ebe349 100644 --- a/awscli/paramfile.py +++ b/awscli/paramfile.py @@ -77,18 +77,19 @@ def get_paramfile(path): if isinstance(path, six.string_types): for prefix in PrefixMap: if path.startswith(prefix): - data = PrefixMap[prefix](prefix, path) + kwargs = KwargsMap.get(prefix, {}) + data = PrefixMap[prefix](prefix, path, **kwargs) return data -def get_file(prefix, path): +def get_file(prefix, path, mode): file_path = path[len(prefix):] file_path = os.path.expanduser(file_path) file_path = os.path.expandvars(file_path) if not os.path.isfile(file_path): raise ResourceLoadingError("file does not exist: %s" % file_path) try: - with compat_open(file_path, 'r') as f: + with compat_open(file_path, mode) as f: return f.read() except (OSError, IOError) as e: raise ResourceLoadingError('Unable to load paramfile %s: %s' % ( @@ -109,5 +110,11 @@ def get_uri(prefix, uri): PrefixMap = {'file://': get_file, + 'fileb://': get_file, 'http://': get_uri, 'https://': get_uri} + +KwargsMap = {'file://': {'mode': 'r'}, + 'fileb://': {'mode': 'rb'}, + 'http://': {}, + 'https://': {}} diff --git a/tests/unit/s3/test_put_object.py b/tests/unit/s3/test_put_object.py index 183f0d287beb..dd3b4c02cbab 100644 --- a/tests/unit/s3/test_put_object.py +++ b/tests/unit/s3/test_put_object.py @@ -15,7 +15,7 @@ import re import copy -from awscli.testutils import BaseAWSCommandParamsTest +from awscli.testutils import BaseAWSCommandParamsTest, FileCreator import six import awscli.clidriver @@ -30,15 +30,20 @@ file_type = io.IOBase -class TestGetObject(BaseAWSCommandParamsTest): +class TestPutObject(BaseAWSCommandParamsTest): maxDiff = None prefix = 's3api put-object' def setUp(self): - super(TestGetObject, self).setUp() + super(TestPutObject, self).setUp() self.file_path = os.path.join(os.path.dirname(__file__), 'test_put_object_data') + self.files = FileCreator() + + def tearDown(self): + super(TestPutObject, self).tearDown() + self.files.remove_all() def test_simple(self): cmdline = self.prefix @@ -87,6 +92,23 @@ def test_website_redirect(self): } self.assert_params_for_cmd2(cmdline, expected) + def test_sse_key_with_binary_file(self): + # Create contents that do not get mapped to ascii + contents = '\xc2' + filename = self.files.create_file('key', contents) + cmdline = self.prefix + cmdline += ' --bucket mybucket' + cmdline += ' --key mykey' + cmdline += ' --sse-customer-algorithm AES256' + cmdline += ' --sse-customer-key fileb://%s' % filename + expected = { + 'Bucket': 'mybucket', + 'Key': 'mykey', + 'SSECustomerAlgorithm': 'AES256', + 'SSECustomerKey': contents + } + self.assert_params_for_cmd2(cmdline, expected) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_paramfile.py b/tests/unit/test_paramfile.py new file mode 100644 index 000000000000..571c39bad7c7 --- /dev/null +++ b/tests/unit/test_paramfile.py @@ -0,0 +1,40 @@ +# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. +import six + +from awscli.paramfile import get_paramfile +from awscli.testutils import unittest, FileCreator + + +class TestParamFile(unittest.TestCase): + def setUp(self): + self.files = FileCreator() + + def tearDown(self): + self.files.remove_all() + + def test_text_file(self): + contents = 'This is a test' + filename = self.files.create_file('foo', contents) + prefixed_filename = 'file://' + filename + data = get_paramfile(prefixed_filename) + self.assertEqual(data, contents) + self.assertIsInstance(data, six.string_types) + + def test_binary_file(self): + contents = 'This is a test' + filename = self.files.create_file('foo', contents) + prefixed_filename = 'fileb://' + filename + data = get_paramfile(prefixed_filename) + self.assertEqual(data, contents) + self.assertIsInstance(data, six.binary_type) From 67594ad8480f0db486b3e942c8944813bdd01d44 Mon Sep 17 00:00:00 2001 From: kyleknap Date: Wed, 19 Nov 2014 17:00:12 -0800 Subject: [PATCH 2/3] Update code with botocore changes --- awscli/arguments.py | 8 ++------ awscli/testutils.py | 7 +++++-- tests/unit/s3/test_put_object.py | 7 ++++--- tests/unit/test_paramfile.py | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/awscli/arguments.py b/awscli/arguments.py index ada21b049f51..2f955e8378d9 100644 --- a/awscli/arguments.py +++ b/awscli/arguments.py @@ -435,12 +435,8 @@ def add_to_params(self, parameters, value): # below. Sometimes this can be more complicated, and subclasses # can customize as they need. unpacked = self._unpack_argument(value) - try: - LOG.debug('Unpacked value of "%s" for parameter "%s": %s', - value, self.py_name, unpacked) - except UnicodeDecodeError: - LOG.debug('Unpacked value of "%r" for parameter "%s": %r', - value, self.py_name, unpacked) + LOG.debug('Unpacked value of %r for parameter "%s": %r', value, + self.py_name, unpacked) parameters[self._serialized_name] = unpacked def _unpack_argument(self, value): diff --git a/awscli/testutils.py b/awscli/testutils.py index 28555d47a741..56c23c5e480d 100644 --- a/awscli/testutils.py +++ b/awscli/testutils.py @@ -357,7 +357,7 @@ def __init__(self): def remove_all(self): shutil.rmtree(self.rootdir) - def create_file(self, filename, contents, mtime=None): + def create_file(self, filename, contents, mtime=None, mode='w'): """Creates a file in a tmpdir ``filename`` should be a relative path, e.g. "foo/bar/baz.txt" @@ -367,13 +367,16 @@ def create_file(self, filename, contents, mtime=None): mtime will be set to the provided value (must be an epoch time). Otherwise the mtime is left untouched. + ``mode`` is the mode the file should be opened either as ``w`` or + `wb``. + Returns the full path to the file. """ full_path = os.path.join(self.rootdir, filename) if not os.path.isdir(os.path.dirname(full_path)): os.makedirs(os.path.dirname(full_path)) - with open(full_path, 'w') as f: + with open(full_path, mode) as f: f.write(contents) current_time = os.path.getmtime(full_path) # Subtract a few years off the last modification date. diff --git a/tests/unit/s3/test_put_object.py b/tests/unit/s3/test_put_object.py index dd3b4c02cbab..f90f7680714e 100644 --- a/tests/unit/s3/test_put_object.py +++ b/tests/unit/s3/test_put_object.py @@ -94,8 +94,8 @@ def test_website_redirect(self): def test_sse_key_with_binary_file(self): # Create contents that do not get mapped to ascii - contents = '\xc2' - filename = self.files.create_file('key', contents) + contents = b'\xc2' + filename = self.files.create_file('key', contents, mode='wb') cmdline = self.prefix cmdline += ' --bucket mybucket' cmdline += ' --key mykey' @@ -105,7 +105,8 @@ def test_sse_key_with_binary_file(self): 'Bucket': 'mybucket', 'Key': 'mykey', 'SSECustomerAlgorithm': 'AES256', - 'SSECustomerKey': contents + 'SSECustomerKey': 'wg==', # Note the key gets base64 encoded. + 'SSECustomerKeyMD5': 'ZGXa0dMXUr4/MoPo9w/u9w==' } self.assert_params_for_cmd2(cmdline, expected) diff --git a/tests/unit/test_paramfile.py b/tests/unit/test_paramfile.py index 571c39bad7c7..65fac896247a 100644 --- a/tests/unit/test_paramfile.py +++ b/tests/unit/test_paramfile.py @@ -36,5 +36,5 @@ def test_binary_file(self): filename = self.files.create_file('foo', contents) prefixed_filename = 'fileb://' + filename data = get_paramfile(prefixed_filename) - self.assertEqual(data, contents) + self.assertEqual(data, b'This is a test') self.assertIsInstance(data, six.binary_type) From 9ee426442bbafaec8140b128ed767e570895a785 Mon Sep 17 00:00:00 2001 From: kyleknap Date: Wed, 19 Nov 2014 17:55:12 -0800 Subject: [PATCH 3/3] Update changelog with feature --- CHANGELOG.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8b0a5f683d09..b1478ddf94a5 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -19,6 +19,7 @@ Next Release (TBD) * bugfix:Timestamp Input: Fix regression where timestamps without any timezone information were not being handled properly (`issue 982 `__) +<<<<<<< HEAD * bugfix:Signature Version 4: You can enable Signature Version 4 for Amazon S3 commands by running ``aws configure set default.s3.signature_version s3v4`` (`issue 1006 `__, @@ -26,6 +27,9 @@ Next Release (TBD) * bugfix:``aws emr``: Fix issue where ``--ssh``, ``--get``, ``--put`` would not work when the cluster was in a waiting state (`issue 1007 `__) +* feature:Binary File Input: Add support for reading file contents as binary + by prepending the filename with ``fileb://`` + (`issue 1010 `__) 1.6.2