Remove httpretty as a test dep

awscli/customizations/s3/fileinfo.py

@@ -9,7 +9,7 @@
 
 from botocore.compat import quote
 from awscli.customizations.s3.utils import find_bucket_key, \
-        retrieve_http_etag, check_etag, check_error, operate, uni_print, \
+        check_etag, check_error, operate, uni_print, \
         guess_content_type, MD5Error
 
 
@@ -290,7 +290,7 @@ def upload(self):
             }
             self._handle_object_params(params)
             response_data, http = operate(self.service, 'PutObject', params)
-            etag = retrieve_http_etag(http)
+            etag = response_data['ETag'][1:-1]
             body.seek(0)
             check_etag(etag, body)
 

awscli/customizations/s3/tasks.py

@@ -6,7 +6,7 @@
 import threading
 
 from awscli.customizations.s3.utils import find_bucket_key, MD5Error, \
-    operate, retrieve_http_etag, ReadFileChunk, relative_path
+    operate, ReadFileChunk, relative_path
 
 
 LOGGER = logging.getLogger(__name__)
@@ -148,7 +148,7 @@ def __call__(self):
                     self._filename.service, 'UploadPart', params)
             finally:
                 body.close()
-            etag = retrieve_http_etag(http)
+            etag = response_data['ETag'][1:-1]
             self._upload_context.announce_finished_part(
                 etag=etag, part_number=self._part_number)
 

awscli/customizations/s3/utils.py

@@ -79,13 +79,6 @@ def get_file_stat(path):
     return stats.st_size, update_time
 
 
-def retrieve_http_etag(http_response):
-    """
-    Retrieves etag from http response.
-    """
-    return http_response.headers['ETag'][1:-1]
-
-
 def check_etag(etag, fileobj):
     """
     This fucntion checks the etag and the md5 checksum to ensure no

tests/unit/__init__.py

@@ -11,25 +11,21 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 from tests import unittest
-from copy import deepcopy
 import os
-import re
+import copy
 import logging
 
-import httpretty
 import mock
 import six
+import requests
 
 from awscli.clidriver import create_clidriver
-from botocore.payload import Payload
 
 
 class BaseAWSCommandParamsTest(unittest.TestCase):
     maxDiff = None
 
     def setUp(self):
-        httpretty.enable()
-        self.register_uri()
         self.last_params = {}
         # awscli/__init__.py injects AWS_DATA_PATH at import time
         # so that we can find cli.json.  This might be fixed in the
@@ -44,29 +40,33 @@ def setUp(self):
         }
         self.environ_patch = mock.patch('os.environ', self.environ)
         self.environ_patch.start()
+        self.http_response = requests.models.Response()
+        self.http_response.status_code = 200
+        self.parsed_response = {}
+        self.make_request_patch = mock.patch('botocore.endpoint.Endpoint.make_request')
+        self.make_request_is_patched = False
 
     def tearDown(self):
         # This clears all the previous registrations.
-        httpretty.httpretty.reset()
-        httpretty.disable()
         self.environ_patch.stop()
-
-    def register_uri(self):
-        httpretty.register_uri(httpretty.POST, re.compile('.*'), body='')
+        if self.make_request_is_patched:
+            self.make_request_patch.stop()
 
     def before_call(self, params, **kwargs):
         self._store_params(params)
 
     def _store_params(self, params):
-        self.last_params = deepcopy(params)
-        for key in self.last_params:
-            value = self.last_params[key]
-            if isinstance(value, Payload):
-                self.last_params[key] = value.getvalue()
+        self.last_params = params
+
+    def patch_make_request(self):
+        make_request_patch = self.make_request_patch.start()
+        make_request_patch.return_value = (self.http_response, self.parsed_response)
+        self.make_request_is_patched = True
 
     def assert_params_for_cmd(self, cmd, params=None, expected_rc=0,
-                              stderr_contains=None):
+                              stderr_contains=None, ignore_params=None):
         logging.debug("Calling cmd: %s", cmd)
+        self.patch_make_request()
         driver = create_clidriver()
         driver.session.register('before-call', self.before_call)
         if not isinstance(cmd, list):
@@ -86,5 +86,12 @@ def assert_params_for_cmd(self, cmd, params=None, expected_rc=0,
             "Unexpected rc (expected: %s, actual: %s) for command: %s" % (
                 expected_rc, rc, cmd))
         if params is not None:
-            self.assertDictEqual(params, self.last_params)
+            last_params = copy.copy(self.last_params)
+            if ignore_params is not None:
+                for key in ignore_params:
+                    try:
+                        del last_params[key]
+                    except KeyError:
+                        pass
+            self.assertDictEqual(params, last_params)
         return rc

tests/unit/customizations/s3/fake_session.py

@@ -159,7 +159,7 @@ def put_object(self, kwargs):
                 etag = m.hexdigest()
             content['Size'] = len(body)
             content['LastModified'] = '2013-07-15T17:03:43.000Z'
-            content['ETag'] = etag
+            content['ETag'] = '"%s"' % etag
             if 'content_type' in kwargs:
                 content['ContentType'] = kwargs['content_type']
             if key in self.session.s3[bucket]:
@@ -174,7 +174,8 @@ def put_object(self, kwargs):
         elif self.session.connection_error:
             self.session.connection_error = False
             raise requests.ConnectionError()
-        return FakeHttp(etag), response_data
+        response_data['ETag'] = '"%s"' % etag
+        return FakeHttp(), response_data
 
     def create_bucket(self, kwargs):
         """
@@ -188,7 +189,8 @@ def create_bucket(self, kwargs):
             self.session.s3[bucket] = {}
         else:
             response_data['Errors'] = [{'Message': 'Bucket already exists'}]
-        return FakeHttp(etag), response_data
+        response_data['ETag'] = '"%s"' % etag
+        return FakeHttp(), response_data
 
     def delete_object(self, kwargs):
         """
@@ -204,7 +206,8 @@ def delete_object(self, kwargs):
                 self.session.s3[bucket].pop(key)
         else:
             response_data['Errors'] = [{'Message': 'Bucket does not exist'}]
-        return FakeHttp(etag), response_data
+        response_data['ETag'] = '"%s"' % etag
+        return FakeHttp(), response_data
 
     def copy_object(self, kwargs):
         """
@@ -227,7 +230,8 @@ def copy_object(self, kwargs):
             self.session.s3[bucket][key] = src
         else:
             response_data['Errors'] = [{'Message': 'Bucket does not exist'}]
-        return FakeHttp(etag), response_data
+        response_data['ETag'] = '"%s"' % etag
+        return FakeHttp(), response_data
 
     def get_object(self, kwargs):
         """
@@ -265,7 +269,7 @@ def get_object(self, kwargs):
         if self.session.connection_error:
             self.session.connection_error = False
             raise requests.ConnectionError
-        return FakeHttp(etag), response_data
+        return FakeHttp(), response_data
 
     def delete_bucket(self, kwargs):
         """
@@ -282,7 +286,8 @@ def delete_bucket(self, kwargs):
                 response_data['Errors'] = [{'Message': 'Bucket not empty'}]
         else:
             response_data['Errors'] = [{'Message': 'Bucket does not exist'}]
-        return FakeHttp(etag), response_data
+        response_data['ETag'] = '"%s"' % etag
+        return FakeHttp(), response_data
 
     def list_buckets(self, kwargs):
         """
@@ -297,7 +302,8 @@ def list_buckets(self, kwargs):
             response_data['Buckets'].append(bucket_dict)
         response_data['Contents'] = sorted(response_data['Buckets'],
                                            key=lambda k: k['Name'])
-        return FakeHttp(etag), response_data
+        response_data['ETag'] = '"%s"' % etag
+        return FakeHttp(), response_data
 
     def list_objects(self, kwargs):
         """
@@ -330,7 +336,8 @@ def list_objects(self, kwargs):
                 response_data['Contents'].append(key_dict)
         response_data['Contents'] = sorted(response_data['Contents'],
                                            key=lambda k: k['Key'])
-        return FakeHttp(etag), response_data
+        response_data['ETag'] = '"%s"' % etag
+        return FakeHttp(), response_data
 
     def create_multi_upload(self, kwargs):
         """
@@ -343,7 +350,7 @@ def create_multi_upload(self, kwargs):
             key = kwargs['key']
             content['ContentType'] = kwargs.get('content_type')
             self.session.s3[bucket][key] = content
-        return FakeHttp(''), {'UploadId': 'upload_id'}
+        return FakeHttp(), {'UploadId': 'upload_id'}
 
     def upload_part(self, kwargs):
         """
@@ -358,14 +365,14 @@ def complete_multi_upload(self, kwargs):
         A function that acts as a mock function for
         CompleteMultipartUpload calls
         """
-        return FakeHttp(''), {}
+        return FakeHttp(), {}
 
     def abort_multi_upload(self, kwargs):
         """
         A function that acts as a mock function for
         CompleteMultipartUpload calls
         """
-        return FakeHttp(''), {}
+        return FakeHttp(), {}
 
 
 class FakeHttp(object):
@@ -374,5 +381,5 @@ class FakeHttp(object):
     call method.  The http responses are only used to retrieve
     etag's.  So only formatted etag's are included in this class.
     """
-    def __init__(self, etag):
-        self.headers = {'ETag': '\'' + etag + '\''}
+    def __init__(self, status_code=200):
+        self.status_code = status_code

tests/unit/customizations/s3/test_copy_params.py

@@ -17,7 +17,6 @@
 import copy
 
 from tests.unit import BaseAWSCommandParamsTest
-import httpretty
 
 if sys.version_info[:2] == (2, 6):
     from StringIO import StringIO
@@ -41,28 +40,13 @@ def setUp(self):
         super(TestGetObject, self).setUp()
         self.file_path = os.path.join(os.path.dirname(__file__),
                                       'test_copy_params_data')
-        self.payload = None
-
-    def _store_params(self, params):
-        # Copy the params dict without the payload attribute.
-        self.payload = params['payload']
-        self.last_params = params.copy()
-        del self.last_params['payload']
-        self.last_params = copy.deepcopy(self.last_params)
-        # There appears to be a bug in httpretty and python3, and we're not
-        # interested in testing this part of the request serialization for
-        # these tests so we're replacing the file like object with nothing.  We
-        # can still verify that the params['payload'] is the expected file like
-        # object that has the correct contents but we won't test that it's
-        # serialized properly.
-        params['payload'] = None
-
-    def register_uri(self):
-        httpretty.register_uri(httpretty.GET, re.compile('.*'), body='',
-                               status_code=200)
-        httpretty.register_uri(httpretty.PUT, re.compile('.*'), body='',
-                               etag='"120ea8a25e5d487bf68b5f7096440019"',
-                               content_length=0)
+        self.parsed_response = {'ETag': '"120ea8a25e5d487bf68b5f7096440019"',}
+
+    def assert_params(self, cmdline, result):
+        self.assert_params_for_cmd(cmdline, result, expected_rc=0,
+                                   ignore_params=['payload'])
+        self.assertIsInstance(self.last_params['payload'].getvalue(),
+                              file_type)
 
     def test_simple(self):
         cmdline = self.prefix
@@ -71,8 +55,7 @@ def test_simple(self):
         result = {'uri_params': {'Bucket': 'mybucket',
                                  'Key': 'mykey'},
                   'headers': {}}
-        self.assert_params_for_cmd(cmdline, result, expected_rc=0)
-        self.assertIsInstance(self.payload.getvalue(), file_type)
+        self.assert_params(cmdline, result)
 
     def test_sse(self):
         cmdline = self.prefix
@@ -82,8 +65,7 @@ def test_sse(self):
         result = {'uri_params': {'Bucket': 'mybucket',
                                  'Key': 'mykey'},
                   'headers': {'x-amz-server-side-encryption': 'AES256'}}
-        self.assert_params_for_cmd(cmdline, result, expected_rc=0)
-        self.assertIsInstance(self.payload.getvalue(), file_type)
+        self.assert_params(cmdline, result)
 
     def test_storage_class(self):
         cmdline = self.prefix
@@ -93,8 +75,7 @@ def test_storage_class(self):
         result = {'uri_params': {'Bucket': 'mybucket',
                                  'Key': 'mykey'},
                   'headers': {'x-amz-storage-class': 'REDUCED_REDUNDANCY'}}
-        self.assert_params_for_cmd(cmdline, result, expected_rc=0)
-        self.assertIsInstance(self.payload.getvalue(), file_type)
+        self.assert_params(cmdline, result)
 
     def test_website_redirect(self):
         cmdline = self.prefix
@@ -104,8 +85,7 @@ def test_website_redirect(self):
         result = {'uri_params': {'Bucket': 'mybucket',
                                  'Key': 'mykey'},
                   'headers': {'x-amz-website-redirect-location': '/foobar'}}
-        self.assert_params_for_cmd(cmdline, result, expected_rc=0)
-        self.assertIsInstance(self.payload.getvalue(), file_type)
+        self.assert_params(cmdline, result)
 
     def test_acl(self):
         cmdline = self.prefix
@@ -115,8 +95,7 @@ def test_acl(self):
         result = {'uri_params': {'Bucket': 'mybucket',
                                  'Key': 'mykey'},
                   'headers': {'x-amz-acl': 'public-read'}}
-        self.assert_params_for_cmd(cmdline, result, expected_rc=0)
-        self.assertIsInstance(self.payload.getvalue(), file_type)
+        self.assert_params(cmdline, result)
 
     def test_content_params(self):
         cmdline = self.prefix
@@ -132,8 +111,7 @@ def test_content_params(self):
                               'Content-Language': 'piglatin',
                               'Content-Disposition': 'attachment;filename="fname.ext"',
                               'Cache-Control': 'max-age=3600,must-revalidate'}}
-        self.assert_params_for_cmd(cmdline, result, expected_rc=0)
-        self.assertIsInstance(self.payload.getvalue(), file_type)
+        self.assert_params(cmdline, result)
 
     def test_grants(self):
         cmdline = self.prefix
@@ -145,15 +123,15 @@ def test_grants(self):
                                  'Key': 'mykey'},
                   'headers': {'x-amz-grant-full-control': 'alice',
                               'x-amz-grant-read': 'bob'}}
-        self.assert_params_for_cmd(cmdline, result, expected_rc=0)
-        self.assertIsInstance(self.payload.getvalue(), file_type)
+        self.assert_params(cmdline, result)
 
     def test_grants_bad(self):
         cmdline = self.prefix
         cmdline += self.file_path
         cmdline += ' s3://mybucket/mykey'
         cmdline += ' --grants read:bob'
-        self.assert_params_for_cmd(cmdline, expected_rc=1)
+        self.assert_params_for_cmd(cmdline, expected_rc=1,
+                                   ignore_params=['payload'])
 
     def test_content_type(self):
         cmdline = self.prefix
@@ -163,8 +141,7 @@ def test_content_type(self):
         result = {'uri_params': {'Bucket': 'mybucket',
                                  'Key': 'mykey'},
                   'headers': {'Content-Type': 'text/xml'}}
-        self.assert_params_for_cmd(cmdline, result, expected_rc=0)
-        self.assertIsInstance(self.payload.getvalue(), file_type)
+        self.assert_params(cmdline, result)
 
 
 if __name__ == "__main__":