From e29a9ea9965c5b451576cf9f32e97cbb750e8185 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Fri, 11 Oct 2013 12:38:58 -0700 Subject: [PATCH 1/3] Pull etag from parsed response, not http headers This was working around a botocore bug, but it's now fixed in botocore so this workaround isn't needed. --- awscli/customizations/s3/fileinfo.py | 4 ++-- awscli/customizations/s3/tasks.py | 4 ++-- awscli/customizations/s3/utils.py | 7 ------- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/awscli/customizations/s3/fileinfo.py b/awscli/customizations/s3/fileinfo.py index 83fab4df2c1c..f06b0f907e14 100644 --- a/awscli/customizations/s3/fileinfo.py +++ b/awscli/customizations/s3/fileinfo.py @@ -9,7 +9,7 @@ from botocore.compat import quote from awscli.customizations.s3.utils import find_bucket_key, \ - retrieve_http_etag, check_etag, check_error, operate, uni_print, \ + check_etag, check_error, operate, uni_print, \ guess_content_type, MD5Error @@ -290,7 +290,7 @@ def upload(self): } self._handle_object_params(params) response_data, http = operate(self.service, 'PutObject', params) - etag = retrieve_http_etag(http) + etag = response_data['ETag'][1:-1] body.seek(0) check_etag(etag, body) diff --git a/awscli/customizations/s3/tasks.py b/awscli/customizations/s3/tasks.py index b4c6bdb4f5e5..d53929c1e49f 100644 --- a/awscli/customizations/s3/tasks.py +++ b/awscli/customizations/s3/tasks.py @@ -6,7 +6,7 @@ import threading from awscli.customizations.s3.utils import find_bucket_key, MD5Error, \ - operate, retrieve_http_etag, ReadFileChunk, relative_path + operate, ReadFileChunk, relative_path LOGGER = logging.getLogger(__name__) @@ -148,7 +148,7 @@ def __call__(self): self._filename.service, 'UploadPart', params) finally: body.close() - etag = retrieve_http_etag(http) + etag = response_data['ETag'][1:-1] self._upload_context.announce_finished_part( etag=etag, part_number=self._part_number) diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py index 896b376ba9bb..536088200be2 100644 --- a/awscli/customizations/s3/utils.py +++ b/awscli/customizations/s3/utils.py @@ -79,13 +79,6 @@ def get_file_stat(path): return stats.st_size, update_time -def retrieve_http_etag(http_response): - """ - Retrieves etag from http response. - """ - return http_response.headers['ETag'][1:-1] - - def check_etag(etag, fileobj): """ This fucntion checks the etag and the md5 checksum to ensure no From e24a106476985263b4d3a6660a869d6008a3366c Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Fri, 11 Oct 2013 14:26:55 -0700 Subject: [PATCH 2/3] Remove httpretty as a test dep Instead we patch things out at the botocore level, specifically botocore.endpoint.Endpoint.make_request. --- tests/unit/__init__.py | 40 ++++++----- tests/unit/customizations/s3/fake_session.py | 35 ++++++---- .../customizations/s3/test_copy_params.py | 57 +++++----------- tests/unit/ec2/test_get_password_data.py | 26 ++++--- .../iam/test_create_virtual_mfa_device.py | 26 +++---- tests/unit/output/test_json_output.py | 15 +---- tests/unit/s3/test_get_object.py | 22 +++--- tests/unit/s3/test_list_objects.py | 29 +++----- tests/unit/s3/test_put_bucket_tagging.py | 20 +----- tests/unit/s3/test_put_object.py | 28 ++------ tests/unit/test_clidriver.py | 67 ++++++++++--------- 11 files changed, 151 insertions(+), 214 deletions(-) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index 45a0db3822ca..91c5f4b61d16 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -14,11 +14,12 @@ from copy import deepcopy import os import re +import copy import logging -import httpretty import mock import six +import requests from awscli.clidriver import create_clidriver from botocore.payload import Payload @@ -28,8 +29,6 @@ class BaseAWSCommandParamsTest(unittest.TestCase): maxDiff = None def setUp(self): - httpretty.enable() - self.register_uri() self.last_params = {} # awscli/__init__.py injects AWS_DATA_PATH at import time # so that we can find cli.json. This might be fixed in the @@ -44,29 +43,33 @@ def setUp(self): } self.environ_patch = mock.patch('os.environ', self.environ) self.environ_patch.start() + self.http_response = requests.models.Response() + self.http_response.status_code = 200 + self.parsed_response = {} + self.make_request_patch = mock.patch('botocore.endpoint.Endpoint.make_request') + self.make_request_is_patched = False def tearDown(self): # This clears all the previous registrations. - httpretty.httpretty.reset() - httpretty.disable() self.environ_patch.stop() - - def register_uri(self): - httpretty.register_uri(httpretty.POST, re.compile('.*'), body='') + if self.make_request_is_patched: + self.make_request_patch.stop() def before_call(self, params, **kwargs): self._store_params(params) def _store_params(self, params): - self.last_params = deepcopy(params) - for key in self.last_params: - value = self.last_params[key] - if isinstance(value, Payload): - self.last_params[key] = value.getvalue() + self.last_params = params + + def patch_make_request(self): + make_request_patch = self.make_request_patch.start() + make_request_patch.return_value = (self.http_response, self.parsed_response) + self.make_request_is_patched = True def assert_params_for_cmd(self, cmd, params=None, expected_rc=0, - stderr_contains=None): + stderr_contains=None, ignore_params=None): logging.debug("Calling cmd: %s", cmd) + self.patch_make_request() driver = create_clidriver() driver.session.register('before-call', self.before_call) if not isinstance(cmd, list): @@ -86,5 +89,12 @@ def assert_params_for_cmd(self, cmd, params=None, expected_rc=0, "Unexpected rc (expected: %s, actual: %s) for command: %s" % ( expected_rc, rc, cmd)) if params is not None: - self.assertDictEqual(params, self.last_params) + last_params = copy.copy(self.last_params) + if ignore_params is not None: + for key in ignore_params: + try: + del last_params[key] + except KeyError: + pass + self.assertDictEqual(params, last_params) return rc diff --git a/tests/unit/customizations/s3/fake_session.py b/tests/unit/customizations/s3/fake_session.py index 15fc2ee0f02b..1574a3604aca 100644 --- a/tests/unit/customizations/s3/fake_session.py +++ b/tests/unit/customizations/s3/fake_session.py @@ -159,7 +159,7 @@ def put_object(self, kwargs): etag = m.hexdigest() content['Size'] = len(body) content['LastModified'] = '2013-07-15T17:03:43.000Z' - content['ETag'] = etag + content['ETag'] = '"%s"' % etag if 'content_type' in kwargs: content['ContentType'] = kwargs['content_type'] if key in self.session.s3[bucket]: @@ -174,7 +174,8 @@ def put_object(self, kwargs): elif self.session.connection_error: self.session.connection_error = False raise requests.ConnectionError() - return FakeHttp(etag), response_data + response_data['ETag'] = '"%s"' % etag + return FakeHttp(), response_data def create_bucket(self, kwargs): """ @@ -188,7 +189,8 @@ def create_bucket(self, kwargs): self.session.s3[bucket] = {} else: response_data['Errors'] = [{'Message': 'Bucket already exists'}] - return FakeHttp(etag), response_data + response_data['ETag'] = '"%s"' % etag + return FakeHttp(), response_data def delete_object(self, kwargs): """ @@ -204,7 +206,8 @@ def delete_object(self, kwargs): self.session.s3[bucket].pop(key) else: response_data['Errors'] = [{'Message': 'Bucket does not exist'}] - return FakeHttp(etag), response_data + response_data['ETag'] = '"%s"' % etag + return FakeHttp(), response_data def copy_object(self, kwargs): """ @@ -227,7 +230,8 @@ def copy_object(self, kwargs): self.session.s3[bucket][key] = src else: response_data['Errors'] = [{'Message': 'Bucket does not exist'}] - return FakeHttp(etag), response_data + response_data['ETag'] = '"%s"' % etag + return FakeHttp(), response_data def get_object(self, kwargs): """ @@ -265,7 +269,7 @@ def get_object(self, kwargs): if self.session.connection_error: self.session.connection_error = False raise requests.ConnectionError - return FakeHttp(etag), response_data + return FakeHttp(), response_data def delete_bucket(self, kwargs): """ @@ -282,7 +286,8 @@ def delete_bucket(self, kwargs): response_data['Errors'] = [{'Message': 'Bucket not empty'}] else: response_data['Errors'] = [{'Message': 'Bucket does not exist'}] - return FakeHttp(etag), response_data + response_data['ETag'] = '"%s"' % etag + return FakeHttp(), response_data def list_buckets(self, kwargs): """ @@ -297,7 +302,8 @@ def list_buckets(self, kwargs): response_data['Buckets'].append(bucket_dict) response_data['Contents'] = sorted(response_data['Buckets'], key=lambda k: k['Name']) - return FakeHttp(etag), response_data + response_data['ETag'] = '"%s"' % etag + return FakeHttp(), response_data def list_objects(self, kwargs): """ @@ -330,7 +336,8 @@ def list_objects(self, kwargs): response_data['Contents'].append(key_dict) response_data['Contents'] = sorted(response_data['Contents'], key=lambda k: k['Key']) - return FakeHttp(etag), response_data + response_data['ETag'] = '"%s"' % etag + return FakeHttp(), response_data def create_multi_upload(self, kwargs): """ @@ -343,7 +350,7 @@ def create_multi_upload(self, kwargs): key = kwargs['key'] content['ContentType'] = kwargs.get('content_type') self.session.s3[bucket][key] = content - return FakeHttp(''), {'UploadId': 'upload_id'} + return FakeHttp(), {'UploadId': 'upload_id'} def upload_part(self, kwargs): """ @@ -358,14 +365,14 @@ def complete_multi_upload(self, kwargs): A function that acts as a mock function for CompleteMultipartUpload calls """ - return FakeHttp(''), {} + return FakeHttp(), {} def abort_multi_upload(self, kwargs): """ A function that acts as a mock function for CompleteMultipartUpload calls """ - return FakeHttp(''), {} + return FakeHttp(), {} class FakeHttp(object): @@ -374,5 +381,5 @@ class FakeHttp(object): call method. The http responses are only used to retrieve etag's. So only formatted etag's are included in this class. """ - def __init__(self, etag): - self.headers = {'ETag': '\'' + etag + '\''} + def __init__(self, status_code=200): + self.status_code = status_code diff --git a/tests/unit/customizations/s3/test_copy_params.py b/tests/unit/customizations/s3/test_copy_params.py index 0ebee9318054..64f49baf6d97 100644 --- a/tests/unit/customizations/s3/test_copy_params.py +++ b/tests/unit/customizations/s3/test_copy_params.py @@ -17,7 +17,6 @@ import copy from tests.unit import BaseAWSCommandParamsTest -import httpretty if sys.version_info[:2] == (2, 6): from StringIO import StringIO @@ -41,28 +40,13 @@ def setUp(self): super(TestGetObject, self).setUp() self.file_path = os.path.join(os.path.dirname(__file__), 'test_copy_params_data') - self.payload = None - - def _store_params(self, params): - # Copy the params dict without the payload attribute. - self.payload = params['payload'] - self.last_params = params.copy() - del self.last_params['payload'] - self.last_params = copy.deepcopy(self.last_params) - # There appears to be a bug in httpretty and python3, and we're not - # interested in testing this part of the request serialization for - # these tests so we're replacing the file like object with nothing. We - # can still verify that the params['payload'] is the expected file like - # object that has the correct contents but we won't test that it's - # serialized properly. - params['payload'] = None - - def register_uri(self): - httpretty.register_uri(httpretty.GET, re.compile('.*'), body='', - status_code=200) - httpretty.register_uri(httpretty.PUT, re.compile('.*'), body='', - etag='"120ea8a25e5d487bf68b5f7096440019"', - content_length=0) + self.parsed_response = {'ETag': '"120ea8a25e5d487bf68b5f7096440019"',} + + def assert_params(self, cmdline, result): + self.assert_params_for_cmd(cmdline, result, expected_rc=0, + ignore_params=['payload']) + self.assertIsInstance(self.last_params['payload'].getvalue(), + file_type) def test_simple(self): cmdline = self.prefix @@ -71,8 +55,7 @@ def test_simple(self): result = {'uri_params': {'Bucket': 'mybucket', 'Key': 'mykey'}, 'headers': {}} - self.assert_params_for_cmd(cmdline, result, expected_rc=0) - self.assertIsInstance(self.payload.getvalue(), file_type) + self.assert_params(cmdline, result) def test_sse(self): cmdline = self.prefix @@ -82,8 +65,7 @@ def test_sse(self): result = {'uri_params': {'Bucket': 'mybucket', 'Key': 'mykey'}, 'headers': {'x-amz-server-side-encryption': 'AES256'}} - self.assert_params_for_cmd(cmdline, result, expected_rc=0) - self.assertIsInstance(self.payload.getvalue(), file_type) + self.assert_params(cmdline, result) def test_storage_class(self): cmdline = self.prefix @@ -93,8 +75,7 @@ def test_storage_class(self): result = {'uri_params': {'Bucket': 'mybucket', 'Key': 'mykey'}, 'headers': {'x-amz-storage-class': 'REDUCED_REDUNDANCY'}} - self.assert_params_for_cmd(cmdline, result, expected_rc=0) - self.assertIsInstance(self.payload.getvalue(), file_type) + self.assert_params(cmdline, result) def test_website_redirect(self): cmdline = self.prefix @@ -104,8 +85,7 @@ def test_website_redirect(self): result = {'uri_params': {'Bucket': 'mybucket', 'Key': 'mykey'}, 'headers': {'x-amz-website-redirect-location': '/foobar'}} - self.assert_params_for_cmd(cmdline, result, expected_rc=0) - self.assertIsInstance(self.payload.getvalue(), file_type) + self.assert_params(cmdline, result) def test_acl(self): cmdline = self.prefix @@ -115,8 +95,7 @@ def test_acl(self): result = {'uri_params': {'Bucket': 'mybucket', 'Key': 'mykey'}, 'headers': {'x-amz-acl': 'public-read'}} - self.assert_params_for_cmd(cmdline, result, expected_rc=0) - self.assertIsInstance(self.payload.getvalue(), file_type) + self.assert_params(cmdline, result) def test_content_params(self): cmdline = self.prefix @@ -132,8 +111,7 @@ def test_content_params(self): 'Content-Language': 'piglatin', 'Content-Disposition': 'attachment;filename="fname.ext"', 'Cache-Control': 'max-age=3600,must-revalidate'}} - self.assert_params_for_cmd(cmdline, result, expected_rc=0) - self.assertIsInstance(self.payload.getvalue(), file_type) + self.assert_params(cmdline, result) def test_grants(self): cmdline = self.prefix @@ -145,15 +123,15 @@ def test_grants(self): 'Key': 'mykey'}, 'headers': {'x-amz-grant-full-control': 'alice', 'x-amz-grant-read': 'bob'}} - self.assert_params_for_cmd(cmdline, result, expected_rc=0) - self.assertIsInstance(self.payload.getvalue(), file_type) + self.assert_params(cmdline, result) def test_grants_bad(self): cmdline = self.prefix cmdline += self.file_path cmdline += ' s3://mybucket/mykey' cmdline += ' --grants read:bob' - self.assert_params_for_cmd(cmdline, expected_rc=1) + self.assert_params_for_cmd(cmdline, expected_rc=1, + ignore_params=['payload']) def test_content_type(self): cmdline = self.prefix @@ -163,8 +141,7 @@ def test_content_type(self): result = {'uri_params': {'Bucket': 'mybucket', 'Key': 'mykey'}, 'headers': {'Content-Type': 'text/xml'}} - self.assert_params_for_cmd(cmdline, result, expected_rc=0) - self.assertIsInstance(self.payload.getvalue(), file_type) + self.assert_params(cmdline, result) if __name__ == "__main__": diff --git a/tests/unit/ec2/test_get_password_data.py b/tests/unit/ec2/test_get_password_data.py index f6301d288c12..502c26e52e7c 100644 --- a/tests/unit/ec2/test_get_password_data.py +++ b/tests/unit/ec2/test_get_password_data.py @@ -16,28 +16,26 @@ import re from six.moves import cStringIO -import httpretty import mock -GET_PASSWORD_DATA_RESPONSE = """ - - 000000000000 - i-12345678 - 2013-07-27T18:29:23.000Z - -GWDnuoj/7pbMQkg125E8oGMUVCI+r98sGbFFl8SX+dEYxMZzz+byYwwjvyg8iSGKaLuLTIWatWopVu5cMWDKH65U4YFL2g3LqyajBrCFnuSE1piTeS/rPQpoSvBN5FGj9HWqNrglWAJgh9OZNSGgpEojBenL/0rwSpDWL7f/f52M5doYA6q+v0ygEoi1Wq6hcmrBfyA4seW1RlKgnUru5Y9oc1hFHi53E3b1EkjGqCsCemVUwumBj8uwCLJRaMcqrCxK1smtAsiSqk0Jk9jpN2vcQgnMPypEdmEEXyWHwq55fjy6ch+sqYcwumIL5QcFW2JQ5+XBEoFhC66gOsAXow== - -""" -PASSWORD_DATA = "GWDnuoj/7pbMQkg125E8oGMUVCI+r98sGbFFl8SX+dEYxMZzz+byYwwjvyg8iSGKaLuLTIWatWopVu5cMWDKH65U4YFL2g3LqyajBrCFnuSE1piTeS/rPQpoSvBN5FGj9HWqNrglWAJgh9OZNSGgpEojBenL/0rwSpDWL7f/f52M5doYA6q+v0ygEoi1Wq6hcmrBfyA4seW1RlKgnUru5Y9oc1hFHi53E3b1EkjGqCsCemVUwumBj8uwCLJRaMcqrCxK1smtAsiSqk0Jk9jpN2vcQgnMPypEdmEEXyWHwq55fjy6ch+sqYcwumIL5QcFW2JQ5+XBEoFhC66gOsAXow==" +PASSWORD_DATA = ("GWDnuoj/7pbMQkg125E8oGMUVCI+r98sGbFFl8SX+dEYxMZzz+byYwwjvyg8i" + "SGKaLuLTIWatWopVu5cMWDKH65U4YFL2g3LqyajBrCFnuSE1piTeS/rPQpoSv" + "BN5FGj9HWqNrglWAJgh9OZNSGgpEojBenL/0rwSpDWL7f/f52M5doYA6q+v0y" + "gEoi1Wq6hcmrBfyA4seW1RlKgnUru5Y9oc1hFHi53E3b1EkjGqCsCemVUwumB" + "j8uwCLJRaMcqrCxK1smtAsiSqk0Jk9jpN2vcQgnMPypEdmEEXyWHwq55fjy6c" + "h+sqYcwumIL5QcFW2JQ5+XBEoFhC66gOsAXow==") + class TestGetPasswordData(BaseAWSCommandParamsTest): prefix = 'ec2 get-password-data' - def register_uri(self): - httpretty.register_uri(httpretty.POST, re.compile('.*'), - body=GET_PASSWORD_DATA_RESPONSE) + def setUp(self): + super(TestGetPasswordData, self).setUp() + self.parsed_response = {'InstanceId': 'i-12345678', + 'Timestamp': '2013-07-27T18:29:23.000Z', + 'PasswordData': PASSWORD_DATA} def test_no_priv_launch_key(self): captured = cStringIO() diff --git a/tests/unit/iam/test_create_virtual_mfa_device.py b/tests/unit/iam/test_create_virtual_mfa_device.py index fddd20425d72..d8b456a450a1 100644 --- a/tests/unit/iam/test_create_virtual_mfa_device.py +++ b/tests/unit/iam/test_create_virtual_mfa_device.py @@ -17,28 +17,21 @@ from six.moves import cStringIO import mock -import httpretty -BODY = """ - - - iVBORw0KGgoAAAANSUhEUgAAAPoAAAD6CAIAAAAHjs1qAAAFiElEQVR42u3bQW7jMBAEwPz/07vHvS0QeLpnZFVf7cgyWRTgJvPzR+Q1+TEEgrsI7iK4i+AugrsI7iK4i+AugrsI7iK4C+4iuIvgLoK7CO4iuIvgLoK7CO4iBe4/rfz/c391k7lLffK5v/r6tZu8Ofu444477rjjjjvuuOOOO+6xwcoZ/WTl5D53cNXlZqG2VPpXxh133HHHHXfccccdd9xxD09/rU7ZylZdVnvo5BY/7rjjjjvuuOOOO+64447713H/RFIOZW0944477rjjjjvuuOOOO+64417ZVM8ZPbLHXiOLO+6444477rjjjjvuuOP+fO5bC2lwMeQKoiN/ew0l7rjjjjvuuOOOO+644457uCLYGmivHuxeBmcfd6/ijrtXccfdq7jj7lXcH5han3Bkeedana9SgTvuuOOOO+6444477ri/hXttc7umMLc2tm5ycI4GnzJfWETijjvuuOOOO+6444477s9oZnKDlZvvGp2tnf/BNx8/NYA77rjjjjvuuOOOO+64v5V7rZkZHKzalXOwcg3J1qorFGK444477rjjjjvuuOOO+1u5fwLrSG9T6zEGxyr3FWqPwkeemcEdd9xxxx133HHHHXfcj3Kv4chNUo17rU7ZmtDjaxJ33HHHHXfccccdd9xxfw33rWYm12McOY/wBQupZgN33HHHHXfccccdd9xxx/2DwTpy0Pzm2sh9oyOgc+sZd9xxxx133HHHHXfcccc9liOVSE3SkYIot2KPFES444477rjjjjvuuOOO+/u4DzYzgxoGpyH35q3R2Nrb758awB133HHHHXfccccdd9zfyj23NmpX3jomkBuc3N7+4DIr+MYdd9xxxx133HHHHXfc38q91gkcUZi7jcGd/9xIHnkG4Y477rjjjjvuuOOOO+6430gOR20hbVUTW4cIrg0O7rjjjjvuuOOOO+644/4a7keKi8G+6At273Nv3vpc3HHHHXfccccdd9xxxx332LjXyocanUfgqJ0pqNU4uOOOO+6444477rjjjjvuS13E1l53jc5Ndjf144477rjjjjvuuOOOO+64x77Skf352mZ+7VhEbo62DiBE6jvccccdd9xxxx133HHHHffs9A8WNbnprz0atv7rYPDrP/IQAe6444477rjjjjvuuON+lPvguK//ci+TrS3RwZ7q5nMEd9xxxx133HHHHXfccce9VSDkippcY5A7nrBVaxyp2nDHHXfccccdd9xxxx133FuTVFs5tUttlVq5pZJ7cCy0arjjjjvuuOOOO+6444477tmd8K0SIDdnNXZHirj+YwV33HHHHXfccccdd9xxfw33mu+bHUiObK4By7E7jgF33HHHHXfccccdd9xxfw33Whcx+Kt/a1Zyn5tbZltkcccdd9xxxx133HHHHXfcb4DOnRp4YgeSQ/l9jRDuuOOOO+6444477rjjjntM4WCBMGhlcHByt1GrcbZOSeCOO+6444477rjjjjvuuD8htUMENcFbN1mrcXIfhDvuuOOOO+6444477rjjvvSrP3cbg3NWG7pcMTV4OuPtRSTuuOOOO+6444477rjjfoV7rXvZWjlbVr6+5CkUNbjjjjvuuOOOO+6444477qsacn+7VeMcaYRyVdsjDxHgjjvuuOOOO+6444477rinNNR6myfexlbXpJnBHXfccccdd9xxxx133J/PvbAFPT5nN4upLQy444477rjjjjvuuOOOO+4t7rWx27pybY/9iLMjvQ3uuOOOO+6444477rjjjvvSfB/Zr86t58H53qpxBpdoYZxxxx133HHHHXfccccd99dwF3lKcBfcRXAXwV0EdxHcRXAXwV0EdxHcRXAXwV1wF8FdBHcR3EVwF8FdBHcR3EVwF8Fd5F/+AgASajf850wfAAAAAElFTkSuQmCC - VFpYTVc2V1lIUFlFRFczSVhLUlpRUTJRVFdUSFRNRDNTQ0c3TkZDUVdQWDVETlNWM0IyUENaQVpWTEpQTlBOTA== - arn:aws:iam::419278470775:mfa/fiebaz - - - - 3d127f08-0f41-11e3-941e-8d1b33bbf528 - -""" - class TestCreateVirtualMFADevice(BaseAWSCommandParamsTest): prefix = 'iam create-virtual-mfa-device' - def register_uri(self): - httpretty.register_uri(httpretty.POST, re.compile('.*'), body=BODY) + def setUp(self): + super(TestCreateVirtualMFADevice, self).setUp() + self.parsed_response = { + "VirtualMFADevice": { + "Base32StringSeed": "VFpYTVc2V1lIUFlFRFczSVhLUlpRUTJRVFdUSFRNRDNTQ0c3TkZDUVdQWDVETlNWM0IyUENaQVpWTEpQTlBOTA==", + "SerialNumber": "arn:aws:iam::419278470775:mfa/fiebaz", + "QRCodePNG": "iVBORw0KGgoAAAANSUhEUgAAAPoAAAD6CAIAAAAHjs1qAAAFiElEQVR42u3bQW7jMBAEwPz/07vHvS0QeLpnZFVf7cgyWRTgJvPzR+Q1+TEEgrsI7iK4i+AugrsI7iK4i+AugrsI7iK4C+4iuIvgLoK7CO4iuIvgLoK7CO4iBe4/rfz/c391k7lLffK5v/r6tZu8Ofu444477rjjjjvuuOOOO+6xwcoZ/WTl5D53cNXlZqG2VPpXxh133HHHHXfccccdd9xxD09/rU7ZylZdVnvo5BY/7rjjjjvuuOOOO+64447713H/RFIOZW0944477rjjjjvuuOOOO+64417ZVM8ZPbLHXiOLO+6444477rjjjjvuuOP+fO5bC2lwMeQKoiN/ew0l7rjjjjvuuOOOO+644457uCLYGmivHuxeBmcfd6/ijrtXccfdq7jj7lXcH5han3Bkeedana9SgTvuuOOOO+6444477ri/hXttc7umMLc2tm5ycI4GnzJfWETijjvuuOOOO+6444477s9oZnKDlZvvGp2tnf/BNx8/NYA77rjjjjvuuOOOO+64v5V7rZkZHKzalXOwcg3J1qorFGK444477rjjjjvuuOOO+1u5fwLrSG9T6zEGxyr3FWqPwkeemcEdd9xxxx133HHHHXfcj3Kv4chNUo17rU7ZmtDjaxJ33HHHHXfccccdd9xxfw33rWYm12McOY/wBQupZgN33HHHHXfccccdd9xxx/2DwTpy0Pzm2sh9oyOgc+sZd9xxxx133HHHHXfcccc9liOVSE3SkYIot2KPFES444477rjjjjvuuOOO+/u4DzYzgxoGpyH35q3R2Nrb758awB133HHHHXfccccdd9zfyj23NmpX3jomkBuc3N7+4DIr+MYdd9xxxx133HHHHXfc38q91gkcUZi7jcGd/9xIHnkG4Y477rjjjjvuuOOOO+6430gOR20hbVUTW4cIrg0O7rjjjjvuuOOOO+644/4a7keKi8G+6At273Nv3vpc3HHHHXfccccdd9xxxx332LjXyocanUfgqJ0pqNU4uOOOO+6444477rjjjjvuS13E1l53jc5Ndjf144477rjjjjvuuOOOO+64x77Skf352mZ+7VhEbo62DiBE6jvccccdd9xxxx133HHHHffs9A8WNbnprz0atv7rYPDrP/IQAe6444477rjjjjvuuON+lPvguK//ci+TrS3RwZ7q5nMEd9xxxx133HHHHXfccce9VSDkippcY5A7nrBVaxyp2nDHHXfccccdd9xxxx133FuTVFs5tUttlVq5pZJ7cCy0arjjjjvuuOOOO+6444477tmd8K0SIDdnNXZHirj+YwV33HHHHXfccccdd9xxfw33mu+bHUiObK4By7E7jgF33HHHHXfccccdd9xxfw33Whcx+Kt/a1Zyn5tbZltkcccdd9xxxx133HHHHXfcb4DOnRp4YgeSQ/l9jRDuuOOOO+6444477rjjjntM4WCBMGhlcHByt1GrcbZOSeCOO+6444477rjjjjvuuD8htUMENcFbN1mrcXIfhDvuuOOOO+6444477rjjvvSrP3cbg3NWG7pcMTV4OuPtRSTuuOOOO+6444477rjjfoV7rXvZWjlbVr6+5CkUNbjjjjvuuOOOO+6444477qsacn+7VeMcaYRyVdsjDxHgjjvuuOOOO+6444477rinNNR6myfexlbXpJnBHXfccccdd9xxxx133J/PvbAFPT5nN4upLQy444477rjjjjvuuOOOO+4t7rWx27pybY/9iLMjvQ3uuOOOO+6444477rjjjvvSfB/Zr86t58H53qpxBpdoYZxxxx133HHHHXfccccd99dwF3lKcBfcRXAXwV0EdxHcRXAXwV0EdxHcRXAXwV1wF8FdBHcR3EVwF8FdBHcR3EVwF8Fd5F/+AgASajf850wfAAAAAElFTkSuQmCC", + } + } def getpath(self, filename): return os.path.join(os.path.abspath(os.path.dirname(__file__)), @@ -78,4 +71,3 @@ def test_bad_filename(self): result = {} with mock.patch('sys.stderr', captured): self.assert_params_for_cmd(cmdline, result, expected_rc=255) - diff --git a/tests/unit/output/test_json_output.py b/tests/unit/output/test_json_output.py index d4c24d9ef34a..4bb6f2f8087e 100644 --- a/tests/unit/output/test_json_output.py +++ b/tests/unit/output/test_json_output.py @@ -17,28 +17,19 @@ import re from six.moves import cStringIO -import httpretty import mock -ADD_USER_RESPONSE = """\ - - - b8ff9277-0c3a-11e3-941e-8d1b33bbf528 - - -""" - class TestGetPasswordData(BaseAWSCommandParamsTest): prefix = 'iam add-user-to-group ' - def register_uri(self): - httpretty.register_uri(httpretty.POST, re.compile('.*'), - body=ADD_USER_RESPONSE) def test_empty_response_prints_nothing(self): captured = cStringIO() + # This is the default response, but we want to be explicit + # that we're returning an empty dict. + self.parsed_response = {} args = ' --group-name foo --user-name bar' cmdline = self.prefix + args result = {'GroupName': 'foo', 'UserName': 'bar'} diff --git a/tests/unit/s3/test_get_object.py b/tests/unit/s3/test_get_object.py index 3bd0e8a91052..1d623fa08c18 100644 --- a/tests/unit/s3/test_get_object.py +++ b/tests/unit/s3/test_get_object.py @@ -15,7 +15,7 @@ import os import re -import httpretty +import six import awscli.clidriver @@ -24,8 +24,9 @@ class TestGetObject(BaseAWSCommandParamsTest): prefix = 's3api get-object' - def register_uri(self): - httpretty.register_uri(httpretty.GET, re.compile('.*'), body='') + def setUp(self): + super(TestGetObject, self).setUp() + self.parsed_response = {'Body': six.StringIO()} def remove_file_if_exists(self, filename): if os.path.isfile(filename): @@ -38,10 +39,9 @@ def test_simple(self): cmdline += ' outfile' result = {'uri_params': {'Bucket': 'mybucket', 'Key': 'mykey'}, - 'headers': {}, - 'payload': None} + 'headers': {},} self.addCleanup(self.remove_file_if_exists, 'outfile') - self.assert_params_for_cmd(cmdline, result) + self.assert_params_for_cmd(cmdline, result, ignore_params=['payload']) def test_range(self): cmdline = self.prefix @@ -51,10 +51,9 @@ def test_range(self): cmdline += ' outfile' result = {'uri_params': {'Bucket': 'mybucket', 'Key': 'mykey'}, - 'headers': {'Range': 'bytes=0-499'}, - 'payload': None} + 'headers': {'Range': 'bytes=0-499'},} self.addCleanup(self.remove_file_if_exists, 'outfile') - self.assert_params_for_cmd(cmdline, result) + self.assert_params_for_cmd(cmdline, result, ignore_params=['payload']) def test_response_headers(self): cmdline = self.prefix @@ -67,10 +66,9 @@ def test_response_headers(self): 'Key': 'mykey', 'ResponseCacheControl': 'No-cache', 'ResponseContentEncoding': 'x-gzip'}, - 'headers': {}, - 'payload': None} + 'headers': {},} self.addCleanup(self.remove_file_if_exists, 'outfile') - self.assert_params_for_cmd(cmdline, result) + self.assert_params_for_cmd(cmdline, result, ignore_params=['payload']) if __name__ == "__main__": diff --git a/tests/unit/s3/test_list_objects.py b/tests/unit/s3/test_list_objects.py index cb6c95fe86e8..4a7773f88d36 100644 --- a/tests/unit/s3/test_list_objects.py +++ b/tests/unit/s3/test_list_objects.py @@ -13,26 +13,22 @@ # language governing permissions and limitations under the License. from tests.unit import BaseAWSCommandParamsTest import re -import httpretty class TestListObjects(BaseAWSCommandParamsTest): prefix = 's3api list-objects' - def register_uri(self): - body = """ - - """ - httpretty.register_uri(httpretty.GET, re.compile('.*'), body=body) + def setUp(self): + super(TestListObjects, self).setUp() + self.parsed_response = {'Contents': []} def test_simple(self): cmdline = self.prefix cmdline += ' --bucket mybucket' result = {'uri_params': {'Bucket': 'mybucket'}, - 'headers': {}, - 'payload': None} - self.assert_params_for_cmd(cmdline, result) + 'headers': {},} + self.assert_params_for_cmd(cmdline, result, ignore_params=['payload']) def test_maxkeys(self): cmdline = self.prefix @@ -41,9 +37,8 @@ def test_maxkeys(self): # show up in the result params. cmdline += ' --max-items 100' result = {'uri_params': {'Bucket': 'mybucket'}, - 'headers': {}, - 'payload': None} - self.assert_params_for_cmd(cmdline, result) + 'headers': {},} + self.assert_params_for_cmd(cmdline, result, ignore_params=['payload']) def test_starting_token(self): # We don't need to test this in depth because botocore @@ -53,17 +48,15 @@ def test_starting_token(self): cmdline += ' --bucket mybucket' cmdline += ' --starting-token foo___2' result = {'uri_params': {'Bucket': 'mybucket', 'Marker': 'foo'}, - 'headers': {}, - 'payload': None} - self.assert_params_for_cmd(cmdline, result) + 'headers': {},} + self.assert_params_for_cmd(cmdline, result, ignore_params=['payload']) def test_no_paginate(self): cmdline = self.prefix cmdline += ' --bucket mybucket --no-paginate' result = {'uri_params': {'Bucket': 'mybucket'}, - 'headers': {}, - 'payload': None} - self.assert_params_for_cmd(cmdline, result) + 'headers': {},} + self.assert_params_for_cmd(cmdline, result, ignore_params=['payload']) if __name__ == "__main__": diff --git a/tests/unit/s3/test_put_bucket_tagging.py b/tests/unit/s3/test_put_bucket_tagging.py index b97d256c5881..f32d502bdf38 100644 --- a/tests/unit/s3/test_put_bucket_tagging.py +++ b/tests/unit/s3/test_put_bucket_tagging.py @@ -15,7 +15,6 @@ import copy from tests.unit import BaseAWSCommandParamsTest -import httpretty import six @@ -40,30 +39,13 @@ def setUp(self): super(TestPutBucketTagging, self).setUp() self.payload = None - def _store_params(self, params): - # Copy the params dict without the payload attribute. - self.payload = params['payload'] - self.last_params = params.copy() - del self.last_params['payload'] - self.last_params = copy.deepcopy(self.last_params) - # There appears to be a bug in httpretty and python3, and we're not - # interested in testing this part of the request serialization for - # these tests so we're replacing the file like object with nothing. We - # can still verify that the params['payload'] is the expected file like - # object that has the correct contents but we won't test that it's - # serialized properly. - params['payload'] = None - - def register_uri(self): - httpretty.register_uri(httpretty.PUT, re.compile('.*'), body='') - def test_simple(self): cmdline = self.prefix cmdline += ' --bucket mybucket' cmdline += ' --tagging %s' % TAGSET result = {'uri_params': {'Bucket': 'mybucket'}, 'headers': {'Content-MD5': '5s++BGwLE2moBAK9duxpFw=='}} - self.assert_params_for_cmd(cmdline, result) + self.assert_params_for_cmd(cmdline, result, ignore_params=['payload']) if __name__ == "__main__": diff --git a/tests/unit/s3/test_put_object.py b/tests/unit/s3/test_put_object.py index 370ecdb52531..74c1d58def64 100644 --- a/tests/unit/s3/test_put_object.py +++ b/tests/unit/s3/test_put_object.py @@ -16,7 +16,6 @@ import copy from tests.unit import BaseAWSCommandParamsTest -import httpretty import six import awscli.clidriver @@ -39,24 +38,6 @@ def setUp(self): super(TestGetObject, self).setUp() self.file_path = os.path.join(os.path.dirname(__file__), 'test_put_object_data') - self.payload = None - - def _store_params(self, params): - # Copy the params dict without the payload attribute. - self.payload = params['payload'] - self.last_params = params.copy() - del self.last_params['payload'] - self.last_params = copy.deepcopy(self.last_params) - # There appears to be a bug in httpretty and python3, and we're not - # interested in testing this part of the request serialization for - # these tests so we're replacing the file like object with nothing. We - # can still verify that the params['payload'] is the expected file like - # object that has the correct contents but we won't test that it's - # serialized properly. - params['payload'] = None - - def register_uri(self): - httpretty.register_uri(httpretty.PUT, re.compile('.*'), body='') def test_simple(self): cmdline = self.prefix @@ -66,8 +47,8 @@ def test_simple(self): result = {'uri_params': {'Bucket': 'mybucket', 'Key': 'mykey'}, 'headers': {}} - self.assert_params_for_cmd(cmdline, result) - self.assertIsInstance(self.payload.getvalue(), file_type) + self.assert_params_for_cmd(cmdline, result, ignore_params=['payload']) + self.assertIsInstance(self.last_params['payload'].getvalue(), file_type) def test_headers(self): cmdline = self.prefix @@ -81,8 +62,9 @@ def test_headers(self): 'headers': {'x-amz-acl': 'public-read', 'Content-Encoding': 'x-gzip', 'Content-Type': 'text/plain'}} - self.assert_params_for_cmd(cmdline, result) - self.assertEqual(self.payload.getvalue().read(), b'This is a test.') + self.assert_params_for_cmd(cmdline, result, ignore_params=['payload']) + payload = self.last_params['payload'].getvalue() + self.assertEqual(payload.name, self.file_path) if __name__ == "__main__": diff --git a/tests/unit/test_clidriver.py b/tests/unit/test_clidriver.py index 5668fd4d44f6..e478a41b6e45 100644 --- a/tests/unit/test_clidriver.py +++ b/tests/unit/test_clidriver.py @@ -15,7 +15,7 @@ import mock import six -import httpretty +import requests.models import awscli from awscli.clidriver import CLIDriver @@ -24,6 +24,7 @@ from botocore.hooks import HierarchicalEmitter from botocore.base import get_search_path from botocore.provider import Provider +import botocore.service GET_DATA = { @@ -292,24 +293,34 @@ def tearDown(self): super(TestAWSCommand, self).tearDown() self.stderr_patch.stop() - def last_request_headers(self): - return httpretty.httpretty.last_request.headers + def record_get_endpoint_args(self, *args, **kwargs): + self.get_endpoint_args = (args, kwargs) + self.real_get_endpoint(*args, **kwargs) - def test_aws_with_region(self): - driver = create_clidriver() - driver.main('ec2 describe-instances --region us-east-1'.split()) - host = self.last_request_headers()['Host'] - self.assertEqual(host, 'ec2.us-east-1.amazonaws.com') + def test_aws_with_endpoint_url(self): + with mock.patch('botocore.service.Service.get_endpoint') as endpoint: + http_response = requests.models.Response() + http_response.status_code = 200 + endpoint.return_value.make_request.return_value = ( + http_response, {}) + self.assert_params_for_cmd( + 'ec2 describe-instances --endpoint-url https://foobar.com/', + expected_rc=0) + endpoint.assert_called_with(region_name=None, + endpoint_url='https://foobar.com/') - driver.main('ec2 describe-instances --region us-west-2'.split()) - host = self.last_request_headers()['Host'] - self.assertEqual(host, 'ec2.us-west-2.amazonaws.com') + def test_aws_with_region(self): + with mock.patch('botocore.service.Service.get_endpoint') as endpoint: + http_response = requests.models.Response() + http_response.status_code = 200 + endpoint.return_value.make_request.return_value = ( + http_response, {}) + self.assert_params_for_cmd( + 'ec2 describe-instances --region us-east-1', + expected_rc=0) + endpoint.assert_called_with(region_name='us-east-1', + endpoint_url=None) - def test_aws_with_endpoint_url(self): - driver = create_clidriver() - driver.main('ec2 describe-instances --endpoint-url https://foobar.com/'.split()) - host = self.last_request_headers()['Host'] - self.assertEqual(host, 'foobar.com') def inject_new_param(self, argument_table, **kwargs): argument = CustomArgument('unknown-arg', {}) @@ -338,6 +349,7 @@ def test_event_emission_for_top_level_params(self): # Now we should get an rc of 0 as the arg is expected # (though nothing actually does anything with the arg). + self.patch_make_request() rc = driver.main('ec2 describe-instances --unknown-arg foo'.split()) self.assertEqual(rc, 0) self.assertEqual(len(args_seen), 1) @@ -348,8 +360,7 @@ def test_empty_params_gracefully_handled(self): # Simulates the equivalent in bash: --identifies "" cmd = 'ses get-identity-dkim-attributes --identities'.split() cmd.append('') - rc = driver.main(cmd) - self.assertEqual(rc, 0) + self.assert_params_for_cmd(cmd,expected_rc=0) def test_file_param_does_not_exist(self): driver = create_clidriver() @@ -373,19 +384,15 @@ def tearDown(self): super(TestHTTPParamFileDoesNotExist, self).tearDown() self.stderr_patch.stop() - def register_uri(self): - httpretty.register_uri(httpretty.GET, 'http://does/not/exist.json', - body='', status=404) - def test_http_file_param_does_not_exist(self): - driver = create_clidriver() - rc = driver.main('ec2 describe-instances ' - '--filters http://does/not/exist.json'.split()) - self.assertEqual(rc, 255) - self.assertIn("Bad value for argument '--filters': " - "Unable to retrieve http://does/not/exist.json: " - "received non 200 status code of 404", - self.stderr.getvalue()) + error_msg = ("Bad value for argument '--filters': " + "Unable to retrieve http://does/not/exist.json: " + "received non 200 status code of 404") + with mock.patch('requests.get') as get: + get.return_value.status_code = 404 + self.assert_params_for_cmd( + 'ec2 describe-instances --filters http://does/not/exist.json', + expected_rc=255, stderr_contains=error_msg) From 9b59102c3dc0a809a8e46d14e9b70bb857d822dc Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Fri, 11 Oct 2013 15:02:10 -0700 Subject: [PATCH 3/3] Remove unused imports --- tests/unit/__init__.py | 3 --- tests/unit/test_argprocess.py | 15 +++++---------- tests/unit/test_clidriver.py | 2 -- tests/unit/test_plugin.py | 2 +- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py index 91c5f4b61d16..432d6fab2964 100644 --- a/tests/unit/__init__.py +++ b/tests/unit/__init__.py @@ -11,9 +11,7 @@ # ANY KIND, either express or implied. See the License for the specific # language governing permissions and limitations under the License. from tests import unittest -from copy import deepcopy import os -import re import copy import logging @@ -22,7 +20,6 @@ import requests from awscli.clidriver import create_clidriver -from botocore.payload import Payload class BaseAWSCommandParamsTest(unittest.TestCase): diff --git a/tests/unit/test_argprocess.py b/tests/unit/test_argprocess.py index 4df46486d14d..59fc8ac0db89 100644 --- a/tests/unit/test_argprocess.py +++ b/tests/unit/test_argprocess.py @@ -226,8 +226,7 @@ def test_improper_separator_for_filters_param(self): p = self.get_param_object('ec2.DescribeInstances.Filters') error_msg = "Error parsing parameter --filters.*should be" with self.assertRaisesRegexp(ParamError, error_msg): - returned = self.simplify( - p, ["Name:tag:Name,Values:foo"]) + self.simplify(p, ["Name:tag:Name,Values:foo"]) def test_unknown_key_for_filters_param(self): p = self.get_param_object('ec2.DescribeInstances.Filters') @@ -263,17 +262,13 @@ def test_csv_syntax_errors(self): p = self.get_param_object('cloudformation.CreateStack.Parameters') error_msg = "Error parsing parameter --parameters.*should be" with self.assertRaisesRegexp(ParamError, error_msg): - returned = self.simplify( - p, ['ParameterKey=key,ParameterValue="foo,bar']) + self.simplify(p, ['ParameterKey=key,ParameterValue="foo,bar']) with self.assertRaisesRegexp(ParamError, error_msg): - returned = self.simplify( - p, ['ParameterKey=key,ParameterValue=foo,bar"']) + self.simplify(p, ['ParameterKey=key,ParameterValue=foo,bar"']) with self.assertRaisesRegexp(ParamError, error_msg): - returned = self.simplify( - p, ['ParameterKey=key,ParameterValue=""foo,bar"']) + self.simplify(p, ['ParameterKey=key,ParameterValue=""foo,bar"']) with self.assertRaisesRegexp(ParamError, error_msg): - returned = self.simplify( - p, ['ParameterKey=key,ParameterValue="foo,bar\'']) + self.simplify(p, ['ParameterKey=key,ParameterValue="foo,bar\'']) class TestDocGen(BaseArgProcessTest): diff --git a/tests/unit/test_clidriver.py b/tests/unit/test_clidriver.py index e478a41b6e45..d97de2763f60 100644 --- a/tests/unit/test_clidriver.py +++ b/tests/unit/test_clidriver.py @@ -24,7 +24,6 @@ from botocore.hooks import HierarchicalEmitter from botocore.base import get_search_path from botocore.provider import Provider -import botocore.service GET_DATA = { @@ -356,7 +355,6 @@ def test_event_emission_for_top_level_params(self): self.assertEqual(args_seen[0].unknown_arg, 'foo') def test_empty_params_gracefully_handled(self): - driver = create_clidriver() # Simulates the equivalent in bash: --identifies "" cmd = 'ses get-identity-dkim-attributes --identities'.split() cmd.append('') diff --git a/tests/unit/test_plugin.py b/tests/unit/test_plugin.py index abfa0cf18fa1..80575b7a2065 100644 --- a/tests/unit/test_plugin.py +++ b/tests/unit/test_plugin.py @@ -69,7 +69,7 @@ def tearDown(self): del sys.modules['__fake_plugin__.__fake__'] def test_plugin_register(self): - emitter = plugin.load_plugins( + plugin.load_plugins( {'fake_plugin': '__fake_plugin__.__fake__.bar'}) self.assertTrue(self.fake_module.called)