From e8209ed933b75d9e9f253cb6ff89256c3cd76458 Mon Sep 17 00:00:00 2001 From: John Boxall Date: Sat, 22 Mar 2014 15:32:43 -0700 Subject: [PATCH] Implements #473, adding a `--size-only` param to `aws s3 sync` --- awscli/customizations/s3/comparator.py | 11 +++++++- awscli/customizations/s3/s3.py | 7 +++-- .../unit/customizations/s3/test_comparator.py | 28 +++++++++++++++++++ 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/awscli/customizations/s3/comparator.py b/awscli/customizations/s3/comparator.py index d97eab8cee7c..555bd24bf2c4 100644 --- a/awscli/customizations/s3/comparator.py +++ b/awscli/customizations/s3/comparator.py @@ -34,6 +34,10 @@ def __init__(self, params=None): if 'delete' in params: self.delete = params['delete'] + self.compare_on_size_only = False + if 'size_only' in params: + self.compare_on_size_only = params['size_only'] + def call(self, src_files, dest_files): """ This function preforms the actual comparisons. The parameters it takes @@ -102,7 +106,12 @@ def call(self, src_files, dest_files): same_size = self.compare_size(src_file, dest_file) same_last_modified_time = self.compare_time(src_file, dest_file) - if (not same_size) or (not same_last_modified_time): + if self.compare_on_size_only: + should_sync = not same_size + else: + should_sync = (not same_size) or (not same_last_modified_time) + + if should_sync: LOG.debug("syncing: %s -> %s, size_changed: %s, " "last_modified_time_changed: %s", src_file.src, src_file.dest, diff --git a/awscli/customizations/s3/s3.py b/awscli/customizations/s3/s3.py index 2bef40e65b0b..882297c64a64 100644 --- a/awscli/customizations/s3/s3.py +++ b/awscli/customizations/s3/s3.py @@ -796,7 +796,7 @@ def add_verify_ssl(self, parsed_globals): 'sse', 'storage-class', 'content-type', 'cache-control', 'content-disposition', 'content-encoding', 'content-language', - 'expires']}, + 'expires', 'size-only']}, 'ls': {'options': {'nargs': '?', 'default': 's3://'}, 'params': ['recursive'], 'default': 's3://', 'command_class': ListCommand}, @@ -830,7 +830,7 @@ def add_verify_ssl(self, parsed_globals): 'dest': 'filters'}}, 'acl': {'options': {'nargs': 1, 'choices': ['private', 'public-read', - 'public-read-write', + 'public-read-write', 'authenticated-read', 'bucket-owner-read', 'bucket-owner-full-control', @@ -847,6 +847,9 @@ def add_verify_ssl(self, parsed_globals): 'content-encoding': {'options': {'nargs': 1}}, 'content-language': {'options': {'nargs': 1}}, 'expires': {'options': {'nargs': 1}}, + 'size-only': {'options': {'action': 'store_true'}, 'documents': + ('Makes the size of each key the only criteria used to ' + 'decide whether to sync from source to destination.')}, 'index-document': {'options': {}, 'documents': ('A suffix that is appended to a request that is for a ' 'directory on the website endpoint (e.g. if the suffix ' diff --git a/tests/unit/customizations/s3/test_comparator.py b/tests/unit/customizations/s3/test_comparator.py index b58a480f7118..7e9e515cf65b 100644 --- a/tests/unit/customizations/s3/test_comparator.py +++ b/tests/unit/customizations/s3/test_comparator.py @@ -300,5 +300,33 @@ def test_empty_src_dest(self): self.assertEqual(result_list, ref_list) +class ComparatorSizeOnlyTest(unittest.TestCase): + def setUp(self): + self.comparator = Comparator({'delete': True, 'size_only': True}) + + def test_compare_size_only(self): + """ + Confirm that files with the same size but different update times are not + synced when `size_only` is set. + """ + time_src = datetime.datetime.now() + time_dst = time_src + datetime.timedelta(days=1) + + src_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_src, src_type='local', + dest_type='s3', operation_name='upload', + service=None, endpoint=None) + + dst_file = FileInfo(src='', dest='', + compare_key='test.py', size=10, + last_update=time_dst, src_type='s3', + dest_type='local', operation_name='', + service=None, endpoint=None) + + files = self.comparator.call(iter([src_file]), iter([dst_file])) + self.assertEqual(sum(1 for _ in files), 0) + + if __name__ == "__main__": unittest.main()