From efbecb77b59838406dc85790bf3599075302136b Mon Sep 17 00:00:00 2001 From: Chad Hinton Date: Mon, 19 Jan 2015 04:57:17 -0500 Subject: [PATCH 1/8] Add --humanize and --summarize options for s3 ls --- awscli/customizations/s3/subcommands.py | 30 ++++++++-- awscli/customizations/s3/utils.py | 16 ++++++ awscli/examples/s3/ls.rst | 21 +++++++ .../unit/customizations/s3/test_ls_command.py | 57 +++++++++++++++++++ .../customizations/s3/test_subcommands.py | 9 ++- 5 files changed, 125 insertions(+), 8 deletions(-) diff --git a/awscli/customizations/s3/subcommands.py b/awscli/customizations/s3/subcommands.py index 17a28438bfe6..a5c767a2d902 100644 --- a/awscli/customizations/s3/subcommands.py +++ b/awscli/customizations/s3/subcommands.py @@ -27,7 +27,7 @@ from awscli.customizations.s3.filters import create_filter from awscli.customizations.s3.s3handler import S3Handler, S3StreamHandler from awscli.customizations.s3.utils import find_bucket_key, uni_print, \ - AppendFilter, find_dest_path_comp_key + AppendFilter, find_dest_path_comp_key, humanize from awscli.customizations.s3.syncstrategy.base import MissingFileSync, \ SizeAndLastModifiedSync, NeverSync @@ -38,6 +38,12 @@ "Command is performed on all files or objects " "under the specified directory or prefix.")} +HUMANIZE = {'name': 'humanize', 'action': 'store_true', 'help_text': ( + "Displays file sizes in human readable format.")} + +SUMMARIZE = {'name': 'summarize', 'action': 'store_true', 'help_text': ( + "Displays summary information (number of objects, total size).")} + DRYRUN = {'name': 'dryrun', 'action': 'store_true', 'help_text': ( "Displays the operations that would be performed using the " @@ -242,13 +248,16 @@ class ListCommand(S3Command): USAGE = " or NONE" ARG_TABLE = [{'name': 'paths', 'nargs': '?', 'default': 's3://', 'positional_arg': True, 'synopsis': USAGE}, RECURSIVE, - PAGE_SIZE] + PAGE_SIZE, HUMANIZE, SUMMARIZE] EXAMPLES = BasicCommand.FROM_FILE('s3/ls.rst') def _run_main(self, parsed_args, parsed_globals): super(ListCommand, self)._run_main(parsed_args, parsed_globals) self._empty_result = False self._at_first_page = True + self._size_accumulator = 0 + self._total_objects = 0 + self._humanize = parsed_args.humanize path = parsed_args.paths if path.startswith('s3://'): path = path[5:] @@ -261,6 +270,8 @@ def _run_main(self, parsed_args, parsed_globals): parsed_args.page_size) else: self._list_all_objects(bucket, key, parsed_args.page_size) + if parsed_args.summarize: + self._print_summary() if key: # User specified a key to look for. We should return an rc of one # if there are no matching keys and/or prefixes or return an rc @@ -276,7 +287,6 @@ def _run_main(self, parsed_args, parsed_globals): return 0 def _list_all_objects(self, bucket, key, page_size=None): - operation = self.service.get_operation('ListObjects') iterator = operation.paginate(self.endpoint, bucket=bucket, prefix=key, delimiter='/', @@ -298,6 +308,8 @@ def _display_page(self, response_data, use_basename=True): uni_print(print_str) for content in contents: last_mod_str = self._make_last_mod_str(content['LastModified']) + self._size_accumulator += int(content['Size']) + self._total_objects += 1 size_str = self._make_size_str(content['Size']) if use_basename: filename_components = content['Key'].split('/') @@ -343,7 +355,7 @@ def _make_last_mod_str(self, last_mod): str(last_mod.day).zfill(2), str(last_mod.hour).zfill(2), str(last_mod.minute).zfill(2), - str(last_mod.second).zfill(2)) + str(last_mod.second).zfill(2)) last_mod_str = "%s-%s-%s %s:%s:%s" % last_mod_tup return last_mod_str.ljust(19, ' ') @@ -351,9 +363,17 @@ def _make_size_str(self, size): """ This function creates the size string when objects are being listed. """ - size_str = str(size) + size_str = humanize(size) if self._humanize else str(size) return size_str.rjust(10, ' ') + def _print_summary(self): + """ + This function prints a summary of total objects and total bytes + """ + print_str = str(self._total_objects) + uni_print("\nTotal Objects: ".rjust(15, ' ') + print_str + "\n") + print_str = humanize(self._size_accumulator) if self._humanize else str(self._size_accumulator) + uni_print("Total Size: ".rjust(15, ' ') + print_str + "\n") class WebsiteCommand(S3Command): NAME = 'website' diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py index 16e22bb935df..a6e5b5e4ae64 100644 --- a/awscli/customizations/s3/utils.py +++ b/awscli/customizations/s3/utils.py @@ -456,3 +456,19 @@ def __new__(cls, message, error=False, total_parts=None, warning=None): class IOCloseRequest(_IOCloseRequest): def __new__(cls, filename, desired_mtime=None): return super(IOCloseRequest, cls).__new__(cls, filename, desired_mtime) + + + +humanize_suffixes = ('kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB') + +def humanize(value): + format='%.1f' + base = 1000 + bytes = float(value) + + if bytes == 1: return '1 Byte' + elif bytes < base: return '%d Bytes' % bytes + + for i,sfx in enumerate(humanize_suffixes): + unit = base ** (i+2) + if bytes < unit: return (format + ' %s') % ((base * bytes / unit), sfx) diff --git a/awscli/examples/s3/ls.rst b/awscli/examples/s3/ls.rst index 1aa93b67dcbd..4e72d709b364 100644 --- a/awscli/examples/s3/ls.rst +++ b/awscli/examples/s3/ls.rst @@ -48,3 +48,24 @@ Output:: 2013-09-02 21:32:57 189 foo/bar/.baz/hooks/foo 2013-09-02 21:32:57 398 z.txt +The following ``ls`` command demonstrates the same command using the --humanize and --summarize options. --humanize +displays file size in Bytes/MB/KB/MB/GB/TB/PB/EB/ZB/YB. --summarize displays the total number of objects and total size +at the end of the result listing:: + + aws s3 ls s3://mybucket --recursive --humanize --summarize + +Output:: + + 2013-09-02 21:37:53 10 Bytes a.txt + 2013-09-02 21:37:53 2.9 MB foo.zip + 2013-09-02 21:32:57 23 Bytes foo/bar/.baz/a + 2013-09-02 21:32:58 41 Bytes foo/bar/.baz/b + 2013-09-02 21:32:57 281 Bytes foo/bar/.baz/c + 2013-09-02 21:32:57 73 Bytes foo/bar/.baz/d + 2013-09-02 21:32:57 452 Bytes foo/bar/.baz/e + 2013-09-02 21:32:57 896 Bytes foo/bar/.baz/hooks/bar + 2013-09-02 21:32:57 189 Bytes foo/bar/.baz/hooks/foo + 2013-09-02 21:32:57 398 Bytes z.txt + + Total Objects: 10 + Total Size: 2.9 MB diff --git a/tests/unit/customizations/s3/test_ls_command.py b/tests/unit/customizations/s3/test_ls_command.py index 1ea3f3844a17..40e891110f34 100644 --- a/tests/unit/customizations/s3/test_ls_command.py +++ b/tests/unit/customizations/s3/test_ls_command.py @@ -116,6 +116,63 @@ def test_fail_rc_no_objects_nor_prefixes(self): self.parsed_responses = [{}] self.run_cmd('s3 ls s3://bucket/foo', expected_rc=1) + def test_humanize_file_size(self): + time_utc = "2014-01-09T20:45:49.000Z" + self.parsed_responses = [{"CommonPrefixes": [], "Contents": [ + {"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}, + {"Key": "onekilobyte.txt", "Size": 1000, "LastModified": time_utc}, + {"Key": "onemegabyte.txt", "Size": 1000**2, "LastModified": time_utc}, + {"Key": "onegigabyte.txt", "Size": 1000**3, "LastModified": time_utc}, + {"Key": "oneterabyte.txt", "Size": 1000**4, "LastModified": time_utc}, + {"Key": "onepetabyte.txt", "Size": 1000**5, "LastModified": time_utc} ]}] + stdout, _, _ = self.run_cmd('s3 ls s3://bucket/ --humanize', expected_rc=0) + call_args = self.operations_called[0][1] + # Time is stored in UTC timezone, but the actual time displayed + # is specific to your tzinfo, so shift the timezone to your local's. + time_local = parser.parse(time_utc).astimezone(tz.tzlocal()) + time_fmt = time_local.strftime('%Y-%m-%d %H:%M:%S') + self.assertIn('%s 1 Byte onebyte.txt\n' % time_fmt, stdout) + self.assertIn('%s 1.0 kB onekilobyte.txt\n' % time_fmt, stdout) + self.assertIn('%s 1.0 MB onemegabyte.txt\n' % time_fmt, stdout) + self.assertIn('%s 1.0 GB onegigabyte.txt\n' % time_fmt, stdout) + self.assertIn('%s 1.0 TB oneterabyte.txt\n' % time_fmt, stdout) + self.assertIn('%s 1.0 PB onepetabyte.txt\n' % time_fmt, stdout) + + def test_summarize(self): + time_utc = "2014-01-09T20:45:49.000Z" + self.parsed_responses = [{"CommonPrefixes": [], "Contents": [ + {"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}, + {"Key": "onekilobyte.txt", "Size": 1000, "LastModified": time_utc}, + {"Key": "onemegabyte.txt", "Size": 1000**2, "LastModified": time_utc}, + {"Key": "onegigabyte.txt", "Size": 1000**3, "LastModified": time_utc}, + {"Key": "oneterabyte.txt", "Size": 1000**4, "LastModified": time_utc}, + {"Key": "onepetabyte.txt", "Size": 1000**5, "LastModified": time_utc} ]}] + stdout, _, _ = self.run_cmd('s3 ls s3://bucket/ --summarize', expected_rc=0) + call_args = self.operations_called[0][1] + # Time is stored in UTC timezone, but the actual time displayed + # is specific to your tzinfo, so shift the timezone to your local's. + time_local = parser.parse(time_utc).astimezone(tz.tzlocal()) + time_fmt = time_local.strftime('%Y-%m-%d %H:%M:%S') + self.assertIn('Total Objects: 6\n', stdout) + self.assertIn('Total Size: 1001001001001001\n', stdout) + + def test_summarize_with_humanize(self): + time_utc = "2014-01-09T20:45:49.000Z" + self.parsed_responses = [{"CommonPrefixes": [], "Contents": [ + {"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}, + {"Key": "onekilobyte.txt", "Size": 1000, "LastModified": time_utc}, + {"Key": "onemegabyte.txt", "Size": 1000**2, "LastModified": time_utc}, + {"Key": "onegigabyte.txt", "Size": 1000**3, "LastModified": time_utc}, + {"Key": "oneterabyte.txt", "Size": 1000**4, "LastModified": time_utc}, + {"Key": "onepetabyte.txt", "Size": 1000**5, "LastModified": time_utc} ]}] + stdout, _, _ = self.run_cmd('s3 ls s3://bucket/ --humanize --summarize', expected_rc=0) + call_args = self.operations_called[0][1] + # Time is stored in UTC timezone, but the actual time displayed + # is specific to your tzinfo, so shift the timezone to your local's. + time_local = parser.parse(time_utc).astimezone(tz.tzlocal()) + time_fmt = time_local.strftime('%Y-%m-%d %H:%M:%S') + self.assertIn('Total Objects: 6\n', stdout) + self.assertIn('Total Size: 1.0 PB\n', stdout) if __name__ == "__main__": unittest.main() diff --git a/tests/unit/customizations/s3/test_subcommands.py b/tests/unit/customizations/s3/test_subcommands.py index 3638d1c97141..6c1253f97baf 100644 --- a/tests/unit/customizations/s3/test_subcommands.py +++ b/tests/unit/customizations/s3/test_subcommands.py @@ -58,7 +58,8 @@ def setUp(self): def test_ls_command_for_bucket(self): ls_command = ListCommand(self.session) - parsed_args = FakeArgs(paths='s3://mybucket/', dir_op=False, page_size='5') + parsed_args = FakeArgs(paths='s3://mybucket/', dir_op=False, page_size='5', + humanize=False, summarize=False) parsed_globals = mock.Mock() ls_command._run_main(parsed_args, parsed_globals) call = self.session.get_service.return_value.get_operation\ @@ -78,7 +79,8 @@ def test_ls_command_for_bucket(self): def test_ls_command_with_no_args(self): ls_command = ListCommand(self.session) parsed_global = FakeArgs(region=None, endpoint_url=None, verify_ssl=None) - parsed_args = FakeArgs(dir_op=False, paths='s3://') + parsed_args = FakeArgs(dir_op=False, paths='s3://', humanize=False, + summarize=False) ls_command._run_main(parsed_args, parsed_global) # We should only be a single call. self.session.get_service.return_value.get_operation.assert_called_with( @@ -98,7 +100,8 @@ def test_ls_with_verify_argument(self): ls_command = ListCommand(self.session) parsed_global = FakeArgs(region='us-west-2', endpoint_url=None, verify_ssl=False) - parsed_args = FakeArgs(paths='s3://', dir_op=False) + parsed_args = FakeArgs(paths='s3://', dir_op=False, humanize=False, + summarize=False) ls_command._run_main(parsed_args, parsed_global) # Verify get_endpoint get_endpoint = self.session.get_service.return_value.get_endpoint From 8e08d8a4db757002c2ff00c2ba44da4221e68777 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Mon, 26 Jan 2015 11:57:51 -0800 Subject: [PATCH 2/8] Rename --humanize to --human-readable To be consistent with ls, rsync, du, and other command line utilities. --- awscli/customizations/s3/subcommands.py | 23 ++++++----- awscli/customizations/s3/utils.py | 41 +++++++++++-------- .../unit/customizations/s3/test_ls_command.py | 9 ++-- .../customizations/s3/test_subcommands.py | 10 ++--- 4 files changed, 47 insertions(+), 36 deletions(-) diff --git a/awscli/customizations/s3/subcommands.py b/awscli/customizations/s3/subcommands.py index a5c767a2d902..4e276d633ecc 100644 --- a/awscli/customizations/s3/subcommands.py +++ b/awscli/customizations/s3/subcommands.py @@ -27,7 +27,7 @@ from awscli.customizations.s3.filters import create_filter from awscli.customizations.s3.s3handler import S3Handler, S3StreamHandler from awscli.customizations.s3.utils import find_bucket_key, uni_print, \ - AppendFilter, find_dest_path_comp_key, humanize + AppendFilter, find_dest_path_comp_key, human_readable_size from awscli.customizations.s3.syncstrategy.base import MissingFileSync, \ SizeAndLastModifiedSync, NeverSync @@ -38,8 +38,8 @@ "Command is performed on all files or objects " "under the specified directory or prefix.")} -HUMANIZE = {'name': 'humanize', 'action': 'store_true', 'help_text': ( - "Displays file sizes in human readable format.")} +HUMAN_READABLE = {'name': 'human-readable', 'action': 'store_true', + 'help_text': "Displays file sizes in human readable format."} SUMMARIZE = {'name': 'summarize', 'action': 'store_true', 'help_text': ( "Displays summary information (number of objects, total size).")} @@ -248,7 +248,7 @@ class ListCommand(S3Command): USAGE = " or NONE" ARG_TABLE = [{'name': 'paths', 'nargs': '?', 'default': 's3://', 'positional_arg': True, 'synopsis': USAGE}, RECURSIVE, - PAGE_SIZE, HUMANIZE, SUMMARIZE] + PAGE_SIZE, HUMAN_READABLE, SUMMARIZE] EXAMPLES = BasicCommand.FROM_FILE('s3/ls.rst') def _run_main(self, parsed_args, parsed_globals): @@ -257,7 +257,7 @@ def _run_main(self, parsed_args, parsed_globals): self._at_first_page = True self._size_accumulator = 0 self._total_objects = 0 - self._humanize = parsed_args.humanize + self._human_readable = parsed_args.human_readable path = parsed_args.paths if path.startswith('s3://'): path = path[5:] @@ -273,7 +273,7 @@ def _run_main(self, parsed_args, parsed_globals): if parsed_args.summarize: self._print_summary() if key: - # User specified a key to look for. We should return an rc of one + # User specified a key to look for. We should return an rc of one # if there are no matching keys and/or prefixes or return an rc # of zero if there are matching keys or prefixes. return self._check_no_objects() @@ -363,7 +363,7 @@ def _make_size_str(self, size): """ This function creates the size string when objects are being listed. """ - size_str = humanize(size) if self._humanize else str(size) + size_str = human_readable_size(size) if self._human_readable else str(size) return size_str.rjust(10, ' ') def _print_summary(self): @@ -372,9 +372,10 @@ def _print_summary(self): """ print_str = str(self._total_objects) uni_print("\nTotal Objects: ".rjust(15, ' ') + print_str + "\n") - print_str = humanize(self._size_accumulator) if self._humanize else str(self._size_accumulator) + print_str = human_readable_size(self._size_accumulator) if self._human_readable else str(self._size_accumulator) uni_print("Total Size: ".rjust(15, ' ') + print_str + "\n") + class WebsiteCommand(S3Command): NAME = 'website' DESCRIPTION = 'Set the website configuration for a bucket.' @@ -573,7 +574,7 @@ def needs_filegenerator(self): return False else: return True - + def choose_sync_strategies(self): """Determines the sync strategy for the command. @@ -668,7 +669,7 @@ def run(self): endpoint=self._endpoint, is_stream=True)] file_info_builder = FileInfoBuilder(self._service, self._endpoint, - self._source_endpoint, self.parameters) + self._source_endpoint, self.parameters) s3handler = S3Handler(self.session, self.parameters, result_queue=result_queue) s3_stream_handler = S3StreamHandler(self.session, self.parameters, @@ -732,7 +733,7 @@ def run(self): # tasks failed and the number of tasks warned. # This means that files[0] now contains a namedtuple with # the number of failed tasks and the number of warned tasks. - # In terms of the RC, we're keeping it simple and saying + # In terms of the RC, we're keeping it simple and saying # that > 0 failed tasks will give a 1 RC and > 0 warned # tasks will give a 2 RC. Otherwise a RC of zero is returned. rc = 0 diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py index a6e5b5e4ae64..8900ec39e824 100644 --- a/awscli/customizations/s3/utils.py +++ b/awscli/customizations/s3/utils.py @@ -31,6 +31,31 @@ from awscli.compat import queue +humanize_suffixes = ('kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB') + + +def human_readable_size(value): + """Convert an size in bytes into a human readable format. + + For example: + + :param value: The size in bytes + :return: The size in a human readable format + """ + + format = '%.1f' + base = 1000 + bytes = float(value) + + if bytes == 1: return '1 Byte' + elif bytes < base: return '%d Bytes' % bytes + + for i,sfx in enumerate(humanize_suffixes): + unit = base ** (i+2) + if bytes < unit: + return (format + ' %s') % ((base * bytes / unit), sfx) + + class AppendFilter(argparse.Action): """ This class is used as an action when parsing the parameters. @@ -456,19 +481,3 @@ def __new__(cls, message, error=False, total_parts=None, warning=None): class IOCloseRequest(_IOCloseRequest): def __new__(cls, filename, desired_mtime=None): return super(IOCloseRequest, cls).__new__(cls, filename, desired_mtime) - - - -humanize_suffixes = ('kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB') - -def humanize(value): - format='%.1f' - base = 1000 - bytes = float(value) - - if bytes == 1: return '1 Byte' - elif bytes < base: return '%d Bytes' % bytes - - for i,sfx in enumerate(humanize_suffixes): - unit = base ** (i+2) - if bytes < unit: return (format + ' %s') % ((base * bytes / unit), sfx) diff --git a/tests/unit/customizations/s3/test_ls_command.py b/tests/unit/customizations/s3/test_ls_command.py index 40e891110f34..afc987cbf530 100644 --- a/tests/unit/customizations/s3/test_ls_command.py +++ b/tests/unit/customizations/s3/test_ls_command.py @@ -116,7 +116,7 @@ def test_fail_rc_no_objects_nor_prefixes(self): self.parsed_responses = [{}] self.run_cmd('s3 ls s3://bucket/foo', expected_rc=1) - def test_humanize_file_size(self): + def test_human_readable_file_size(self): time_utc = "2014-01-09T20:45:49.000Z" self.parsed_responses = [{"CommonPrefixes": [], "Contents": [ {"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}, @@ -125,7 +125,8 @@ def test_humanize_file_size(self): {"Key": "onegigabyte.txt", "Size": 1000**3, "LastModified": time_utc}, {"Key": "oneterabyte.txt", "Size": 1000**4, "LastModified": time_utc}, {"Key": "onepetabyte.txt", "Size": 1000**5, "LastModified": time_utc} ]}] - stdout, _, _ = self.run_cmd('s3 ls s3://bucket/ --humanize', expected_rc=0) + stdout, _, _ = self.run_cmd('s3 ls s3://bucket/ --human-readable', + expected_rc=0) call_args = self.operations_called[0][1] # Time is stored in UTC timezone, but the actual time displayed # is specific to your tzinfo, so shift the timezone to your local's. @@ -156,7 +157,7 @@ def test_summarize(self): self.assertIn('Total Objects: 6\n', stdout) self.assertIn('Total Size: 1001001001001001\n', stdout) - def test_summarize_with_humanize(self): + def test_summarize_with_human_readable(self): time_utc = "2014-01-09T20:45:49.000Z" self.parsed_responses = [{"CommonPrefixes": [], "Contents": [ {"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}, @@ -165,7 +166,7 @@ def test_summarize_with_humanize(self): {"Key": "onegigabyte.txt", "Size": 1000**3, "LastModified": time_utc}, {"Key": "oneterabyte.txt", "Size": 1000**4, "LastModified": time_utc}, {"Key": "onepetabyte.txt", "Size": 1000**5, "LastModified": time_utc} ]}] - stdout, _, _ = self.run_cmd('s3 ls s3://bucket/ --humanize --summarize', expected_rc=0) + stdout, _, _ = self.run_cmd('s3 ls s3://bucket/ --human-readable --summarize', expected_rc=0) call_args = self.operations_called[0][1] # Time is stored in UTC timezone, but the actual time displayed # is specific to your tzinfo, so shift the timezone to your local's. diff --git a/tests/unit/customizations/s3/test_subcommands.py b/tests/unit/customizations/s3/test_subcommands.py index 6c1253f97baf..f99ab2504e07 100644 --- a/tests/unit/customizations/s3/test_subcommands.py +++ b/tests/unit/customizations/s3/test_subcommands.py @@ -59,7 +59,7 @@ def setUp(self): def test_ls_command_for_bucket(self): ls_command = ListCommand(self.session) parsed_args = FakeArgs(paths='s3://mybucket/', dir_op=False, page_size='5', - humanize=False, summarize=False) + human_readable=False, summarize=False) parsed_globals = mock.Mock() ls_command._run_main(parsed_args, parsed_globals) call = self.session.get_service.return_value.get_operation\ @@ -79,8 +79,8 @@ def test_ls_command_for_bucket(self): def test_ls_command_with_no_args(self): ls_command = ListCommand(self.session) parsed_global = FakeArgs(region=None, endpoint_url=None, verify_ssl=None) - parsed_args = FakeArgs(dir_op=False, paths='s3://', humanize=False, - summarize=False) + parsed_args = FakeArgs(dir_op=False, paths='s3://', + human_readable=False, summarize=False) ls_command._run_main(parsed_args, parsed_global) # We should only be a single call. self.session.get_service.return_value.get_operation.assert_called_with( @@ -100,8 +100,8 @@ def test_ls_with_verify_argument(self): ls_command = ListCommand(self.session) parsed_global = FakeArgs(region='us-west-2', endpoint_url=None, verify_ssl=False) - parsed_args = FakeArgs(paths='s3://', dir_op=False, humanize=False, - summarize=False) + parsed_args = FakeArgs(paths='s3://', dir_op=False, + human_readable=False, summarize=False) ls_command._run_main(parsed_args, parsed_global) # Verify get_endpoint get_endpoint = self.session.get_service.return_value.get_endpoint From c9d291ffc2a37e6480e2887cf658733362eecbe5 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Mon, 26 Jan 2015 12:49:43 -0800 Subject: [PATCH 3/8] Switch to base-2 units This uses the IEC binary prefixes for file sizes so 1024**2 == 1 MiB. --- awscli/customizations/s3/utils.py | 35 ++++++++++----- .../unit/customizations/s3/test_ls_command.py | 45 ++++++++++--------- tests/unit/customizations/s3/test_utils.py | 25 ++++++++++- 3 files changed, 70 insertions(+), 35 deletions(-) diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py index 8900ec39e824..a47dc2df2b1f 100644 --- a/awscli/customizations/s3/utils.py +++ b/awscli/customizations/s3/utils.py @@ -31,29 +31,40 @@ from awscli.compat import queue -humanize_suffixes = ('kB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB') +humanize_suffixes = ('KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB') def human_readable_size(value): """Convert an size in bytes into a human readable format. - For example: + For example:: + + >>> human_readable_size(1) + '1 Byte' + >>> human_readable_size(10) + '10 Bytes' + >>> human_readable_size(1024) + '1.0 KiB' + >>> human_readable_size(1024 * 1024) + '1.0 MiB' :param value: The size in bytes - :return: The size in a human readable format - """ + :return: The size in a human readable format based on base-2 units. - format = '%.1f' - base = 1000 - bytes = float(value) + """ + one_decimal_point = '%.1f' + base = 1024 + bytes_int = float(value) - if bytes == 1: return '1 Byte' - elif bytes < base: return '%d Bytes' % bytes + if bytes_int == 1: + return '1 Byte' + elif bytes_int < base: + return '%d Bytes' % bytes_int - for i,sfx in enumerate(humanize_suffixes): + for i, suffix in enumerate(humanize_suffixes): unit = base ** (i+2) - if bytes < unit: - return (format + ' %s') % ((base * bytes / unit), sfx) + if round((bytes_int / unit) * base) < base: + return '%.1f %s' % ((base * bytes_int / unit), suffix) class AppendFilter(argparse.Action): diff --git a/tests/unit/customizations/s3/test_ls_command.py b/tests/unit/customizations/s3/test_ls_command.py index afc987cbf530..d363c4aabc10 100644 --- a/tests/unit/customizations/s3/test_ls_command.py +++ b/tests/unit/customizations/s3/test_ls_command.py @@ -120,11 +120,11 @@ def test_human_readable_file_size(self): time_utc = "2014-01-09T20:45:49.000Z" self.parsed_responses = [{"CommonPrefixes": [], "Contents": [ {"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}, - {"Key": "onekilobyte.txt", "Size": 1000, "LastModified": time_utc}, - {"Key": "onemegabyte.txt", "Size": 1000**2, "LastModified": time_utc}, - {"Key": "onegigabyte.txt", "Size": 1000**3, "LastModified": time_utc}, - {"Key": "oneterabyte.txt", "Size": 1000**4, "LastModified": time_utc}, - {"Key": "onepetabyte.txt", "Size": 1000**5, "LastModified": time_utc} ]}] + {"Key": "onekilobyte.txt", "Size": 1024, "LastModified": time_utc}, + {"Key": "onemegabyte.txt", "Size": 1024 ** 2, "LastModified": time_utc}, + {"Key": "onegigabyte.txt", "Size": 1024 ** 3, "LastModified": time_utc}, + {"Key": "oneterabyte.txt", "Size": 1024 ** 4, "LastModified": time_utc}, + {"Key": "onepetabyte.txt", "Size": 1024 ** 5, "LastModified": time_utc} ]}] stdout, _, _ = self.run_cmd('s3 ls s3://bucket/ --human-readable', expected_rc=0) call_args = self.operations_called[0][1] @@ -133,21 +133,21 @@ def test_human_readable_file_size(self): time_local = parser.parse(time_utc).astimezone(tz.tzlocal()) time_fmt = time_local.strftime('%Y-%m-%d %H:%M:%S') self.assertIn('%s 1 Byte onebyte.txt\n' % time_fmt, stdout) - self.assertIn('%s 1.0 kB onekilobyte.txt\n' % time_fmt, stdout) - self.assertIn('%s 1.0 MB onemegabyte.txt\n' % time_fmt, stdout) - self.assertIn('%s 1.0 GB onegigabyte.txt\n' % time_fmt, stdout) - self.assertIn('%s 1.0 TB oneterabyte.txt\n' % time_fmt, stdout) - self.assertIn('%s 1.0 PB onepetabyte.txt\n' % time_fmt, stdout) + self.assertIn('%s 1.0 KiB onekilobyte.txt\n' % time_fmt, stdout) + self.assertIn('%s 1.0 MiB onemegabyte.txt\n' % time_fmt, stdout) + self.assertIn('%s 1.0 GiB onegigabyte.txt\n' % time_fmt, stdout) + self.assertIn('%s 1.0 TiB oneterabyte.txt\n' % time_fmt, stdout) + self.assertIn('%s 1.0 PiB onepetabyte.txt\n' % time_fmt, stdout) def test_summarize(self): time_utc = "2014-01-09T20:45:49.000Z" self.parsed_responses = [{"CommonPrefixes": [], "Contents": [ {"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}, - {"Key": "onekilobyte.txt", "Size": 1000, "LastModified": time_utc}, - {"Key": "onemegabyte.txt", "Size": 1000**2, "LastModified": time_utc}, - {"Key": "onegigabyte.txt", "Size": 1000**3, "LastModified": time_utc}, - {"Key": "oneterabyte.txt", "Size": 1000**4, "LastModified": time_utc}, - {"Key": "onepetabyte.txt", "Size": 1000**5, "LastModified": time_utc} ]}] + {"Key": "onekilobyte.txt", "Size": 1024, "LastModified": time_utc}, + {"Key": "onemegabyte.txt", "Size": 1024 ** 2, "LastModified": time_utc}, + {"Key": "onegigabyte.txt", "Size": 1024 ** 3, "LastModified": time_utc}, + {"Key": "oneterabyte.txt", "Size": 1024 ** 4, "LastModified": time_utc}, + {"Key": "onepetabyte.txt", "Size": 1024 ** 5, "LastModified": time_utc} ]}] stdout, _, _ = self.run_cmd('s3 ls s3://bucket/ --summarize', expected_rc=0) call_args = self.operations_called[0][1] # Time is stored in UTC timezone, but the actual time displayed @@ -155,17 +155,17 @@ def test_summarize(self): time_local = parser.parse(time_utc).astimezone(tz.tzlocal()) time_fmt = time_local.strftime('%Y-%m-%d %H:%M:%S') self.assertIn('Total Objects: 6\n', stdout) - self.assertIn('Total Size: 1001001001001001\n', stdout) + self.assertIn('Total Size: 1127000493261825\n', stdout) def test_summarize_with_human_readable(self): time_utc = "2014-01-09T20:45:49.000Z" self.parsed_responses = [{"CommonPrefixes": [], "Contents": [ {"Key": "onebyte.txt", "Size": 1, "LastModified": time_utc}, - {"Key": "onekilobyte.txt", "Size": 1000, "LastModified": time_utc}, - {"Key": "onemegabyte.txt", "Size": 1000**2, "LastModified": time_utc}, - {"Key": "onegigabyte.txt", "Size": 1000**3, "LastModified": time_utc}, - {"Key": "oneterabyte.txt", "Size": 1000**4, "LastModified": time_utc}, - {"Key": "onepetabyte.txt", "Size": 1000**5, "LastModified": time_utc} ]}] + {"Key": "onekilobyte.txt", "Size": 1024, "LastModified": time_utc}, + {"Key": "onemegabyte.txt", "Size": 1024 ** 2, "LastModified": time_utc}, + {"Key": "onegigabyte.txt", "Size": 1024 ** 3, "LastModified": time_utc}, + {"Key": "oneterabyte.txt", "Size": 1024 ** 4, "LastModified": time_utc}, + {"Key": "onepetabyte.txt", "Size": 1024 ** 5, "LastModified": time_utc} ]}] stdout, _, _ = self.run_cmd('s3 ls s3://bucket/ --human-readable --summarize', expected_rc=0) call_args = self.operations_called[0][1] # Time is stored in UTC timezone, but the actual time displayed @@ -173,7 +173,8 @@ def test_summarize_with_human_readable(self): time_local = parser.parse(time_utc).astimezone(tz.tzlocal()) time_fmt = time_local.strftime('%Y-%m-%d %H:%M:%S') self.assertIn('Total Objects: 6\n', stdout) - self.assertIn('Total Size: 1.0 PB\n', stdout) + self.assertIn('Total Size: 1.0 PiB\n', stdout) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/customizations/s3/test_utils.py b/tests/unit/customizations/s3/test_utils.py index 01c8d1f069c2..2a2355f252da 100644 --- a/tests/unit/customizations/s3/test_utils.py +++ b/tests/unit/customizations/s3/test_utils.py @@ -9,6 +9,7 @@ import mock from dateutil.tz import tzlocal +from nose.tools import assert_equal from botocore.hooks import HierarchicalEmitter from awscli.customizations.s3.utils import find_bucket_key, find_chunksize @@ -19,10 +20,32 @@ from awscli.customizations.s3.utils import ScopedEventHandler from awscli.customizations.s3.utils import get_file_stat from awscli.customizations.s3.utils import AppendFilter -from awscli.customizations.s3.utils import create_warning +from awscli.customizations.s3.utils import create_warning +from awscli.customizations.s3.utils import human_readable_size from awscli.customizations.s3.constants import MAX_SINGLE_UPLOAD_SIZE +def test_human_readable_size(): + yield _test_human_size_matches, 1, '1 Byte' + yield _test_human_size_matches, 10, '10 Bytes' + yield _test_human_size_matches, 1000, '1000 Bytes' + yield _test_human_size_matches, 1024, '1.0 KiB' + yield _test_human_size_matches, 1024 ** 2, '1.0 MiB' + yield _test_human_size_matches, 1024 ** 2, '1.0 MiB' + yield _test_human_size_matches, 1024 ** 3, '1.0 GiB' + yield _test_human_size_matches, 1024 ** 4, '1.0 TiB' + yield _test_human_size_matches, 1024 ** 5, '1.0 PiB' + yield _test_human_size_matches, 1024 ** 6, '1.0 EiB' + + # Round to the nearest block. + yield _test_human_size_matches, 1024 ** 2 - 1, '1.0 MiB' + yield _test_human_size_matches, 1024 ** 3 - 1, '1.0 GiB' + + +def _test_human_size_matches(bytes_int, expected): + assert_equal(human_readable_size(bytes_int), expected) + + class AppendFilterTest(unittest.TestCase): def test_call(self): parser = argparse.ArgumentParser() From 81920d35a3973f38dc3fd22868f3fbc01a8b3b04 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Mon, 26 Jan 2015 12:57:07 -0800 Subject: [PATCH 4/8] Update doc example --- awscli/examples/s3/ls.rst | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/awscli/examples/s3/ls.rst b/awscli/examples/s3/ls.rst index 4e72d709b364..65f61f1a2232 100644 --- a/awscli/examples/s3/ls.rst +++ b/awscli/examples/s3/ls.rst @@ -48,16 +48,17 @@ Output:: 2013-09-02 21:32:57 189 foo/bar/.baz/hooks/foo 2013-09-02 21:32:57 398 z.txt -The following ``ls`` command demonstrates the same command using the --humanize and --summarize options. --humanize -displays file size in Bytes/MB/KB/MB/GB/TB/PB/EB/ZB/YB. --summarize displays the total number of objects and total size -at the end of the result listing:: +The following ``ls`` command demonstrates the same command using the --humanize +and --summarize options. --humanize displays file size in +Bytes/MiB/KiB/GiB/TiB/PiB/EiB. --summarize displays the total number of objects +and total size at the end of the result listing:: aws s3 ls s3://mybucket --recursive --humanize --summarize Output:: 2013-09-02 21:37:53 10 Bytes a.txt - 2013-09-02 21:37:53 2.9 MB foo.zip + 2013-09-02 21:37:53 2.9 MiB foo.zip 2013-09-02 21:32:57 23 Bytes foo/bar/.baz/a 2013-09-02 21:32:58 41 Bytes foo/bar/.baz/b 2013-09-02 21:32:57 281 Bytes foo/bar/.baz/c @@ -68,4 +69,4 @@ Output:: 2013-09-02 21:32:57 398 Bytes z.txt Total Objects: 10 - Total Size: 2.9 MB + Total Size: 2.9 MiB From 12966ea268f088f1bf07e8e9bf0b9a67ef39e8ee Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Mon, 26 Jan 2015 13:07:18 -0800 Subject: [PATCH 5/8] Update changelog with new feature --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ca7da716b446..d88bb31208df 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,13 @@ CHANGELOG ========= +Next Release (TBD) +================== + +* feature:``aws s3 ls``: Add ``--human-readable`` and ``--summarize`` options + (`issue 1103 `__) + + 1.7.3 ===== From 6cb01e603272cfd327874630c8432246c7a13270 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Mon, 26 Jan 2015 15:44:30 -0800 Subject: [PATCH 6/8] Update CHANGELOG with kinesis fix --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d88bb31208df..da653e4c1d51 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,6 +7,9 @@ Next Release (TBD) * feature:``aws s3 ls``: Add ``--human-readable`` and ``--summarize`` options (`issue 1103 `__) +* bugfix:``aws kinesis put-records``: Fix issue with base64 encoding for + blob types + (`botocore issue 413 `__) 1.7.3 From a113b960d9d4689fd894cc9d29766c62e23c2d02 Mon Sep 17 00:00:00 2001 From: kyleknap Date: Mon, 26 Jan 2015 19:21:08 -0800 Subject: [PATCH 7/8] Update changelog with new service updates --- CHANGELOG.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index da653e4c1d51..33ca53dd9501 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,9 @@ CHANGELOG Next Release (TBD) ================== +* feature:``aws dynamodb``: Add support for online indexing. +* feature:``aws importexport get-shipping-label``: Add support for + ``get-shipping-label``. * feature:``aws s3 ls``: Add ``--human-readable`` and ``--summarize`` options (`issue 1103 `__) * bugfix:``aws kinesis put-records``: Fix issue with base64 encoding for From 2754f2493b8e94b3f398392196b102b91b06a4b6 Mon Sep 17 00:00:00 2001 From: AWS Date: Mon, 26 Jan 2015 19:45:26 -0800 Subject: [PATCH 8/8] Bumping version to 1.7.4 --- CHANGELOG.rst | 4 ++-- awscli/__init__.py | 2 +- doc/source/conf.py | 2 +- setup.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 33ca53dd9501..50ea34c0db2e 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,8 +2,8 @@ CHANGELOG ========= -Next Release (TBD) -================== +1.7.4 +===== * feature:``aws dynamodb``: Add support for online indexing. * feature:``aws importexport get-shipping-label``: Add support for diff --git a/awscli/__init__.py b/awscli/__init__.py index 654c6c6ccb28..096ff0cd6470 100644 --- a/awscli/__init__.py +++ b/awscli/__init__.py @@ -17,7 +17,7 @@ """ import os -__version__ = '1.7.3' +__version__ = '1.7.4' # # Get our data path to be added to botocore's search path diff --git a/doc/source/conf.py b/doc/source/conf.py index 642ec1b1a143..b2518c1a5664 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -52,7 +52,7 @@ # The short X.Y version. version = '1.7' # The full version, including alpha/beta/rc tags. -release = '1.7.3' +release = '1.7.4' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/setup.py b/setup.py index 753a2a689d12..105591079ad4 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import awscli -requires = ['botocore>=0.84.0,<0.85.0', +requires = ['botocore>=0.85.0,<0.86.0', 'bcdoc>=0.12.0,<0.13.0', 'colorama==0.2.5', 'docutils>=0.10',