From 777b0aa960745bf7ce09a101a1b59d2aa84c06e5 Mon Sep 17 00:00:00 2001 From: James Saryerwinnie Date: Thu, 1 Sep 2016 10:22:20 -0700 Subject: [PATCH] Explicitly translate supported config values to TransferConfig The previous implementation made the open-ended assumption that runtime config translates 1-1 to transfer configs. This also had the (unintended) side effect that everything in transfer config was implicitly supported in the ~/.aws/config file because the config is passed directly to `TransferConfig`. I've updated the code to have a fixed list of supported config values. This means that we need to explicitly update this list as we decide to support for transfer config options. --- .changes/next-release/bugfix-s3-56735.json | 5 +++++ awscli/customizations/s3/transferconfig.py | 11 ++++++----- tests/unit/customizations/s3/test_transferconfig.py | 8 +++++++- 3 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 .changes/next-release/bugfix-s3-56735.json diff --git a/.changes/next-release/bugfix-s3-56735.json b/.changes/next-release/bugfix-s3-56735.json new file mode 100644 index 000000000000..cc92c0aec7e6 --- /dev/null +++ b/.changes/next-release/bugfix-s3-56735.json @@ -0,0 +1,5 @@ +{ + "category": "``s3``", + "type": "bugfix", + "description": "Fix issue where setting ``addressing_style`` and ``use_accelerate_endpoint`` in your config file would cause ``aws s3`` commands using S3 streams to fail (`#2146 `__)" +} diff --git a/awscli/customizations/s3/transferconfig.py b/awscli/customizations/s3/transferconfig.py index bf720d86b220..575299308033 100644 --- a/awscli/customizations/s3/transferconfig.py +++ b/awscli/customizations/s3/transferconfig.py @@ -91,12 +91,13 @@ def create_transfer_config_from_runtime_config(runtime_config): """ translation_map = { 'max_concurrent_requests': 'max_request_concurrency', - 'max_queue_size': 'max_request_queue_size' + 'max_queue_size': 'max_request_queue_size', + 'multipart_threshold': 'multipart_threshold', + 'multipart_chunksize': 'multipart_chunksize', } kwargs = {} - for key, value in runtime_config.items(): - new_key = translation_map.get(key, key) - kwargs[new_key] = value - + if key not in translation_map: + continue + kwargs[translation_map[key]] = value return TransferConfig(**kwargs) diff --git a/tests/unit/customizations/s3/test_transferconfig.py b/tests/unit/customizations/s3/test_transferconfig.py index 1a8717898dfe..2684af3e6de0 100644 --- a/tests/unit/customizations/s3/test_transferconfig.py +++ b/tests/unit/customizations/s3/test_transferconfig.py @@ -77,7 +77,12 @@ def test_convert(self): 'multipart_threshold': 1, 'multipart_chunksize': 2, 'max_concurrent_requests': 3, - 'max_queue_size': 4 + 'max_queue_size': 4, + 'addressing_style': 'path', + 'use_accelerate_endpoint': True, + # This is a TransferConfig only option, it should + # just be ignored if it's in the ~/.aws/config for now. + 'max_in_memory_upload_chunks': 1000, } result = transferconfig.create_transfer_config_from_runtime_config( runtime_config) @@ -85,3 +90,4 @@ def test_convert(self): self.assertEqual(result.multipart_chunksize, 2) self.assertEqual(result.max_request_concurrency, 3) self.assertEqual(result.max_request_queue_size, 4) + self.assertNotEqual(result.max_in_memory_upload_chunks, 1000)