Add support for --sse aws:kms and --sse-c

CHANGELOG.rst

@@ -7,6 +7,9 @@ Next Release (TBD)
 * bugfix:``aws opsworks register``: Support ``--no-verify-ssl``
   argument for the ``aws opsworks register`` command
   (`issue 1632 <https://github.com/aws/aws-cli/pull/1632>`__)
+* feature:``s3``: Add support for Server-Side Encryption with KMS
+  and Server-Side Encryption with Customer-Provided Keys.
+  (`issue 1623 <https://github.com/aws/aws-cli/pull/1623>`__)
 
 
 1.9.7

awscli/arguments.py

@@ -206,14 +206,16 @@ class CustomArgument(BaseCLIArgument):
     def __init__(self, name, help_text='', dest=None, default=None,
                  action=None, required=None, choices=None, nargs=None,
                  cli_type_name=None, group_name=None, positional_arg=False,
-                 no_paramfile=False, argument_model=None, synopsis=''):
+                 no_paramfile=False, argument_model=None, synopsis='',
+                 const=None):
         self._name = name
         self._help = help_text
         self._dest = dest
         self._default = default
         self._action = action
         self._required = required
         self._nargs = nargs
+        self._const = const
         self._cli_type_name = cli_type_name
         self._group_name = group_name
         self._positional_arg = positional_arg
@@ -275,6 +277,8 @@ def add_to_parser(self, parser):
             kwargs['required'] = self._required
         if self._nargs is not None:
             kwargs['nargs'] = self._nargs
+        if self._const is not None:
+            kwargs['const'] = self._const
         parser.add_argument(cli_name, **kwargs)
 
     @property

awscli/customizations/s3/filegenerator.py

@@ -116,14 +116,17 @@ class FileGenerator(object):
     ``FileInfo`` objects to send to a ``Comparator`` or ``S3Handler``.
     """
     def __init__(self, client, operation_name, follow_symlinks=True,
-                 page_size=None, result_queue=None):
+                 page_size=None, result_queue=None, request_parameters=None):
         self._client = client
         self.operation_name = operation_name
         self.follow_symlinks = follow_symlinks
         self.page_size = page_size
         self.result_queue = result_queue
         if not result_queue:
             self.result_queue = queue.Queue()
+        self.request_parameters = {}
+        if request_parameters is not None:
+            self.request_parameters = request_parameters
 
     def call(self, files):
         """
@@ -320,7 +323,9 @@ def _list_single_object(self, s3_path):
         # instead use a HeadObject request.
         bucket, key = find_bucket_key(s3_path)
         try:
-            response = self._client.head_object(Bucket=bucket, Key=key)
+            params = {'Bucket': bucket, 'Key': key}
+            params.update(self.request_parameters.get('HeadObject', {}))
+            response = self._client.head_object(**params)
         except ClientError as e:
             # We want to try to give a more helpful error message.
             # This is what the customer is going to see so we want to

awscli/customizations/s3/fileinfo.py

@@ -11,7 +11,8 @@
 
 from botocore.compat import quote
 from awscli.customizations.s3.utils import find_bucket_key, \
-    uni_print, guess_content_type, MD5Error, bytes_print, set_file_utime
+    uni_print, guess_content_type, MD5Error, bytes_print, set_file_utime, \
+    RequestParamsMapper
 
 
 LOGGER = logging.getLogger(__name__)
@@ -38,7 +39,6 @@ def save_file(filename, response_data, last_update, is_stream=False):
     """
     body = response_data['Body']
     etag = response_data['ETag'][1:-1]
-    sse = response_data.get('ServerSideEncryption', None)
     if not is_stream:
         d = os.path.dirname(filename)
         try:
@@ -59,7 +59,7 @@ def save_file(filename, response_data, last_update, is_stream=False):
         with open(filename, 'wb') as out_file:
             write_to_file(out_file, etag, md5, file_chunks)
 
-    if not _is_multipart_etag(etag) and sse != 'aws:kms':
+    if _can_validate_md5_with_etag(etag, response_data):
         if etag != md5.hexdigest():
             if not is_stream:
                 os.remove(filename)
@@ -75,6 +75,15 @@ def save_file(filename, response_data, last_update, is_stream=False):
         sys.stdout.flush()
 
 
+def _can_validate_md5_with_etag(etag, response_data):
+    sse = response_data.get('ServerSideEncryption', None)
+    sse_customer_algorithm = response_data.get('SSECustomerAlgorithm', None)
+    if not _is_multipart_etag(etag) and sse != 'aws:kms' and \
+            sse_customer_algorithm is None:
+        return True
+    return False
+
+
 def write_to_file(out_file, etag, md5, file_chunks, is_stream=False):
     """
     Updates the etag for each file chunk.  It will write to the file if it a
@@ -188,11 +197,9 @@ def __init__(self, src, dest=None, compare_key=None, size=None,
         self.size = size
         self.last_update = last_update
         # Usually inject ``parameters`` from ``BasicTask`` class.
+        self.parameters = {}
         if parameters is not None:
             self.parameters = parameters
-        else:
-            self.parameters = {'acl': None,
-                               'sse': None}
         self.source_client = source_client
         self.is_stream = is_stream
         self.associated_response_data = associated_response_data
@@ -204,60 +211,10 @@ def set_size_from_s3(self):
         bucket, key = find_bucket_key(self.src)
         params = {'Bucket': bucket,
                   'Key': key}
+        RequestParamsMapper.map_head_object_params(params, self.parameters)
         response_data = self.client.head_object(**params)
         self.size = int(response_data['ContentLength'])
 
-    def _permission_to_param(self, permission):
-        if permission == 'read':
-            return 'GrantRead'
-        if permission == 'full':
-            return 'GrantFullControl'
-        if permission == 'readacl':
-            return 'GrantReadACP'
-        if permission == 'writeacl':
-            return 'GrantWriteACP'
-        raise ValueError('permission must be one of: '
-                         'read|readacl|writeacl|full')
-
-    def _handle_object_params(self, params):
-        if self.parameters['acl']:
-            params['ACL'] = self.parameters['acl'][0]
-        if self.parameters['grants']:
-            for grant in self.parameters['grants']:
-                try:
-                    permission, grantee = grant.split('=', 1)
-                except ValueError:
-                    raise ValueError('grants should be of the form '
-                                     'permission=principal')
-                params[self._permission_to_param(permission)] = grantee
-        if self.parameters['sse']:
-            params['ServerSideEncryption'] = 'AES256'
-        if self.parameters['storage_class']:
-            params['StorageClass'] = self.parameters['storage_class'][0]
-        if self.parameters['website_redirect']:
-            params['WebsiteRedirectLocation'] = \
-                    self.parameters['website_redirect'][0]
-        if self.parameters['guess_mime_type']:
-            self._inject_content_type(params, self.src)
-        if self.parameters['content_type']:
-            params['ContentType'] = self.parameters['content_type'][0]
-        if self.parameters['cache_control']:
-            params['CacheControl'] = self.parameters['cache_control'][0]
-        if self.parameters['content_disposition']:
-            params['ContentDisposition'] = \
-                    self.parameters['content_disposition'][0]
-        if self.parameters['content_encoding']:
-            params['ContentEncoding'] = self.parameters['content_encoding'][0]
-        if self.parameters['content_language']:
-            params['ContentLanguage'] = self.parameters['content_language'][0]
-        if self.parameters['expires']:
-            params['Expires'] = self.parameters['expires'][0]
-
-    def _handle_metadata_directive(self, params):
-        if self.parameters['metadata_directive']:
-            params['MetadataDirective'] = \
-                self.parameters['metadata_directive'][0]
-
     def is_glacier_compatible(self):
         """Determines if a file info object is glacier compatible
 
@@ -301,10 +258,14 @@ def _handle_upload(self, body):
             'Key': key,
             'Body': body,
         }
-        self._handle_object_params(params)
+        self._inject_content_type(params)
+        RequestParamsMapper.map_put_object_params(params, self.parameters)
         response_data = self.client.put_object(**params)
 
-    def _inject_content_type(self, params, filename):
+    def _inject_content_type(self, params):
+        if not self.parameters['guess_mime_type']:
+            return
+        filename = self.src
         # Add a content type param if we can guess the type.
         try:
             guessed_type = guess_content_type(filename)
@@ -331,6 +292,7 @@ def download(self):
         """
         bucket, key = find_bucket_key(self.src)
         params = {'Bucket': bucket, 'Key': key}
+        RequestParamsMapper.map_get_object_params(params, self.parameters)
         response_data = self.client.get_object(**params)
         save_file(self.dest, response_data, self.last_update,
                   self.is_stream)
@@ -343,9 +305,9 @@ def copy(self):
         bucket, key = find_bucket_key(self.dest)
         params = {'Bucket': bucket,
                   'CopySource': copy_source, 'Key': key}
-        self._handle_object_params(params)
-        self._handle_metadata_directive(params)
-        self.client.copy_object(**params)
+        self._inject_content_type(params)
+        RequestParamsMapper.map_copy_object_params(params, self.parameters)
+        response_data = self.client.copy_object(**params)
 
     def delete(self):
         """
@@ -378,7 +340,9 @@ def move(self):
     def create_multipart_upload(self):
         bucket, key = find_bucket_key(self.dest)
         params = {'Bucket': bucket, 'Key': key}
-        self._handle_object_params(params)
+        self._inject_content_type(params)
+        RequestParamsMapper.map_create_multipart_upload_params(
+            params, self.parameters)
         response_data = self.client.create_multipart_upload(**params)
         upload_id = response_data['UploadId']
         return upload_id

awscli/customizations/s3/s3handler.py

@@ -55,16 +55,19 @@ def __init__(self, session, params, result_queue=None,
         self.result_queue = result_queue
         if not self.result_queue:
             self.result_queue = queue.Queue()
-        self.params = {'dryrun': False, 'quiet': False, 'acl': None,
-                       'guess_mime_type': True, 'sse': False,
-                       'storage_class': None, 'website_redirect': None,
-                       'content_type': None, 'cache_control': None,
-                       'content_disposition': None, 'content_encoding': None,
-                       'content_language': None, 'expires': None,
-                       'grants': None, 'only_show_errors': False,
-                       'is_stream': False, 'paths_type': None,
-                       'expected_size': None, 'metadata_directive': None,
-                       'ignore_glacier_warnings': False}
+        self.params = {
+            'dryrun': False, 'quiet': False, 'acl': None,
+            'guess_mime_type': True, 'sse_c_copy_source': None,
+            'sse_c_copy_source_key': None, 'sse': None,
+            'sse_c': None, 'sse_c_key': None, 'sse_kms_key_id': None,
+            'storage_class': None, 'website_redirect': None,
+            'content_type': None, 'cache_control': None,
+            'content_disposition': None, 'content_encoding': None,
+            'content_language': None, 'expires': None, 'grants': None,
+            'only_show_errors': False, 'is_stream': False,
+            'paths_type': None, 'expected_size': None,
+            'metadata_directive': None, 'ignore_glacier_warnings': False
+        }
         self.params['region'] = params['region']
         for key in self.params.keys():
             if key in params:
@@ -287,7 +290,8 @@ def _do_enqueue_range_download_tasks(self, filename, chunksize,
             task = tasks.DownloadPartTask(
                 part_number=i, chunk_size=chunksize,
                 result_queue=self.result_queue, filename=filename,
-                context=context, io_queue=self.write_queue)
+                context=context, io_queue=self.write_queue,
+                params=self.params)
             self.executor.submit(task)
 
     def _enqueue_multipart_upload_tasks(self, filename,
@@ -350,7 +354,8 @@ def _enqueue_upload_single_part_task(self, part_number, chunk_size,
                                          payload=None):
         kwargs = {'part_number': part_number, 'chunk_size': chunk_size,
                   'result_queue': self.result_queue,
-                  'upload_context': upload_context, 'filename': filename}
+                  'upload_context': upload_context, 'filename': filename,
+                  'params': self.params}
         if payload:
             kwargs['payload'] = payload
         task = task_class(**kwargs)