Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add SSE-C and SSE-KMS support to s3 subcommands #1487

Closed
wants to merge 1 commit into from
+886 −44
Closed

Add SSE-C and SSE-KMS support to s3 subcommands #1487

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions TODO_sse
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# TODO migrate test_sse shell script "tests" to functional written in python
tests

# TODO test_sse tests different ways of passing customer key and customer key
md5 on the command line, but does not test the same for copy source customer
key/md5 (although these two use the same code path for arg processing)

# TODO update aws s3 {cp,sync} manpages with examples

# TODO maybe add debug logging for new options and option encrichment, etc.?

# TODO is there a better way to do the opt parsing within the awscli framework?

# TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
SSECustomerKeyMD5, and/or SSEKMSKeyId values present in response data after
certain APIs (head_object(), put_object(), get_object(), copy_object(),
create_multipart_upload(), upload_part(), upload_part_copy()) have expected
values given the request (and, not specific to SSE, there is no checking of
these now for ETag, etc.). Approximate source locations as of 2015-08-27
(these comments were removed afterwards, so source line numbers are slightly
different; look for use of client APIs above):

awscli/customizations/s3/filegenerator.py
317: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,

awscli/customizations/s3/fileinfo.py
196: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
292: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
311: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
329: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
372: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,

awscli/customizations/s3/tasks.py
180: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
261: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
401: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
535: # TODO validate that ServerSideEncryption and/or SSEKMSKeyId values

(complete_multipart_upload() has only ServerSideEncryption and/or
SSEKMSKeyId. The rest have all four.)

# TODO consider storing md5sum/sha256sum in object metadata (this is not
specific to SSE, but to anything that alters ETag like multipart) so it could
later be used (perhaps for sync)

14 changes: 12 additions & 2 deletions awscli/customizations/s3/filegenerator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,14 +115,19 @@ class FileGenerator(object):
``FileInfo`` objects to send to a ``Comparator`` or ``S3Handler``.
"""
def __init__(self, client, operation_name, follow_symlinks=True,
page_size=None, result_queue=None):
page_size=None, result_queue=None,
sse_customer_algorithm=None, sse_customer_key=None,
sse_customer_key_md5=None):
self._client = client
self.operation_name = operation_name
self.follow_symlinks = follow_symlinks
self.page_size = page_size
self.result_queue = result_queue
if not result_queue:
self.result_queue = queue.Queue()
self.sse_customer_algorithm = sse_customer_algorithm
self.sse_customer_key = sse_customer_key
self.sse_customer_key_md5 = sse_customer_key_md5

def call(self, files):
"""
Expand Down Expand Up @@ -303,7 +308,12 @@ def _list_single_object(self, s3_path):
# instead use a HeadObject request.
bucket, key = find_bucket_key(s3_path)
try:
response = self._client.head_object(Bucket=bucket, Key=key)
params = {'Bucket': bucket, 'Key': key}
if self.sse_customer_key:
params['SSECustomerAlgorithm'] = self.sse_customer_algorithm
params['SSECustomerKey'] = self.sse_customer_key
params['SSECustomerKeyMD5'] = self.sse_customer_key_md5
response = self._client.head_object(**params)
except ClientError as e:
# We want to try to give a more helpful error message.
# This is what the customer is going to see so we want to
Expand Down
38 changes: 34 additions & 4 deletions awscli/customizations/s3/fileinfo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def save_file(filename, response_data, last_update, is_stream=False):
body = response_data['Body']
etag = response_data['ETag'][1:-1]
sse = response_data.get('ServerSideEncryption', None)
sse_customer_algorithm = response_data.get('SSECustomerAlgorithm', None)
if not is_stream:
d = os.path.dirname(filename)
try:
Expand All @@ -55,7 +56,8 @@ def save_file(filename, response_data, last_update, is_stream=False):
with open(filename, 'wb') as out_file:
write_to_file(out_file, etag, md5, file_chunks)

if not _is_multipart_etag(etag) and sse != 'aws:kms':
if not _is_multipart_etag(etag) and sse != 'aws:kms' and \
sse_customer_algorithm is None:
if etag != md5.hexdigest():
if not is_stream:
os.remove(filename)
Expand Down Expand Up @@ -189,6 +191,7 @@ def set_size_from_s3(self):
bucket, key = find_bucket_key(self.src)
params = {'Bucket': bucket,
'Key': key}
self._handle_sse_params(params)
response_data = self.client.head_object(**params)
self.size = int(response_data['ContentLength'])

Expand All @@ -215,8 +218,6 @@ def _handle_object_params(self, params):
raise ValueError('grants should be of the form '
'permission=principal')
params[self._permission_to_param(permission)] = grantee
if self.parameters['sse']:
params['ServerSideEncryption'] = 'AES256'
if self.parameters['storage_class']:
params['StorageClass'] = self.parameters['storage_class'][0]
if self.parameters['website_redirect']:
Expand All @@ -238,6 +239,27 @@ def _handle_object_params(self, params):
if self.parameters['expires']:
params['Expires'] = self.parameters['expires'][0]

def _handle_sse_params(self, params):
if self.parameters['sse_copy_source_customer_key']:
params['CopySourceSSECustomerAlgorithm'] = \
self.parameters['sse_copy_source_customer_algorithm']
params['CopySourceSSECustomerKey'] = \
self.parameters['sse_copy_source_customer_key']
params['CopySourceSSECustomerKeyMD5'] = \
self.parameters['sse_copy_source_customer_key_md5']
if self.parameters['sse_class'] == 'C':
params['SSECustomerAlgorithm'] = \
self.parameters['sse_customer_algorithm']
params['SSECustomerKey'] = \
self.parameters['sse_customer_key']
params['SSECustomerKeyMD5'] = \
self.parameters['sse_customer_key_md5']
if self.parameters['sse_class'] == 'KMS':
params['ServerSideEncryption'] = 'aws:kms'
params['SSEKMSKeyId'] = self.parameters['sse_kms_key_id']
if self.parameters['sse_class'] == 'S3':
params['ServerSideEncryption'] = 'AES256'

def _handle_metadata_directive(self, params):
if self.parameters['metadata_directive']:
params['MetadataDirective'] = \
Expand All @@ -262,6 +284,7 @@ def _handle_upload(self, body):
'Body': body,
}
self._handle_object_params(params)
self._handle_sse_params(params)
response_data = self.client.put_object(**params)

def _inject_content_type(self, params, filename):
Expand All @@ -277,6 +300,7 @@ def download(self):
"""
bucket, key = find_bucket_key(self.src)
params = {'Bucket': bucket, 'Key': key}
self._handle_sse_params(params)
response_data = self.client.get_object(**params)
save_file(self.dest, response_data, self.last_update,
self.is_stream)
Expand All @@ -290,8 +314,9 @@ def copy(self):
params = {'Bucket': bucket,
'CopySource': copy_source, 'Key': key}
self._handle_object_params(params)
self._handle_sse_params(params)
self._handle_metadata_directive(params)
self.client.copy_object(**params)
response_data = self.client.copy_object(**params)

def delete(self):
"""
Expand Down Expand Up @@ -325,6 +350,11 @@ def create_multipart_upload(self):
bucket, key = find_bucket_key(self.dest)
params = {'Bucket': bucket, 'Key': key}
self._handle_object_params(params)
self._handle_sse_params(params)
params = params.copy()
params.pop('CopySourceSSECustomerAlgorithm', None)
params.pop('CopySourceSSECustomerKey', None)
params.pop('CopySourceSSECustomerKeyMD5', None)
response_data = self.client.create_multipart_upload(**params)
upload_id = response_data['UploadId']
return upload_id
42 changes: 31 additions & 11 deletions awscli/customizations/s3/s3handler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,33 @@ def __init__(self, session, params, result_queue=None,
self.result_queue = result_queue
if not self.result_queue:
self.result_queue = queue.Queue()
self.params = {'dryrun': False, 'quiet': False, 'acl': None,
'guess_mime_type': True, 'sse': False,
'storage_class': None, 'website_redirect': None,
'content_type': None, 'cache_control': None,
'content_disposition': None, 'content_encoding': None,
'content_language': None, 'expires': None,
'grants': None, 'only_show_errors': False,
'is_stream': False, 'paths_type': None,
'expected_size': None, 'metadata_directive': None}
self.params = {'dryrun': False,
'quiet': False,
'acl': None,
'guess_mime_type': True,
'sse_copy_source_customer_algorithm': None,
'sse_copy_source_customer_key': None,
'sse_copy_source_customer_key_md5': None,
'sse': False,
'sse_class': None,
'sse_customer_algorithm': None,
'sse_customer_key': None,
'sse_customer_key_md5': None,
'sse_kms_key_id': None,
'storage_class': None,
'website_redirect': None,
'content_type': None,
'cache_control': None,
'content_disposition': None,
'content_encoding': None,
'content_language': None,
'expires': None,
'grants': None,
'only_show_errors': False,
'is_stream': False,
'paths_type': None,
'expected_size': None,
'metadata_directive': None}
self.params['region'] = params['region']
for key in self.params.keys():
if key in params:
Expand Down Expand Up @@ -269,7 +287,8 @@ def _do_enqueue_range_download_tasks(self, filename, chunksize,
task = tasks.DownloadPartTask(
part_number=i, chunk_size=chunksize,
result_queue=self.result_queue, filename=filename,
context=context, io_queue=self.write_queue)
context=context, io_queue=self.write_queue,
params=self.params)
self.executor.submit(task)

def _enqueue_multipart_upload_tasks(self, filename,
Expand Down Expand Up @@ -332,7 +351,8 @@ def _enqueue_upload_single_part_task(self, part_number, chunk_size,
payload=None):
kwargs = {'part_number': part_number, 'chunk_size': chunk_size,
'result_queue': self.result_queue,
'upload_context': upload_context, 'filename': filename}
'upload_context': upload_context, 'filename': filename,
'params': self.params}
if payload:
kwargs['payload'] = payload
task = task_class(**kwargs)
Expand Down
Loading