Skip to content

Commit

Permalink
Add SSE-C and SSE-KMS support to s3 subcommands
Browse files Browse the repository at this point in the history 
-- add SSE-C and SSE-KMS support to the aws s3 subcommands (SSE-{C,KMS}
are two newer forms of S3 Server Side Encryption)

-- the added SSE-C and SSE-KMS support coexists with existing SSE-S3
support without breaking the current meaning of the --sse command line
option

-- tested with (src, dst) of locals3, s3local, and s3s3, including copy
from objects written as {S3,SSE-C,SSE-KMS,SSE-S3} to
{S3,SSE-C,SSE-KMS,SSE-S3} both with the same and with different keys
(for SSE-C and SSE-KMS)

-- add a shell script to test different permutations of SSE (integration
into the Python functional tests is a TODO) and s3local, locals3, s3s3
src and dst with aws s3 cp and aws s3 sync
  • Loading branch information
@0xkag
0xkag committed Sep 3, 2015
1 parent a3d052d commit c2b3849
Show file tree
Hide file tree
Showing 10 changed files with 886 additions and 44 deletions.
45 changes: 45 additions & 0 deletions TODO_sse
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# TODO migrate test_sse shell script "tests" to functional written in python
tests

# TODO test_sse tests different ways of passing customer key and customer key
md5 on the command line, but does not test the same for copy source customer
key/md5 (although these two use the same code path for arg processing)

# TODO update aws s3 {cp,sync} manpages with examples

# TODO maybe add debug logging for new options and option encrichment, etc.?

# TODO is there a better way to do the opt parsing within the awscli framework?

# TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
SSECustomerKeyMD5, and/or SSEKMSKeyId values present in response data after
certain APIs (head_object(), put_object(), get_object(), copy_object(),
create_multipart_upload(), upload_part(), upload_part_copy()) have expected
values given the request (and, not specific to SSE, there is no checking of
these now for ETag, etc.). Approximate source locations as of 2015-08-27
(these comments were removed afterwards, so source line numbers are slightly
different; look for use of client APIs above):

awscli/customizations/s3/filegenerator.py
317: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,

awscli/customizations/s3/fileinfo.py
196: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
292: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
311: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
329: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
372: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,

awscli/customizations/s3/tasks.py
180: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
261: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
401: # TODO validate that ServerSideEncryption, SSECustomerAlgorithm,
535: # TODO validate that ServerSideEncryption and/or SSEKMSKeyId values

(complete_multipart_upload() has only ServerSideEncryption and/or
SSEKMSKeyId. The rest have all four.)

# TODO consider storing md5sum/sha256sum in object metadata (this is not
specific to SSE, but to anything that alters ETag like multipart) so it could
later be used (perhaps for sync)

14 changes: 12 additions & 2 deletions awscli/customizations/s3/filegenerator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,14 +115,19 @@ class FileGenerator(object):
``FileInfo`` objects to send to a ``Comparator`` or ``S3Handler``.
"""
def __init__(self, client, operation_name, follow_symlinks=True,
page_size=None, result_queue=None):
page_size=None, result_queue=None,
sse_customer_algorithm=None, sse_customer_key=None,
sse_customer_key_md5=None):
self._client = client
self.operation_name = operation_name
self.follow_symlinks = follow_symlinks
self.page_size = page_size
self.result_queue = result_queue
if not result_queue:
self.result_queue = queue.Queue()
self.sse_customer_algorithm = sse_customer_algorithm
self.sse_customer_key = sse_customer_key
self.sse_customer_key_md5 = sse_customer_key_md5

def call(self, files):
"""
Expand Down Expand Up @@ -303,7 +308,12 @@ def _list_single_object(self, s3_path):
# instead use a HeadObject request.
bucket, key = find_bucket_key(s3_path)
try:
response = self._client.head_object(Bucket=bucket, Key=key)
params = {'Bucket': bucket, 'Key': key}
if self.sse_customer_key:
params['SSECustomerAlgorithm'] = self.sse_customer_algorithm
params['SSECustomerKey'] = self.sse_customer_key
params['SSECustomerKeyMD5'] = self.sse_customer_key_md5
response = self._client.head_object(**params)
except ClientError as e:
# We want to try to give a more helpful error message.
# This is what the customer is going to see so we want to
Expand Down
38 changes: 34 additions & 4 deletions awscli/customizations/s3/fileinfo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ def save_file(filename, response_data, last_update, is_stream=False):
body = response_data['Body']
etag = response_data['ETag'][1:-1]
sse = response_data.get('ServerSideEncryption', None)
sse_customer_algorithm = response_data.get('SSECustomerAlgorithm', None)
if not is_stream:
d = os.path.dirname(filename)
try:
Expand All @@ -55,7 +56,8 @@ def save_file(filename, response_data, last_update, is_stream=False):
with open(filename, 'wb') as out_file:
write_to_file(out_file, etag, md5, file_chunks)

if not _is_multipart_etag(etag) and sse != 'aws:kms':
if not _is_multipart_etag(etag) and sse != 'aws:kms' and \
sse_customer_algorithm is None:
if etag != md5.hexdigest():
if not is_stream:
os.remove(filename)
Expand Down Expand Up @@ -189,6 +191,7 @@ def set_size_from_s3(self):
bucket, key = find_bucket_key(self.src)
params = {'Bucket': bucket,
'Key': key}
self._handle_sse_params(params)
response_data = self.client.head_object(**params)
self.size = int(response_data['ContentLength'])

Expand All @@ -215,8 +218,6 @@ def _handle_object_params(self, params):
raise ValueError('grants should be of the form '
'permission=principal')
params[self._permission_to_param(permission)] = grantee
if self.parameters['sse']:
params['ServerSideEncryption'] = 'AES256'
if self.parameters['storage_class']:
params['StorageClass'] = self.parameters['storage_class'][0]
if self.parameters['website_redirect']:
Expand All @@ -238,6 +239,27 @@ def _handle_object_params(self, params):
if self.parameters['expires']:
params['Expires'] = self.parameters['expires'][0]

def _handle_sse_params(self, params):
if self.parameters['sse_copy_source_customer_key']:
params['CopySourceSSECustomerAlgorithm'] = \
self.parameters['sse_copy_source_customer_algorithm']
params['CopySourceSSECustomerKey'] = \
self.parameters['sse_copy_source_customer_key']
params['CopySourceSSECustomerKeyMD5'] = \
self.parameters['sse_copy_source_customer_key_md5']
if self.parameters['sse_class'] == 'C':
params['SSECustomerAlgorithm'] = \
self.parameters['sse_customer_algorithm']
params['SSECustomerKey'] = \
self.parameters['sse_customer_key']
params['SSECustomerKeyMD5'] = \
self.parameters['sse_customer_key_md5']
if self.parameters['sse_class'] == 'KMS':
params['ServerSideEncryption'] = 'aws:kms'
params['SSEKMSKeyId'] = self.parameters['sse_kms_key_id']
if self.parameters['sse_class'] == 'S3':
params['ServerSideEncryption'] = 'AES256'

def _handle_metadata_directive(self, params):
if self.parameters['metadata_directive']:
params['MetadataDirective'] = \
Expand All @@ -262,6 +284,7 @@ def _handle_upload(self, body):
'Body': body,
}
self._handle_object_params(params)
self._handle_sse_params(params)
response_data = self.client.put_object(**params)

def _inject_content_type(self, params, filename):
Expand All @@ -277,6 +300,7 @@ def download(self):
"""
bucket, key = find_bucket_key(self.src)
params = {'Bucket': bucket, 'Key': key}
self._handle_sse_params(params)
response_data = self.client.get_object(**params)
save_file(self.dest, response_data, self.last_update,
self.is_stream)
Expand All @@ -290,8 +314,9 @@ def copy(self):
params = {'Bucket': bucket,
'CopySource': copy_source, 'Key': key}
self._handle_object_params(params)
self._handle_sse_params(params)
self._handle_metadata_directive(params)
self.client.copy_object(**params)
response_data = self.client.copy_object(**params)

def delete(self):
"""
Expand Down Expand Up @@ -325,6 +350,11 @@ def create_multipart_upload(self):
bucket, key = find_bucket_key(self.dest)
params = {'Bucket': bucket, 'Key': key}
self._handle_object_params(params)
self._handle_sse_params(params)
params = params.copy()
params.pop('CopySourceSSECustomerAlgorithm', None)
params.pop('CopySourceSSECustomerKey', None)
params.pop('CopySourceSSECustomerKeyMD5', None)
response_data = self.client.create_multipart_upload(**params)
upload_id = response_data['UploadId']
return upload_id
42 changes: 31 additions & 11 deletions awscli/customizations/s3/s3handler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,15 +55,33 @@ def __init__(self, session, params, result_queue=None,
self.result_queue = result_queue
if not self.result_queue:
self.result_queue = queue.Queue()
self.params = {'dryrun': False, 'quiet': False, 'acl': None,
'guess_mime_type': True, 'sse': False,
'storage_class': None, 'website_redirect': None,
'content_type': None, 'cache_control': None,
'content_disposition': None, 'content_encoding': None,
'content_language': None, 'expires': None,
'grants': None, 'only_show_errors': False,
'is_stream': False, 'paths_type': None,
'expected_size': None, 'metadata_directive': None}
self.params = {'dryrun': False,
'quiet': False,
'acl': None,
'guess_mime_type': True,
'sse_copy_source_customer_algorithm': None,
'sse_copy_source_customer_key': None,
'sse_copy_source_customer_key_md5': None,
'sse': False,
'sse_class': None,
'sse_customer_algorithm': None,
'sse_customer_key': None,
'sse_customer_key_md5': None,
'sse_kms_key_id': None,
'storage_class': None,
'website_redirect': None,
'content_type': None,
'cache_control': None,
'content_disposition': None,
'content_encoding': None,
'content_language': None,
'expires': None,
'grants': None,
'only_show_errors': False,
'is_stream': False,
'paths_type': None,
'expected_size': None,
'metadata_directive': None}
self.params['region'] = params['region']
for key in self.params.keys():
if key in params:
Expand Down Expand Up @@ -269,7 +287,8 @@ def _do_enqueue_range_download_tasks(self, filename, chunksize,
task = tasks.DownloadPartTask(
part_number=i, chunk_size=chunksize,
result_queue=self.result_queue, filename=filename,
context=context, io_queue=self.write_queue)
context=context, io_queue=self.write_queue,
params=self.params)
self.executor.submit(task)

def _enqueue_multipart_upload_tasks(self, filename,
Expand Down Expand Up @@ -332,7 +351,8 @@ def _enqueue_upload_single_part_task(self, part_number, chunk_size,
payload=None):
kwargs = {'part_number': part_number, 'chunk_size': chunk_size,
'result_queue': self.result_queue,
'upload_context': upload_context, 'filename': filename}
'upload_context': upload_context, 'filename': filename,
'params': self.params}
if payload:
kwargs['payload'] = payload
task = task_class(**kwargs)
Expand Down
Loading

0 comments on commit c2b3849

Please sign in to comment.