diff --git a/.gitignore b/.gitignore index 8bb22129f381..1530bb30d659 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,5 @@ doc/source/tutorial/services.rst # Pyenv .python-version +# virtualenv +.venv diff --git a/awscli/customizations/s3/filegenerator.py b/awscli/customizations/s3/filegenerator.py index f972f316b4a5..c2ef18822cca 100644 --- a/awscli/customizations/s3/filegenerator.py +++ b/awscli/customizations/s3/filegenerator.py @@ -31,7 +31,7 @@ def is_special_file(path): """ This function checks to see if a special file. It checks if the file is a character special device, block special device, FIFO, or - socket. + socket. """ mode = os.stat(path).st_mode # Character special device. @@ -318,8 +318,9 @@ def list_objects(self, s3_path, dir_op): yield self._list_single_object(s3_path) else: lister = BucketLister(self._client) + request_payer = self.request_parameters.get('RequestPayer') for key in lister.list_objects(bucket=bucket, prefix=prefix, - page_size=self.page_size): + page_size=self.page_size, request_payer=request_payer): source_path, response_data = key if response_data['Size'] == 0 and source_path.endswith('/'): if self.operation_name == 'delete': diff --git a/awscli/customizations/s3/s3handler.py b/awscli/customizations/s3/s3handler.py index 70a6d74bc41b..6e4c7f1e82a8 100644 --- a/awscli/customizations/s3/s3handler.py +++ b/awscli/customizations/s3/s3handler.py @@ -497,7 +497,7 @@ def _format_local_path(self, path): class DeleteRequestSubmitter(BaseTransferRequestSubmitter): - REQUEST_MAPPER_METHOD = None + REQUEST_MAPPER_METHOD = RequestParamsMapper.map_delete_params RESULT_SUBSCRIBER_CLASS = DeleteResultSubscriber def can_submit(self, fileinfo): diff --git a/awscli/customizations/s3/subcommands.py b/awscli/customizations/s3/subcommands.py index da30d6692e6d..b7f8ae4f1268 100644 --- a/awscli/customizations/s3/subcommands.py +++ b/awscli/customizations/s3/subcommands.py @@ -725,7 +725,7 @@ class CpCommand(S3TransferCommand): "or " ARG_TABLE = [{'name': 'paths', 'nargs': 2, 'positional_arg': True, 'synopsis': USAGE}] + TRANSFER_ARGS + \ - [METADATA, METADATA_DIRECTIVE, EXPECTED_SIZE, RECURSIVE] + [METADATA, METADATA_DIRECTIVE, EXPECTED_SIZE, RECURSIVE, REQUEST_PAYER] class MvCommand(S3TransferCommand): @@ -744,7 +744,7 @@ class RmCommand(S3TransferCommand): USAGE = "" ARG_TABLE = [{'name': 'paths', 'nargs': 1, 'positional_arg': True, 'synopsis': USAGE}, DRYRUN, QUIET, RECURSIVE, INCLUDE, - EXCLUDE, ONLY_SHOW_ERRORS, PAGE_SIZE] + EXCLUDE, ONLY_SHOW_ERRORS, PAGE_SIZE, REQUEST_PAYER] class SyncCommand(S3TransferCommand): @@ -757,7 +757,7 @@ class SyncCommand(S3TransferCommand): " or " ARG_TABLE = [{'name': 'paths', 'nargs': 2, 'positional_arg': True, 'synopsis': USAGE}] + TRANSFER_ARGS + \ - [METADATA, METADATA_DIRECTIVE] + [METADATA, METADATA_DIRECTIVE, REQUEST_PAYER] class MbCommand(S3Command): @@ -985,11 +985,18 @@ def run(self): 'result_queue': result_queue, } - fgen_request_parameters = {} + fgen_request_parameters = { + 'RequestPayer': self.parameters.get('request_payer'), + } fgen_head_object_params = {} fgen_request_parameters['HeadObject'] = fgen_head_object_params fgen_kwargs['request_parameters'] = fgen_request_parameters + rgen_request_parameters = { + 'RequestPayer': self.parameters.get('request_payer'), + } + rgen_kwargs['request_parameters'] = rgen_request_parameters + # SSE-C may be neaded for HeadObject for copies/downloads/deletes # If the operation is s3 to s3, the FileGenerator should use the # copy source key and algorithm. Otherwise, use the regular @@ -1111,6 +1118,8 @@ def __init__(self, cmd, parameters, usage): self.parameters['follow_symlinks'] = True if 'source_region' not in parameters: self.parameters['source_region'] = None + if 'request_payer' not in parameters: + self.parameters['request_payer'] = None if self.cmd in ['sync', 'mb', 'rb']: self.parameters['dir_op'] = True if self.cmd == 'mv': diff --git a/awscli/customizations/s3/utils.py b/awscli/customizations/s3/utils.py index 9d3e491e1f46..df9f42310255 100644 --- a/awscli/customizations/s3/utils.py +++ b/awscli/customizations/s3/utils.py @@ -357,10 +357,12 @@ def __init__(self, client, date_parser=_date_parser): self._client = client self._date_parser = date_parser - def list_objects(self, bucket, prefix=None, page_size=None): + def list_objects(self, bucket, prefix=None, page_size=None, request_payer=None): kwargs = {'Bucket': bucket, 'PaginationConfig': {'PageSize': page_size}} if prefix is not None: kwargs['Prefix'] = prefix + if request_payer is not None: + kwargs['RequestPayer'] = request_payer paginator = self._client.get_paginator('list_objects') pages = paginator.paginate(**kwargs) @@ -424,11 +426,13 @@ def map_put_object_params(cls, request_params, cli_params): cls._set_metadata_params(request_params, cli_params) cls._set_sse_request_params(request_params, cli_params) cls._set_sse_c_request_params(request_params, cli_params) + cls._set_request_payer_param(request_params, cli_params) @classmethod def map_get_object_params(cls, request_params, cli_params): """Map CLI params to GetObject request params""" cls._set_sse_c_request_params(request_params, cli_params) + cls._set_request_payer_param(request_params, cli_params) @classmethod def map_copy_object_params(cls, request_params, cli_params): @@ -440,11 +444,13 @@ def map_copy_object_params(cls, request_params, cli_params): cls._set_sse_request_params(request_params, cli_params) cls._set_sse_c_and_copy_source_request_params( request_params, cli_params) + cls._set_request_payer_param(request_params, cli_params) @classmethod def map_head_object_params(cls, request_params, cli_params): """Map CLI params to HeadObject request params""" cls._set_sse_c_request_params(request_params, cli_params) + cls._set_request_payer_param(request_params, cli_params) @classmethod def map_create_multipart_upload_params(cls, request_params, cli_params): @@ -453,21 +459,33 @@ def map_create_multipart_upload_params(cls, request_params, cli_params): cls._set_sse_request_params(request_params, cli_params) cls._set_sse_c_request_params(request_params, cli_params) cls._set_metadata_params(request_params, cli_params) + cls._set_request_payer_param(request_params, cli_params) @classmethod def map_upload_part_params(cls, request_params, cli_params): """Map CLI params to UploadPart request params""" cls._set_sse_c_request_params(request_params, cli_params) + cls._set_request_payer_param(request_params, cli_params) @classmethod def map_upload_part_copy_params(cls, request_params, cli_params): """Map CLI params to UploadPartCopy request params""" cls._set_sse_c_and_copy_source_request_params( request_params, cli_params) + cls._set_request_payer_param(request_params, cli_params) + + @classmethod + def map_delete_params(cls, request_params, cli_params): + cls._set_request_payer_param(request_params, cli_params) + + @classmethod + def _set_request_payer_param(cls, request_params, cli_params): + if cli_params.get('request_payer'): + request_params['RequestPayer'] = cli_params['request_payer'] @classmethod def _set_general_object_params(cls, request_params, cli_params): - # Paramters set in this method should be applicable to the following + # Parameters set in this method should be applicable to the following # operations involving objects: PutObject, CopyObject, and # CreateMultipartUpload. general_param_translation = {