Add s3 presign command

.changes/next-release/feature-s3-23563.json

@@ -0,0 +1,5 @@
+{
+  "type": "feature",
+  "category": "``s3``",
+  "description": "Add a new ``aws s3 presign`` command, closes `#462 <https://github.com/aws/aws-cli/issues/462>`__"
+}

awscli/customizations/s3/s3.py

@@ -13,7 +13,8 @@
 from awscli.customizations import utils
 from awscli.customizations.commands import BasicCommand
 from awscli.customizations.s3.subcommands import ListCommand, WebsiteCommand, \
-    CpCommand, MvCommand, RmCommand, SyncCommand, MbCommand, RbCommand
+    CpCommand, MvCommand, RmCommand, SyncCommand, MbCommand, RbCommand, \
+    PresignCommand
 from awscli.customizations.s3.syncstrategy.register import \
     register_sync_strategies
 
@@ -58,7 +59,8 @@ class S3(BasicCommand):
         {'name': 'rm', 'command_class': RmCommand},
         {'name': 'sync', 'command_class': SyncCommand},
         {'name': 'mb', 'command_class': MbCommand},
-        {'name': 'rb', 'command_class': RbCommand}
+        {'name': 'rb', 'command_class': RbCommand},
+        {'name': 'presign', 'command_class': PresignCommand},
     ]
 
     def _run_main(self, parsed_args, parsed_globals):

awscli/customizations/s3/subcommands.py

@@ -605,6 +605,36 @@ def _get_bucket_name(self, path):
         return path
 
 
+class PresignCommand(S3Command):
+    NAME = 'presign'
+    DESCRIPTION = ("Generate a pre-signed URL for an Amazon S3 object. "
+                   "This allows anyone who receives the pre-signed URL "
+                   "to retrieve the S3 object with an HTTP GET request.")
+    USAGE = "<S3Uri>"
+    ARG_TABLE = [{'name': 'path',
+                  'positional_arg': True, 'synopsis': USAGE},
+                 {'name': 'expires-in', 'default': 3600,
+                  'cli_type_name': 'integer',
+                  'help_text': (
+                      'Number of seconds until the pre-signed '
+                      'URL expires.  Default is 3600 seconds.')}]
+
+    def _run_main(self, parsed_args, parsed_globals):
+        super(PresignCommand, self)._run_main(parsed_args, parsed_globals)
+        path = parsed_args.path
+        if path.startswith('s3://'):
+            path = path[5:]
+        bucket, key = find_bucket_key(path)
+        url = self.client.generate_presigned_url(
+            'get_object',
+            {'Bucket': bucket, 'Key': key},
+            ExpiresIn=parsed_args.expires_in
+        )
+        uni_print(url)
+        uni_print('\n')
+        return 0
+
+
 class S3TransferCommand(S3Command):
     def _run_main(self, parsed_args, parsed_globals):
         super(S3TransferCommand, self)._run_main(parsed_args, parsed_globals)

awscli/testutils.py

@@ -307,6 +307,7 @@ class BaseAWSCommandParamsTest(unittest.TestCase):
 
     def setUp(self):
         self.last_params = {}
+        self.last_kwargs = None
         # awscli/__init__.py injects AWS_DATA_PATH at import time
         # so that we can find cli.json.  This might be fixed in the
         # future, but for now we just grab that value out of the real
@@ -317,6 +318,8 @@ def setUp(self):
             'AWS_DEFAULT_REGION': 'us-east-1',
             'AWS_ACCESS_KEY_ID': 'access_key',
             'AWS_SECRET_ACCESS_KEY': 'secret_key',
+            'AWS_CONFIG_FILE': '',
+            'AWS_SHARED_CREDENTIALS_FILE': '',
         }
         self.environ_patch = mock.patch('os.environ', self.environ)
         self.environ_patch.start()

tests/functional/s3/test_presign_command.py

@@ -0,0 +1,187 @@
+# Copyright 2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+import datetime
+
+from botocore.compat import urlsplit
+from awscli.testutils import BaseAWSCommandParamsTest, mock, temporary_file
+from awscli.testutils import create_clidriver
+
+
+# Values used to fix time.time() and datetime.datetime.utcnow()
+# so we know the exact values of the signatures generated.
+FROZEN_TIMESTAMP = 1471305652
+DEFAULT_EXPIRES = 3600
+FROZEN_TIME = mock.Mock(return_value=FROZEN_TIMESTAMP)
+FROZEN_DATETIME = mock.Mock(
+    return_value=datetime.datetime(2016, 8, 18, 14, 33, 3, 0))
+
+
+class TestPresignCommand(BaseAWSCommandParamsTest):
+
+    prefix = 's3 presign '
+
+    def enable_addressing_mode_in_config(self, fileobj, mode):
+        fileobj.write(
+            "[default]\n"
+            "s3 =\n"
+            "    addressing_style = %s\n" % mode
+        )
+        fileobj.flush()
+        self.environ['AWS_CONFIG_FILE'] = fileobj.name
+        self.driver = create_clidriver()
+
+    def enable_sigv4_from_config_file(self, fileobj):
+        fileobj.write(
+            "[default]\n"
+            "s3 =\n"
+            "    signature_version = s3v4\n"
+        )
+        fileobj.flush()
+        self.environ['AWS_CONFIG_FILE'] = fileobj.name
+        self.driver = create_clidriver()
+
+    def assert_presigned_url_matches(self, actual_url, expected_match):
+        """Verify generated presigned URL matches expected dict.
+
+        This method compares an actual URL against a dict of expected
+        values.  The reason that the "expected_match" is a dict instead
+        of the expected presigned URL is because the query params
+        are unordered so we can't guarantee an expected query param
+        ordering.
+
+        """
+        parts = urlsplit(actual_url)
+        self.assertEqual(parts.netloc, expected_match['hostname'])
+        self.assertEqual(parts.path, expected_match['path'])
+        query_params = self.parse_query_string(parts.query)
+        self.assertEqual(query_params, expected_match['query_params'])
+
+    def parse_query_string(self, query_string):
+        pairs = []
+        for part in query_string.split('&'):
+            pairs.append(part.split('=', 1))
+        return dict(pairs)
+
+    def get_presigned_url_for_cmd(self, cmdline):
+        with mock.patch('time.time', FROZEN_TIME):
+            with mock.patch('datetime.datetime') as d:
+                d.utcnow = FROZEN_DATETIME
+                stdout = self.assert_params_for_cmd(cmdline, None)[0].strip()
+                return stdout
+
+    def test_generates_a_url(self):
+        stdout = self.get_presigned_url_for_cmd(
+            self.prefix + 's3://bucket/key')
+
+        self.assert_presigned_url_matches(
+            stdout, {
+                'hostname': 'bucket.s3.amazonaws.com',
+                'path': '/key',
+                'query_params': {
+                    'AWSAccessKeyId': 'access_key',
+                    'Expires': str(FROZEN_TIMESTAMP + DEFAULT_EXPIRES),
+                    'Signature': '2m9M0eLB%2BqI0nUpkyTskKmHd0Ig%3D',
+                }
+            }
+        )
+
+    def test_handles_non_dns_compatible_buckets(self):
+        stdout = self.get_presigned_url_for_cmd(
+            self.prefix + 's3://bucket.dots/key')
+
+        self.assert_presigned_url_matches(
+            stdout, {
+                'hostname': 's3.amazonaws.com',
+                'path': '/bucket.dots/key',
+                'query_params': {
+                    'AWSAccessKeyId': 'access_key',
+                    'Expires': str(FROZEN_TIMESTAMP + DEFAULT_EXPIRES),
+                    'Signature': '0IiC2vxub438EVcKfEFEMHuoHRw%3D',
+                }
+            }
+        )
+
+    def test_handles_expires_in(self):
+        expires_in = 1000
+        stdout = self.get_presigned_url_for_cmd(
+            self.prefix + 's3://bucket/key --expires-in %s' % expires_in)
+
+        self.assert_presigned_url_matches(
+            stdout, {
+                'hostname': 'bucket.s3.amazonaws.com',
+                'path': '/key',
+                'query_params': {
+                    'AWSAccessKeyId': 'access_key',
+                    'Expires': str(FROZEN_TIMESTAMP + expires_in),
+                    'Signature': 'WZEMcfBNlzfTZBq3bOvYef1cfoU%3D',
+                }
+            }
+        )
+
+    def test_handles_sigv4(self):
+        with temporary_file('w') as f:
+            self.enable_sigv4_from_config_file(f)
+            stdout = self.get_presigned_url_for_cmd(
+                self.prefix + 's3://bucket/key')
+
+        expected = {
+            'hostname': 's3.amazonaws.com',
+            'path': '/bucket/key',
+            'query_params': {
+                'X-Amz-Algorithm': 'AWS4-HMAC-SHA256',
+                'X-Amz-Credential': (
+                    'access_key%2F20160818%2Fus-east-1'
+                    '%2Fs3%2Faws4_request'),
+                'X-Amz-Date': '20160818T143303Z',
+                'X-Amz-Expires': '3600',
+                'X-Amz-Signature': (
+                    'd20178280d7521b384730c678549f6344401ae040bec559ad0602'
+                    '0854c6c718f'),
+                'X-Amz-SignedHeaders': 'host'
+            }
+        }
+        self.assert_presigned_url_matches(stdout, expected)
+
+    def test_s3_prefix_not_needed(self):
+        # Consistent with the 'ls' command.
+        stdout = self.get_presigned_url_for_cmd(
+            self.prefix + 'bucket/key')
+
+        self.assert_presigned_url_matches(
+            stdout, {
+                'hostname': 'bucket.s3.amazonaws.com',
+                'path': '/key',
+                'query_params': {
+                    'AWSAccessKeyId': 'access_key',
+                    'Expires': str(FROZEN_TIMESTAMP + DEFAULT_EXPIRES),
+                    'Signature': '2m9M0eLB%2BqI0nUpkyTskKmHd0Ig%3D',
+                }
+            }
+        )
+
+    def test_can_support_addressing_mode_config(self):
+        with temporary_file('w') as f:
+            self.enable_addressing_mode_in_config(f, 'path')
+            stdout = self.get_presigned_url_for_cmd(
+                self.prefix + 's3://bucket/key')
+        self.assert_presigned_url_matches(
+            stdout, {
+                'hostname': 's3.amazonaws.com',
+                'path': '/bucket/key',
+                'query_params': {
+                    'AWSAccessKeyId': 'access_key',
+                    'Expires': str(FROZEN_TIMESTAMP + DEFAULT_EXPIRES),
+                    'Signature': '2m9M0eLB%2BqI0nUpkyTskKmHd0Ig%3D',
+                }
+            }
+        )

tests/integration/customizations/s3/test_plugin.py

@@ -29,7 +29,7 @@
 import copy
 import logging
 
-from awscli.compat import six
+from awscli.compat import six, urlopen
 from nose.plugins.attrib import attr
 import botocore.session
 
@@ -2089,3 +2089,16 @@ def test_smoke_sync_sse_c(self):
         self.assertTrue(os.path.isfile(file_name))
         with open(file_name, 'r') as f:
             self.assertEqual(f.read(), contents)
+
+
+class TestPresignCommand(BaseS3IntegrationTest):
+
+    def test_can_retrieve_presigned_url(self):
+        bucket_name = _SHARED_BUCKET
+        original_contents = b'this is foo.txt'
+        self.put_object(bucket_name, 'foo.txt', original_contents)
+        p = aws('s3 presign s3://%s/foo.txt' % (bucket_name,))
+        self.assert_no_errors(p)
+        url = p.stdout.strip()
+        contents = urlopen(url).read()
+        self.assertEqual(contents, original_contents)