Add a content parameter to aws_s3 put operation (#20)

* Document usage of aws_s3 content parameter
* Added utility function to calculate s3 etag from content instead of file
* Implemented s3 put from bytes content
* Replace file md5 utility call with direct calculation
* Added integration tests for static content and template put
* Add idempotency test
* Validate src as path
* Add new content_base64 parameter
* Add test case for S3 upload from b64 encoded binary
* Add changelog fragment

changelogs/fragments/20-aws_s3-content.yml

@@ -0,0 +1,2 @@
+minor_changes:
+- aws_s3 - Add support for uploading templated content (https://github.com/ansible-collections/amazon.aws/pull/20).

plugins/module_utils/s3.py

@@ -48,3 +48,36 @@ def calculate_etag(module, filename, etag, s3, bucket, obj, version=None):
         return '"{0}-{1}"'.format(digest_squared.hexdigest(), len(digests))
     else:  # Compute the MD5 sum normally
         return '"{0}"'.format(module.md5(filename))
+
+
+def calculate_etag_content(module, content, etag, s3, bucket, obj, version=None):
+    if not HAS_MD5:
+        return None
+
+    if '-' in etag:
+        # Multi-part ETag; a hash of the hashes of each part.
+        parts = int(etag[1:-1].split('-')[1])
+        digests = []
+        offset = 0
+
+        s3_kwargs = dict(
+            Bucket=bucket,
+            Key=obj,
+        )
+        if version:
+            s3_kwargs['VersionId'] = version
+
+        for part_num in range(1, parts + 1):
+            s3_kwargs['PartNumber'] = part_num
+            try:
+                head = s3.head_object(**s3_kwargs)
+            except (BotoCoreError, ClientError) as e:
+                module.fail_json_aws(e, msg="Failed to get head object")
+            length = int(head['ContentLength'])
+            digests.append(md5(content[offset:offset + length]))
+            offset += length
+
+        digest_squared = md5(b''.join(m.digest() for m in digests))
+        return '"{0}-{1}"'.format(digest_squared.hexdigest(), len(digests))
+    else:  # Compute the MD5 sum normally
+        return '"{0}"'.format(md5(content).hexdigest())

plugins/modules/aws_s3.py

@@ -126,6 +126,22 @@
   src:
     description:
       - The source file path when performing a PUT operation.
+      - Either I(content), I(content_base64) or I(src) must be specified for a PUT operation. Ignored otherwise.
+    type: path
+  content:
+    description:
+      - The content to PUT into an object.
+      - The parameter value will be treated as a string and converted to UTF-8 before sending it to S3.
+        To send binary data, use the I(content_base64) parameter instead.
+      - Either I(content), I(content_base64) or I(src) must be specified for a PUT operation. Ignored otherwise.
+    version_added: "1.3.0"
+    type: str
+  content_base64:
+    description:
+      - The base64-encoded binary data to PUT into an object.
+      - Use this if you need to put raw binary data, and don't forget to encode in base64.
+      - Either I(content), I(content_base64) or I(src) must be specified for a PUT operation. Ignored otherwise.
+    version_added: "1.3.0"
     type: str
   ignore_nonexistent_bucket:
     description:
@@ -155,6 +171,13 @@
     src: /usr/local/myfile.txt
     mode: put
 
+- name: PUT operation from a rendered template
+  aws_s3:
+    bucket: mybucket
+    object: /object.yaml
+    content: "{{ lookup('template', 'templates/object.yaml.j2') }}"
+    mode: put
+
 - name: Simple PUT operation in Ceph RGW S3
   amazon.aws.aws_s3:
     bucket: mybucket
@@ -275,7 +298,9 @@
 
 import mimetypes
 import os
+import io
 from ssl import SSLError
+import base64
 
 try:
     import botocore
@@ -294,6 +319,7 @@
 from ..module_utils.ec2 import get_aws_connection_info
 from ..module_utils.s3 import HAS_MD5
 from ..module_utils.s3 import calculate_etag
+from ..module_utils.s3 import calculate_etag_content
 
 IGNORE_S3_DROP_IN_EXCEPTIONS = ['XNotImplemented', 'NotImplemented']
 
@@ -319,9 +345,12 @@ def key_check(module, s3, bucket, obj, version=None, validate=True):
     return True
 
 
-def etag_compare(module, local_file, s3, bucket, obj, version=None):
+def etag_compare(module, s3, bucket, obj, version=None, local_file=None, content=None):
     s3_etag = get_etag(s3, bucket, obj, version=version)
-    local_etag = calculate_etag(module, local_file, s3_etag, s3, bucket, obj, version)
+    if local_file is not None:
+        local_etag = calculate_etag(module, local_file, s3_etag, s3, bucket, obj, version)
+    else:
+        local_etag = calculate_etag_content(module, content, s3_etag, s3, bucket, obj, version)
 
     return s3_etag == local_etag
 
@@ -479,7 +508,7 @@ def option_in_extra_args(option):
         return allowed_extra_args[temp_option]
 
 
-def upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, headers):
+def upload_s3file(module, s3, bucket, obj, expiry, metadata, encrypt, headers, src=None, content=None):
     if module.check_mode:
         module.exit_json(msg="PUT operation skipped - running in check mode", changed=True)
     try:
@@ -500,13 +529,19 @@ def upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, heade
                     extra['Metadata'][option] = metadata[option]
 
         if 'ContentType' not in extra:
-            content_type = mimetypes.guess_type(src)[0]
+            content_type = None
+            if src is not None:
+                content_type = mimetypes.guess_type(src)[0]
             if content_type is None:
                 # s3 default content type
                 content_type = 'binary/octet-stream'
             extra['ContentType'] = content_type
 
-        s3.upload_file(Filename=src, Bucket=bucket, Key=obj, ExtraArgs=extra)
+        if src is not None:
+            s3.upload_file(Filename=src, Bucket=bucket, Key=obj, ExtraArgs=extra)
+        else:
+            f = io.BytesIO(content)
+            s3.upload_fileobj(Fileobj=f, Bucket=bucket, Key=obj, ExtraArgs=extra)
     except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
         module.fail_json_aws(e, msg="Unable to complete PUT operation.")
     try:
@@ -652,17 +687,20 @@ def main():
         s3_url=dict(aliases=['S3_URL']),
         dualstack=dict(default='no', type='bool'),
         rgw=dict(default='no', type='bool'),
-        src=dict(),
+        src=dict(type='path'),
+        content=dict(),
+        content_base64=dict(),
         ignore_nonexistent_bucket=dict(default=False, type='bool'),
         encryption_kms_key_id=dict()
     )
     module = AnsibleAWSModule(
         argument_spec=argument_spec,
         supports_check_mode=True,
-        required_if=[['mode', 'put', ['src', 'object']],
+        required_if=[['mode', 'put', ['object']],
                      ['mode', 'get', ['dest', 'object']],
                      ['mode', 'getstr', ['object']],
                      ['mode', 'geturl', ['object']]],
+        mutually_exclusive=[['content', 'content_base64', 'src']],
     )
 
     bucket = module.params.get('bucket')
@@ -683,6 +721,8 @@ def main():
     dualstack = module.params.get('dualstack')
     rgw = module.params.get('rgw')
     src = module.params.get('src')
+    content = module.params.get('content')
+    content_base64 = module.params.get('content_base64')
     ignore_nonexistent_bucket = module.params.get('ignore_nonexistent_bucket')
 
     object_canned_acl = ["private", "public-read", "public-read-write", "aws-exec-read", "authenticated-read", "bucket-owner-read", "bucket-owner-full-control"]
@@ -762,10 +802,10 @@ def main():
             else:
                 module.fail_json(msg="Key %s does not exist." % obj)
 
-        if path_check(dest) and overwrite != 'always':
+        if dest and path_check(dest) and overwrite != 'always':
             if overwrite == 'never':
                 module.exit_json(msg="Local object already exists and overwrite is disabled.", changed=False)
-            if etag_compare(module, dest, s3, bucket, obj, version=version):
+            if etag_compare(module, s3, bucket, obj, version=version, local_file=dest):
                 module.exit_json(msg="Local and remote object are identical, ignoring. Use overwrite=always parameter to force.", changed=False)
 
         try:
@@ -779,8 +819,10 @@ def main():
         # if putting an object in a bucket yet to be created, acls for the bucket and/or the object may be specified
         # these were separated into the variables bucket_acl and object_acl above
 
-        if not path_check(src):
-            module.fail_json(msg="Local object for PUT does not exist")
+        if content is None and content_base64 is None and src is None:
+            module.fail_json('Either content, content_base64 or src must be specified for PUT operations')
+        if src is not None and not path_check(src):
+            module.fail_json('Local object "%s" does not exist for PUT operation' % (src))
 
         if bucketrtn:
             keyrtn = key_check(module, s3, bucket, obj, version=version, validate=validate)
@@ -790,14 +832,21 @@ def main():
             module.params['permission'] = bucket_acl
             create_bucket(module, s3, bucket, location)
 
+        # the content will be uploaded as a byte string, so we must encode it first
+        bincontent = None
+        if content is not None:
+            bincontent = content.encode('utf-8')
+        if content_base64 is not None:
+            bincontent = base64.standard_b64decode(content_base64)
+
         if keyrtn and overwrite != 'always':
-            if overwrite == 'never' or etag_compare(module, src, s3, bucket, obj):
+            if overwrite == 'never' or etag_compare(module, s3, bucket, obj, version=version, local_file=src, content=bincontent):
                 # Return the download URL for the existing object
                 get_download_url(module, s3, bucket, obj, expiry, changed=False)
 
         # only use valid object acls for the upload_s3file function
         module.params['permission'] = object_acl
-        upload_s3file(module, s3, bucket, obj, src, expiry, metadata, encrypt, headers)
+        upload_s3file(module, s3, bucket, obj, expiry, metadata, encrypt, headers, src=src, content=bincontent)
 
     # Delete an object from a bucket, not the entire bucket
     if mode == 'delobj':

tests/integration/targets/aws_s3/tasks/main.yml

@@ -526,6 +526,108 @@
               - result is not changed
       when: ansible_system == 'Linux' or ansible_distribution == 'MacOSX'
 
+    - name: create an object from static content
+      aws_s3:
+        bucket: "{{ bucket_name }}"
+        object: put-content.txt
+        mode: put
+        content: >-
+          test content
+      register: result
+
+    - assert:
+        that:
+          - result is changed
+
+    - name: ensure idempotency on static content
+      aws_s3:
+        bucket: "{{ bucket_name }}"
+        object: put-content.txt
+        mode: put
+        overwrite: different
+        content: >-
+          test content
+      register: result
+
+    - assert:
+        that:
+          - result is not changed
+
+    - name: fetch test content
+      aws_s3:
+        bucket: "{{ bucket_name }}"
+        mode: getstr
+        object: put-content.txt
+      register: result
+
+    - assert:
+        that:
+          - result.contents == "test content"
+
+    - set_fact:
+        put_template_text: test template
+
+    - name: create an object from a template
+      aws_s3:
+        bucket: "{{ bucket_name }}"
+        object: put-template.txt
+        mode: put
+        content: "{{ lookup('template', 'templates/put-template.txt.j2') }}"
+      register: result
+
+    - assert:
+        that:
+          - result is changed
+
+    - name: fetch template content
+      aws_s3:
+        bucket: "{{ bucket_name }}"
+        mode: getstr
+        object: put-template.txt
+      register: result
+
+    - assert:
+        that:
+          - result.contents == "{{ lookup('template', 'templates/put-template.txt.j2') }}"
+
+    # at present, there is no lookup that can process binary data, so we use slurp instead
+    - slurp:
+        src: "{{ role_path }}/files/test.png"
+      register: put_binary
+
+    - name: create an object from binary data
+      aws_s3:
+        bucket: "{{ bucket_name }}"
+        object: put-binary.bin
+        mode: put
+        content_base64: "{{ put_binary.content }}"
+      register: result
+
+    - assert:
+        that:
+          - result is changed
+
+    - name: fetch binary content
+      aws_s3:
+        bucket: "{{ bucket_name }}"
+        mode: get
+        dest: "{{ tmpdir.path }}/download_binary.bin"
+        object: put-binary.bin
+      register: result
+
+    - name: stat the files so we can compare the checksums
+      stat:
+        path: "{{ item }}"
+        get_checksum: yes
+      loop:
+      - "{{ role_path }}/files/test.png"
+      - "{{ tmpdir.path }}/download_binary.bin"
+      register: binary_files
+
+    - assert:
+        that:
+          - binary_files.results[0].stat.checksum == binary_files.results[1].stat.checksum
+
   always:
     - name: remove uploaded files
       aws_s3:
@@ -537,6 +639,9 @@
         - delete.txt
         - delete_encrypt.txt
         - delete_encrypt_kms.txt
+        - put-content.txt
+        - put-template.txt
+        - put-binary.txt
       ignore_errors: yes
 
     - name: delete temporary files
@@ -550,3 +655,9 @@
         bucket: "{{ bucket_name }}"
         mode: delete
       ignore_errors: yes
+
+    - name: delete the dot bucket
+      aws_s3:
+        bucket: "{{ bucket_name + '.bucket' }}"
+        mode: delete
+      ignore_errors: yes

tests/integration/targets/aws_s3/templates/put-template.txt.j2

@@ -0,0 +1,2 @@
+template:
+{{ put_template_text }}