diff --git a/changelogs/fragments/335-aws_s3-tagging-object-feature.yaml b/changelogs/fragments/335-aws_s3-tagging-object-feature.yaml new file mode 100644 index 00000000000..c7dfca64fae --- /dev/null +++ b/changelogs/fragments/335-aws_s3-tagging-object-feature.yaml @@ -0,0 +1,2 @@ +minor_changes: +- aws_s3 - add ``tags`` and ``purge_tags`` features for an S3 object (https://github.com/ansible-collections/amazon.aws/pull/335) diff --git a/plugins/modules/aws_s3.py b/plugins/modules/aws_s3.py index f7628ae9ad1..4b0d340e55f 100644 --- a/plugins/modules/aws_s3.py +++ b/plugins/modules/aws_s3.py @@ -155,9 +155,23 @@ description: - KMS key id to use when encrypting objects using I(encrypting=aws:kms). Ignored if I(encryption) is not C(aws:kms). type: str + tags: + description: + - Tags dict to apply to the S3 object. + type: dict + version_added: 2.0.0 + purge_tags: + description: + - Whether or not to remove tags assigned to the S3 object if not specified in the playbook. + - To remove all tags set I(tags) to an empty dictionary in conjunction with this. + type: bool + default: True + version_added: 2.0.0 +requirements: [ "boto3", "botocore" ] author: - "Lester Wade (@lwade)" - "Sloane Hertel (@s-hertel)" + - "Alina Buzachis (@linabuzachis)" extends_documentation_fragment: - amazon.aws.aws - amazon.aws.ec2 @@ -302,6 +316,7 @@ import io from ssl import SSLError import base64 +import time try: import botocore @@ -318,6 +333,8 @@ from ..module_utils.ec2 import AWSRetry from ..module_utils.ec2 import boto3_conn from ..module_utils.ec2 import get_aws_connection_info +from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict from ..module_utils.s3 import HAS_MD5 from ..module_utils.s3 import calculate_etag from ..module_utils.s3 import calculate_etag_content @@ -468,7 +485,7 @@ def delete_key(module, s3, bucket, obj): module.fail_json_aws(e, msg="Failed while trying to delete %s." % obj) -def create_dirkey(module, s3, bucket, obj, encrypt): +def create_dirkey(module, s3, bucket, obj, encrypt, expiry): if module.check_mode: module.exit_json(msg="PUT operation skipped - running in check mode", changed=True) try: @@ -485,7 +502,20 @@ def create_dirkey(module, s3, bucket, obj, encrypt): module.warn("PutObjectAcl is not implemented by your storage provider. Set the permissions parameters to the empty list to avoid this warning") except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Failed while creating object %s." % obj) - module.exit_json(msg="Virtual directory %s created in bucket %s" % (obj, bucket), changed=True) + + # Tags + tags, changed = ensure_tags(s3, module, bucket, obj) + + try: + url = s3.generate_presigned_url(ClientMethod='put_object', + Params={'Bucket': bucket, 'Key': obj}, + ExpiresIn=expiry) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Unable to generate presigned URL") + + url = put_download_url(module, s3, bucket, obj, expiry) + + module.exit_json(msg="Virtual directory %s created in bucket %s" % (obj, bucket), url=url, tags=tags, changed=True) def path_check(path): @@ -560,13 +590,13 @@ def upload_s3file(module, s3, bucket, obj, expiry, metadata, encrypt, headers, s module.warn("PutObjectAcl is not implemented by your storage provider. Set the permission parameters to the empty list to avoid this warning") except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Unable to set object ACL") - try: - url = s3.generate_presigned_url(ClientMethod='put_object', - Params={'Bucket': bucket, 'Key': obj}, - ExpiresIn=expiry) - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg="Unable to generate presigned URL") - module.exit_json(msg="PUT operation complete", url=url, changed=True) + + # Tags + tags, changed = ensure_tags(s3, module, bucket, obj) + + url = put_download_url(module, s3, bucket, obj, expiry) + + module.exit_json(msg="PUT operation complete", url=url, tags=tags, changed=True) def download_s3file(module, s3, bucket, obj, dest, retries, version=None): @@ -624,16 +654,26 @@ def download_s3str(module, s3, bucket, obj, version=None, validate=True): module.fail_json_aws(e, msg="Failed while getting contents of object %s as a string." % obj) -def get_download_url(module, s3, bucket, obj, expiry, changed=True): +def get_download_url(module, s3, bucket, obj, expiry, tags=None, changed=True): try: url = s3.generate_presigned_url(ClientMethod='get_object', Params={'Bucket': bucket, 'Key': obj}, ExpiresIn=expiry) - module.exit_json(msg="Download url:", url=url, expiry=expiry, changed=changed) + module.exit_json(msg="Download url:", url=url, tags=tags, expiry=expiry, changed=changed) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg="Failed while getting download url.") +def put_download_url(module, s3, bucket, obj, expiry): + try: + url = s3.generate_presigned_url(ClientMethod='put_object', + Params={'Bucket': bucket, 'Key': obj}, + ExpiresIn=expiry) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Unable to generate presigned URL") + return url + + def is_fakes3(s3_url): """ Return True if s3_url has scheme fakes3:// """ if s3_url is not None: @@ -675,6 +715,80 @@ def get_s3_connection(module, aws_connect_kwargs, location, rgw, s3_url, sig_4=F return boto3_conn(**params) +def get_current_object_tags_dict(s3, bucket, obj, version=None): + try: + if version: + current_tags = s3.get_object_tagging(Bucket=bucket, Key=obj, VersionId=version).get('TagSet') + else: + current_tags = s3.get_object_tagging(Bucket=bucket, Key=obj).get('TagSet') + except is_boto3_error_code('NoSuchTagSet'): + return {} + except is_boto3_error_code('NoSuchTagSetError'): # pylint: disable=duplicate-except + return {} + + return boto3_tag_list_to_ansible_dict(current_tags) + + +@AWSRetry.jittered_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) +def put_object_tagging(s3, bucket, obj, tags): + s3.put_object_tagging(Bucket=bucket, Key=obj, Tagging={'TagSet': ansible_dict_to_boto3_tag_list(tags)}) + + +@AWSRetry.jittered_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) +def delete_object_tagging(s3, bucket, obj): + s3.delete_object_tagging(Bucket=bucket, Key=obj) + + +def wait_tags_are_applied(module, s3, bucket, obj, expected_tags_dict, version=None): + for dummy in range(0, 12): + try: + current_tags_dict = get_current_object_tags_dict(s3, bucket, obj, version) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Failed to get object tags.") + if current_tags_dict != expected_tags_dict: + time.sleep(5) + else: + return current_tags_dict + + module.fail_json(msg="Object tags failed to apply in the expected time.", + requested_tags=expected_tags_dict, live_tags=current_tags_dict) + + +def ensure_tags(client, module, bucket, obj): + tags = module.params.get("tags") + purge_tags = module.params.get("purge_tags") + changed = False + + try: + current_tags_dict = get_current_object_tags_dict(client, bucket, obj) + except is_boto3_error_code(IGNORE_S3_DROP_IN_EXCEPTIONS): + module.warn("GetObjectTagging is not implemented by your storage provider. Set the permission parameters to the empty list to avoid this warning.") + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except + module.fail_json_aws(e, msg="Failed to get object tags.") + else: + if tags is not None: + if not purge_tags: + # Ensure existing tags that aren't updated by desired tags remain + current_copy = current_tags_dict.copy() + current_copy.update(tags) + tags = current_copy + if current_tags_dict != tags: + if tags: + try: + put_object_tagging(client, bucket, obj, tags) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Failed to update object tags.") + else: + if purge_tags: + try: + delete_object_tagging(client, bucket, obj) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Failed to delete object tags.") + current_tags_dict = wait_tags_are_applied(module, client, bucket, obj, tags) + changed = True + return current_tags_dict, changed + + def main(): argument_spec = dict( bucket=dict(required=True), @@ -700,7 +814,9 @@ def main(): content=dict(), content_base64=dict(), ignore_nonexistent_bucket=dict(default=False, type='bool'), - encryption_kms_key_id=dict() + encryption_kms_key_id=dict(), + tags=dict(type='dict'), + purge_tags=dict(type='bool', default=True), ) module = AnsibleAWSModule( argument_spec=argument_spec, @@ -831,9 +947,9 @@ def main(): # these were separated into the variables bucket_acl and object_acl above if content is None and content_base64 is None and src is None: - module.fail_json('Either content, content_base64 or src must be specified for PUT operations') + module.fail_json(msg='Either content, content_base64 or src must be specified for PUT operations') if src is not None and not path_check(src): - module.fail_json('Local object "%s" does not exist for PUT operation' % (src)) + module.fail_json(msg='Local object "%s" does not exist for PUT operation' % (src)) keyrtn = None if bucketrtn: @@ -853,8 +969,9 @@ def main(): if keyrtn and overwrite != 'always': if overwrite == 'never' or etag_compare(module, s3, bucket, obj, version=version, local_file=src, content=bincontent): - # Return the download URL for the existing object - get_download_url(module, s3, bucket, obj, expiry, changed=False) + # Return the download URL for the existing object and ensure tags are updated + tags, tags_update = ensure_tags(s3, module, bucket, obj) + get_download_url(module, s3, bucket, obj, expiry, tags, changed=tags_update) # only use valid object acls for the upload_s3file function module.params['permission'] = object_acl @@ -915,14 +1032,14 @@ def main(): else: # setting valid object acls for the create_dirkey function module.params['permission'] = object_acl - create_dirkey(module, s3, bucket, dirobj, encrypt) + create_dirkey(module, s3, bucket, dirobj, encrypt, expiry) else: # only use valid bucket acls for the create_bucket function module.params['permission'] = bucket_acl created = create_bucket(module, s3, bucket, location) # only use valid object acls for the create_dirkey function module.params['permission'] = object_acl - create_dirkey(module, s3, bucket, dirobj, encrypt) + create_dirkey(module, s3, bucket, dirobj, encrypt, expiry) # Support for grabbing the time-expired URL for an object in S3/Walrus. if mode == 'geturl': @@ -931,7 +1048,8 @@ def main(): keyrtn = key_check(module, s3, bucket, obj, version=version, validate=validate) if keyrtn: - get_download_url(module, s3, bucket, obj, expiry) + tags = get_current_object_tags_dict(s3, bucket, obj, version=version) + get_download_url(module, s3, bucket, obj, expiry, tags) else: module.fail_json(msg="Key %s does not exist." % obj) diff --git a/tests/integration/targets/aws_s3/tasks/main.yml b/tests/integration/targets/aws_s3/tasks/main.yml index 968116f014c..14344d63fd8 100644 --- a/tests/integration/targets/aws_s3/tasks/main.yml +++ b/tests/integration/targets/aws_s3/tasks/main.yml @@ -687,7 +687,253 @@ - assert: that: - binary_files.results[0].stat.checksum == binary_files.results[1].stat.checksum + + # ============================================================ + - name: 'Run tagging tests' + block: + # ============================================================ + + - name: create an object from static content + aws_s3: + bucket: "{{ bucket_name }}" + object: put-content.txt + mode: put + content: >- + test content + tags: + tag_one: '{{ resource_prefix }} One' + "Tag Two": 'two {{ resource_prefix }}' + register: result + + - assert: + that: + - result is changed + - "'tags' in result" + - (result.tags | length) == 2 + - result.tags["tag_one"] == '{{ resource_prefix }} One' + - result.tags["Tag Two"] == 'two {{ resource_prefix }}' + + - name: ensure idempotency on static content + aws_s3: + bucket: "{{ bucket_name }}" + object: put-content.txt + mode: put + overwrite: different + content: >- + test content + tags: + tag_one: '{{ resource_prefix }} One' + "Tag Two": 'two {{ resource_prefix }}' + register: result + + - assert: + that: + - result is not changed + - "'tags' in result" + - (result.tags | length) == 2 + - result.tags["tag_one"] == '{{ resource_prefix }} One' + - result.tags["Tag Two"] == 'two {{ resource_prefix }}' + + - name: Remove a tag from an S3 object + aws_s3: + bucket: "{{ bucket_name }}" + object: put-content.txt + mode: put + content: >- + test content + tags: + tag_one: '{{ resource_prefix }} One' + register: result + + - assert: + that: + - result is changed + - "'tags' in result" + - (result.tags | length) == 1 + - result.tags["tag_one"] == "{{ resource_prefix }} One" + - "'Tag Two' not in result.tags" + + - name: Remove the tag from an S3 object (idempotency) + aws_s3: + bucket: "{{ bucket_name }}" + object: put-content.txt + mode: put + content: >- + test content + overwrite: different + tags: + tag_one: '{{ resource_prefix }} One' + register: result + + - assert: + that: + - result is not changed + - "'tags' in result" + - (result.tags | length) == 1 + - result.tags["tag_one"] == "{{ resource_prefix }} One" + - "'Tag Two' not in result.tags" + + - name: Add a tag for an S3 object with purge_tags False + aws_s3: + bucket: "{{ bucket_name }}" + object: put-content.txt + mode: put + overwrite: different + content: >- + test content + tags: + tag_three: '{{ resource_prefix }} Three' + purge_tags: no + register: result + + - assert: + that: + - result is changed + - "'tags' in result" + - (result.tags | length) == 2 + - result.tags["tag_three"] == '{{ resource_prefix }} Three' + - result.tags["tag_one"] == '{{ resource_prefix }} One' + + - name: Add a tag for an S3 object with purge_tags False (idempotency) + aws_s3: + bucket: "{{ bucket_name }}" + object: put-content.txt + mode: put + overwrite: different + content: >- + test content + tags: + tag_three: '{{ resource_prefix }} Three' + purge_tags: no + register: result + + - assert: + that: + - result is not changed + - "'tags' in result" + - (result.tags | length) == 2 + - result.tags["tag_three"] == '{{ resource_prefix }} Three' + - result.tags["tag_one"] == '{{ resource_prefix }} One' + + - name: Update tags for an S3 object with purge_tags False + aws_s3: + bucket: "{{ bucket_name }}" + object: put-content.txt + mode: put + overwrite: different + content: >- + test content + tags: + "TagFour": '{{ resource_prefix }} tag_four' + purge_tags: no + register: result + + - assert: + that: + - result is changed + - "'tags' in result" + - (result.tags | length) == 3 + - result.tags["tag_one"] == '{{ resource_prefix }} One' + - result.tags["tag_three"] == '{{ resource_prefix }} Three' + - result.tags["TagFour"] == '{{ resource_prefix }} tag_four' + + - name: Update tags for an S3 object with purge_tags False (idempotency) + aws_s3: + bucket: "{{ bucket_name }}" + object: put-content.txt + mode: put + overwrite: different + content: >- + test content + tags: + "TagFour": '{{ resource_prefix }} tag_four' + purge_tags: no + register: result + + - assert: + that: + - result is not changed + - "'tags' in result" + - (result.tags | length) == 3 + - result.tags["tag_one"] == '{{ resource_prefix }} One' + - result.tags["tag_three"] == '{{ resource_prefix }} Three' + - result.tags["TagFour"] == '{{ resource_prefix }} tag_four' + + - name: Specify empty tags for an S3 object with purge_tags False + aws_s3: + bucket: "{{ bucket_name }}" + object: put-content.txt + mode: put + overwrite: different + content: >- + test content + tags: {} + purge_tags: no + register: result + + - assert: + that: + - result is not changed + - "'tags' in result" + - (result.tags | length) == 3 + - result.tags["tag_one"] == '{{ resource_prefix }} One' + - result.tags["tag_three"] == '{{ resource_prefix }} Three' + - result.tags["TagFour"] == '{{ resource_prefix }} tag_four' + + - name: Do not specify any tag to ensure previous tags are not removed + aws_s3: + bucket: "{{ bucket_name }}" + object: put-content.txt + mode: put + overwrite: different + content: >- + test content + register: result + + - assert: + that: + - result is not changed + - "'tags' in result" + - (result.tags | length) == 3 + - result.tags["tag_one"] == '{{ resource_prefix }} One' + - result.tags["tag_three"] == '{{ resource_prefix }} Three' + - result.tags["TagFour"] == '{{ resource_prefix }} tag_four' + + - name: Remove all tags + aws_s3: + bucket: "{{ bucket_name }}" + object: put-content.txt + mode: put + overwrite: different + content: >- + test content + tags: {} + register: result + + - assert: + that: + - result is changed + - "'tags' in result" + - (result.tags | length) == 0 + + - name: Remove all tags (idempotency) + aws_s3: + bucket: "{{ bucket_name }}" + object: put-content.txt + mode: put + overwrite: different + content: >- + test content + tags: {} + register: result + + - assert: + that: + - result is not changed + - "'tags' in result" + - (result.tags | length) == 0 + always: - name: remove uploaded files aws_s3: