From 5da4a559d827e4cfdcfab40e8c80a0f0ab333b53 Mon Sep 17 00:00:00 2001 From: Alexander Braverman Masis Date: Fri, 28 Jun 2024 09:17:35 -0400 Subject: [PATCH] s3_bucket transfer accleration (#2046) Add support for S3 Bucket Transfer Acceleration: https://docs.aws.amazon.com/AmazonS3/latest/userguide/transfer-acceleration.html Reviewed-by: Helen Bailey Reviewed-by: Alina Buzachis Reviewed-by: Mike Graves (cherry picked from commit 3a6e448a93a24c474d281aa49492a3903b662ef7) --- .../fragments/s3_bucket-accelerate_option.yml | 2 + plugins/modules/s3_bucket.py | 109 ++++++++- tests/integration/targets/s3_bucket/inventory | 1 + .../roles/s3_bucket/tasks/accelerate.yml | 215 ++++++++++++++++++ 4 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 changelogs/fragments/s3_bucket-accelerate_option.yml create mode 100644 tests/integration/targets/s3_bucket/roles/s3_bucket/tasks/accelerate.yml diff --git a/changelogs/fragments/s3_bucket-accelerate_option.yml b/changelogs/fragments/s3_bucket-accelerate_option.yml new file mode 100644 index 00000000000..b0c4d3b0187 --- /dev/null +++ b/changelogs/fragments/s3_bucket-accelerate_option.yml @@ -0,0 +1,2 @@ +minor_changes: + - s3_bucket - Add support for enabling Amazon S3 Transfer Acceleration by setting the ``accelerate_enabled`` option (https://github.com/ansible-collections/amazon.aws/pull/2046). diff --git a/plugins/modules/s3_bucket.py b/plugins/modules/s3_bucket.py index d259286f9c0..a3defc81dc4 100644 --- a/plugins/modules/s3_bucket.py +++ b/plugins/modules/s3_bucket.py @@ -166,6 +166,12 @@ type: bool default: false version_added: 6.0.0 + accelerate_enabled: + description: + - Enables Amazon S3 Transfer Acceleration, sent data will be routed to Amazon S3 over an optimized network path. + type: bool + default: false + version_added: 8.1.0 extends_documentation_fragment: - amazon.aws.common.modules @@ -286,6 +292,12 @@ name: mys3bucket state: present acl: public-read + +# Enable transfer acceleration +- amazon.aws.s3_bucket: + name: mys3bucket + state: present + accelerate_enabled: true """ RETURN = r""" @@ -346,8 +358,13 @@ acl: description: S3 bucket's canned ACL. type: dict - returned: I(state=present) + returned: I(state=present). sample: 'public-read' +accelerate_enabled: + description: S3 bucket acceleration status. + type: bool + returned: O(state=present) + sample: true """ import json @@ -832,6 +849,49 @@ def handle_bucket_object_lock(s3_client, module: AnsibleAWSModule, name: str) -> return object_lock_result +def handle_bucket_accelerate(s3_client, module: AnsibleAWSModule, name: str) -> tuple[bool, bool]: + """ + Manage transfer accelerate for an S3 bucket. + Parameters: + s3_client (boto3.client): The Boto3 S3 client object. + module (AnsibleAWSModule): The Ansible module object. + name (str): The name of the bucket to handle transfer accelerate for. + Returns: + A tuple containing a boolean indicating whether transfer accelerate setting was changed + and a boolean indicating the transfer accelerate status. + """ + accelerate_enabled = module.params.get("accelerate_enabled") + accelerate_enabled_result = False + accelerate_enabled_changed = False + try: + accelerate_status = get_bucket_accelerate_status(s3_client, name) + accelerate_enabled_result = accelerate_status + except is_boto3_error_code(["NotImplemented", "XNotImplemented"]) as e: + if accelerate_enabled is not None: + module.fail_json_aws(e, msg="Fetching bucket transfer acceleration state is not supported") + except is_boto3_error_code("AccessDenied") as e: # pylint: disable=duplicate-except + if accelerate_enabled is not None: + module.fail_json_aws(e, msg="Permission denied fetching transfer acceleration for bucket") + except ( + botocore.exceptions.BotoCoreError, + botocore.exceptions.ClientError, + ) as e: # pylint: disable=duplicate-except + module.fail_json_aws(e, msg="Failed to fetch bucket transfer acceleration state") + else: + try: + if not accelerate_enabled and accelerate_status: + delete_bucket_accelerate_configuration(s3_client, name) + accelerate_enabled_changed = True + accelerate_enabled_result = False + if accelerate_enabled and not accelerate_status: + put_bucket_accelerate_configuration(s3_client, name) + accelerate_enabled_changed = True + accelerate_enabled_result = True + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: + module.fail_json_aws(e, msg="Failed to update bucket transfer acceleration") + return accelerate_enabled_changed, accelerate_enabled_result + + def create_or_update_bucket(s3_client, module: AnsibleAWSModule): """ Create or update an S3 bucket along with its associated configurations. @@ -908,6 +968,10 @@ def create_or_update_bucket(s3_client, module: AnsibleAWSModule): bucket_object_lock_result = handle_bucket_object_lock(s3_client, module, name) result["object_lock_enabled"] = bucket_object_lock_result + # -- Transfer Acceleration + bucket_accelerate_changed, bucket_accelerate_result = handle_bucket_accelerate(s3_client, module, name) + result["accelerate_enabled"] = bucket_accelerate_result + # Module exit changed = ( changed @@ -919,6 +983,7 @@ def create_or_update_bucket(s3_client, module: AnsibleAWSModule): or encryption_changed or bucket_ownership_changed or bucket_acl_changed + or bucket_accelerate_changed ) module.exit_json(changed=changed, name=name, **result) @@ -973,6 +1038,47 @@ def create_bucket(s3_client, bucket_name: str, location: str, object_lock_enable return False +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=["NoSuchBucket", "OperationAborted"]) +def put_bucket_accelerate_configuration(s3_client, bucket_name): + """ + Enable transfer accelerate for the S3 bucket. + Parameters: + s3_client (boto3.client): The Boto3 S3 client object. + bucket_name (str): The name of the S3 bucket. + Returns: + None + """ + s3_client.put_bucket_accelerate_configuration(Bucket=bucket_name, AccelerateConfiguration={"Status": "Enabled"}) + + +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=["NoSuchBucket", "OperationAborted"]) +def delete_bucket_accelerate_configuration(s3_client, bucket_name): + """ + Disable transfer accelerate for the S3 bucket. + Parameters: + s3_client (boto3.client): The Boto3 S3 client object. + bucket_name (str): The name of the S3 bucket. + Returns: + None + """ + + s3_client.put_bucket_accelerate_configuration(Bucket=bucket_name, AccelerateConfiguration={"Status": "Suspended"}) + + +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=["NoSuchBucket", "OperationAborted"]) +def get_bucket_accelerate_status(s3_client, bucket_name) -> bool: + """ + Get transfer accelerate status of the S3 bucket. + Parameters: + s3_client (boto3.client): The Boto3 S3 client object. + bucket_name (str): The name of the S3 bucket. + Returns: + Transfer accelerate status of the S3 bucket. + """ + accelerate_configuration = s3_client.get_bucket_accelerate_configuration(Bucket=bucket_name) + return accelerate_configuration.get("Status") == "Enabled" + + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=["NoSuchBucket", "OperationAborted"]) def put_bucket_tagging(s3_client, bucket_name: str, tags: dict): """ @@ -1732,6 +1838,7 @@ def main(): acl=dict(type="str", choices=["private", "public-read", "public-read-write", "authenticated-read"]), validate_bucket_name=dict(type="bool", default=True), dualstack=dict(default=False, type="bool"), + accelerate_enabled=dict(default=False, type="bool"), object_lock_enabled=dict(type="bool"), ) diff --git a/tests/integration/targets/s3_bucket/inventory b/tests/integration/targets/s3_bucket/inventory index db31e4a9b37..945118fc2ad 100644 --- a/tests/integration/targets/s3_bucket/inventory +++ b/tests/integration/targets/s3_bucket/inventory @@ -11,6 +11,7 @@ encryption_sse public_access acl object_lock +accelerate [all:vars] ansible_connection=local diff --git a/tests/integration/targets/s3_bucket/roles/s3_bucket/tasks/accelerate.yml b/tests/integration/targets/s3_bucket/roles/s3_bucket/tasks/accelerate.yml new file mode 100644 index 00000000000..f4def35f60c --- /dev/null +++ b/tests/integration/targets/s3_bucket/roles/s3_bucket/tasks/accelerate.yml @@ -0,0 +1,215 @@ +--- +- module_defaults: + group/aws: + access_key: "{{ aws_access_key }}" + secret_key: "{{ aws_secret_key }}" + session_token: "{{ security_token | default(omit) }}" + region: "{{ aws_region }}" + block: + - ansible.builtin.set_fact: + local_bucket_name: "{{ bucket_name | hash('md5')}}-accelerate" + + # ============================================================ + + - name: Create a simple bucket + amazon.aws.s3_bucket: + name: "{{ local_bucket_name }}" + state: present + register: output + + - ansible.builtin.assert: + that: + - output.changed + - not output.accelerate_enabled + + - name: Re-disable transfer acceleration (idempotency) + amazon.aws.s3_bucket: + name: "{{ local_bucket_name }}" + state: present + accelerate_enabled: false + register: output + + - ansible.builtin.assert: + that: + - not output.changed + - not output.accelerate_enabled + + - name: Enable transfer acceleration + amazon.aws.s3_bucket: + name: "{{ local_bucket_name }}" + state: present + accelerate_enabled: true + register: output + ignore_errors: false + + - ansible.builtin.assert: + that: + - output.changed + - output.accelerate_enabled + + - name: Assert transfer acceleration enabled + amazon.aws.s3_bucket_info: + name: "{{ local_bucket_name }}" + bucket_facts: + bucket_accelerate_configuration: true + register: output + + - ansible.builtin.assert: + that: + - item.bucket_accelerate_configuration["Status"] == "Enabled" + loop: "{{ output.buckets }}" + loop_control: + label: "{{ item.name }}" + + - name: Re-Enable transfer acceleration (idempotency) + amazon.aws.s3_bucket: + name: "{{ local_bucket_name }}" + state: present + accelerate_enabled: true + register: output + + - ansible.builtin.assert: + that: + - not output.changed + - output.accelerate_enabled + + - name: Assert transfer acceleration enabled + amazon.aws.s3_bucket_info: + name: "{{ local_bucket_name }}" + bucket_facts: + bucket_accelerate_configuration: true + register: output + + - ansible.builtin.assert: + that: + - item.bucket_accelerate_configuration["Status"] == "Enabled" + loop: "{{ output.buckets }}" + loop_control: + label: "{{ item.name }}" + + - name: Delete test s3 bucket + amazon.aws.s3_bucket: + name: "{{ local_bucket_name }}" + state: absent + register: output + + - ansible.builtin.assert: + that: + - output.changed + + # ============================================================ + + - name: Create a bucket with transfer accelerate enabled + amazon.aws.s3_bucket: + name: "{{ local_bucket_name }}-2" + state: present + accelerate_enabled: true + register: output + + - ansible.builtin.assert: + that: + - output.changed + - output.accelerate_enabled + + - name: Assert transfer acceleration enabled + amazon.aws.s3_bucket_info: + name: "{{ local_bucket_name }}" + bucket_facts: + bucket_accelerate_configuration: true + register: output + + - ansible.builtin.assert: + that: + - item.bucket_accelerate_configuration["Status"] == "Enabled" + loop: "{{ output.buckets }}" + loop_control: + label: "{{ item.name }}" + + - name: Disable transfer accelerate + amazon.aws.s3_bucket: + name: "{{ local_bucket_name }}-2" + state: present + accelerate_enabled: false + register: output + ignore_errors: false + + - ansible.builtin.assert: + that: + - output.changed + - not output.accelerate_enabled + + - name: Assert transfer acceleration disabled + amazon.aws.s3_bucket_info: + name: "{{ local_bucket_name }}" + bucket_facts: + bucket_accelerate_configuration: true + register: output + + - ansible.builtin.assert: + that: + - item.bucket_accelerate_configuration["Status"] == "Disabled" + loop: "{{ output.buckets }}" + loop_control: + label: "{{ item.name }}" + + - name: Re-Enable transfer accelerate (idempotency) + amazon.aws.s3_bucket: + name: "{{ local_bucket_name }}-2" + state: present + accelerate_enabled: true + register: output + + - ansible.builtin.assert: + that: + - output.changed + - output.accelerate_enabled + + - name: Assert transfer acceleration enabled + amazon.aws.s3_bucket_info: + name: "{{ local_bucket_name }}" + bucket_facts: + bucket_accelerate_configuration: true + register: output + + - ansible.builtin.assert: + that: + - item.bucket_accelerate_configuration["Status"] == "Enabled" + loop: "{{ output.buckets }}" + loop_control: + label: "{{ item.name }}" + + - name: Touch bucket with transfer accelerate enabled (idempotency) + amazon.aws.s3_bucket: + name: "{{ local_bucket_name }}-2" + state: present + accelerate_enabled: true + register: output + + - ansible.builtin.assert: + that: + - not output.changed + - output.accelerate_enabled + + - name: Delete test s3 bucket + amazon.aws.s3_bucket: + name: "{{ local_bucket_name }}-2" + state: absent + register: output + + - ansible.builtin.assert: + that: + - output.changed + + # ============================================================ + always: + - name: Ensure all buckets are deleted + amazon.aws.s3_bucket: + name: "{{ local_bucket_name }}" + state: absent + ignore_errors: true + + - name: Ensure all buckets are deleted + amazon.aws.s3_bucket: + name: "{{ local_bucket_name }}-2" + state: absent + ignore_errors: true