From 9c0967cb2ed8c8f4529395cd21e25aaa51bcffea Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Thu, 25 Mar 2021 15:50:16 +0100 Subject: [PATCH] Stability - s3_lifecycle (#448) * Add aws retry * re-enable lifecycle tests * changelog * Rename S3 bucket used by tests * Cleanup docs * Use datetime to be a little more flexible with the date formats * Rework and add support for waiting for changes to propagate * Wait for changes to propagate in tests s3 lifecycle events are especially susceptible to eventual consistency * Test examples are also idempotent... * More changelogs * Catch Amazon's "Nothing Changed" error (can happen when just trying to enable/disable a rule) --- .../fragments/448-s3_lifecycle-stability.yml | 4 + plugins/modules/s3_lifecycle.py | 303 ++++++++++++------ .../integration/targets/s3_lifecycle/aliases | 5 +- .../targets/s3_lifecycle/defaults/main.yml | 1 + .../targets/s3_lifecycle/tasks/main.yml | 213 +++++++++--- 5 files changed, 377 insertions(+), 149 deletions(-) create mode 100644 changelogs/fragments/448-s3_lifecycle-stability.yml create mode 100644 tests/integration/targets/s3_lifecycle/defaults/main.yml diff --git a/changelogs/fragments/448-s3_lifecycle-stability.yml b/changelogs/fragments/448-s3_lifecycle-stability.yml new file mode 100644 index 00000000000..972a91eb813 --- /dev/null +++ b/changelogs/fragments/448-s3_lifecycle-stability.yml @@ -0,0 +1,4 @@ +minor_changes: +- s3_lifecycle - Add retries on common AWS failures (https://github.com/ansible-collections/community.aws/pull/448). +- s3_lifecycle - Fix idempotency when using dates instead of days (https://github.com/ansible-collections/community.aws/pull/448). +- s3_lifecycle - Add a ``wait`` parameter to wait for changes to propagate after being set (https://github.com/ansible-collections/community.aws/pull/448). diff --git a/plugins/modules/s3_lifecycle.py b/plugins/modules/s3_lifecycle.py index 0bc4a328680..9cec1402eb1 100644 --- a/plugins/modules/s3_lifecycle.py +++ b/plugins/modules/s3_lifecycle.py @@ -10,105 +10,110 @@ --- module: s3_lifecycle version_added: 1.0.0 -short_description: Manage s3 bucket lifecycle rules in AWS +short_description: Manage S3 bucket lifecycle rules in AWS description: - - Manage s3 bucket lifecycle rules in AWS + - Manage S3 bucket lifecycle rules in AWS. author: "Rob White (@wimnat)" notes: - - If specifying expiration time as days then transition time must also be specified in days - - If specifying expiration time as a date then transition time must also be specified as a date + - If specifying expiration time as days then transition time must also be specified in days. + - If specifying expiration time as a date then transition time must also be specified as a date. options: name: description: - - "Name of the s3 bucket" + - Name of the S3 bucket. required: true type: str expiration_date: description: - - > - Indicates the lifetime of the objects that are subject to the rule by the date they will expire. The value must be ISO-8601 format, the time must - be midnight and a GMT timezone must be specified. + - Indicates the lifetime of the objects that are subject to the rule by the date they will expire. + - The value must be ISO-8601 format, the time must be midnight and a GMT timezone must be specified. type: str expiration_days: description: - - "Indicates the lifetime, in days, of the objects that are subject to the rule. The value must be a non-zero positive integer." + - Indicates the lifetime, in days, of the objects that are subject to the rule. + - The value must be a non-zero positive integer. type: int prefix: description: - - "Prefix identifying one or more objects to which the rule applies. If no prefix is specified, the rule will apply to the whole bucket." + - Prefix identifying one or more objects to which the rule applies. + - If no prefix is specified, the rule will apply to the whole bucket. type: str purge_transitions: description: - - > - "Whether to replace all the current transition(s) with the new transition(s). When false, the provided transition(s) - will be added, replacing transitions with the same storage_class. When true, existing transitions will be removed and - replaced with the new transition(s) + - Whether to replace all the current transition(s) with the new transition(s). + - When C(false), the provided transition(s) will be added, replacing transitions + with the same storage_class. When true, existing transitions will be removed + and replaced with the new transition(s) default: true type: bool noncurrent_version_expiration_days: description: - - 'Delete noncurrent versions this many days after they become noncurrent' + - The number of days after which non-current versions should be deleted. required: false type: int noncurrent_version_storage_class: description: - - 'Transition noncurrent versions to this storage class' + - The storage class to which non-current versions are transitioned. default: glacier choices: ['glacier', 'onezone_ia', 'standard_ia', 'intelligent_tiering', 'deep_archive'] required: false type: str noncurrent_version_transition_days: description: - - 'Transition noncurrent versions this many days after they become noncurrent' + - The number of days after which non-current versions will be transitioned + to the storage class specified in I(noncurrent_version_storage_class). required: false type: int noncurrent_version_transitions: description: - - > - A list of transition behaviors to be applied to noncurrent versions for the rule. Each storage class may be used only once. Each transition - behavior contains these elements + - A list of transition behaviors to be applied to noncurrent versions for the rule. + - Each storage class may be used only once. Each transition behavior contains these elements I(transition_days) I(storage_class) type: list elements: dict rule_id: description: - - "Unique identifier for the rule. The value cannot be longer than 255 characters. A unique value for the rule will be generated if no value is provided." + - Unique identifier for the rule. + - The value cannot be longer than 255 characters. + - A unique value for the rule will be generated if no value is provided. type: str state: description: - - "Create or remove the lifecycle rule" + - Create or remove the lifecycle rule. default: present choices: [ 'present', 'absent' ] type: str status: description: - - "If 'enabled', the rule is currently being applied. If 'disabled', the rule is not currently being applied." + - If C(enabled), the rule is currently being applied. + - If C(disabled), the rule is not currently being applied. default: enabled choices: [ 'enabled', 'disabled' ] type: str storage_class: description: - - "The storage class to transition to." - - "The 'standard_ia' class is only being available from Ansible version 2.2." + - The storage class to transition to. default: glacier choices: [ 'glacier', 'onezone_ia', 'standard_ia', 'intelligent_tiering', 'deep_archive'] type: str transition_date: description: - - > - Indicates the lifetime of the objects that are subject to the rule by the date they will transition to a different storage class. - The value must be ISO-8601 format, the time must be midnight and a GMT timezone must be specified. If transition_days is not specified, - this parameter is required." + - Indicates the lifetime of the objects that are subject to the rule by the date they + will transition to a different storage class. + - The value must be ISO-8601 format, the time must be midnight and a GMT timezone must + be specified. + - If (transition_days) is not specified, this parameter is required. type: str transition_days: description: - - "Indicates when, in days, an object transitions to a different storage class. If transition_date is not specified, this parameter is required." + - Indicates when, in days, an object transitions to a different storage class. + - If I(transition_date) is not specified, this parameter is required. type: int transitions: description: - - A list of transition behaviors to be applied to the rule. Each storage class may be used only once. Each transition - behavior may contain these elements + - A list of transition behaviors to be applied to the rule. + - Each storage class may be used only once. Each transition behavior may contain these elements I(transition_days) I(transition_date) I(storage_class) @@ -118,6 +123,12 @@ description: - The I(requester_pays) option does nothing and will be removed after 2022-06-01 type: bool + wait: + description: + - Wait for the configuration to complete before returning. + version_added: 1.5.0 + type: bool + default: no extends_documentation_fragment: - amazon.aws.aws - amazon.aws.ec2 @@ -192,6 +203,13 @@ from copy import deepcopy import datetime +import time + +try: + from dateutil import parser as date_parser + HAS_DATEUTIL = True +except ImportError: + HAS_DATEUTIL = False try: import botocore @@ -200,12 +218,39 @@ from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_code +from ansible_collections.amazon.aws.plugins.module_utils.core import is_boto3_error_message +from ansible_collections.amazon.aws.plugins.module_utils.core import normalize_boto3_result +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry -def create_lifecycle_rule(client, module): +def parse_date(date): + if date is None: + return None + try: + if HAS_DATEUTIL: + return date_parser.parse(date) + else: + # Very simplistic + return datetime.datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.000Z") + except ValueError: + return None + + +def fetch_rules(client, module, name): + # Get the bucket's current lifecycle rules + try: + current_lifecycle = client.get_bucket_lifecycle_configuration(aws_retry=True, Bucket=name) + current_lifecycle_rules = normalize_boto3_result(current_lifecycle['Rules']) + except is_boto3_error_code('NoSuchLifecycleConfiguration'): + current_lifecycle_rules = [] + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except + module.fail_json_aws(e) + return current_lifecycle_rules + +def build_rule(client, module): name = module.params.get("name") - expiration_date = module.params.get("expiration_date") + expiration_date = parse_date(module.params.get("expiration_date")) expiration_days = module.params.get("expiration_days") noncurrent_version_expiration_days = module.params.get("noncurrent_version_expiration_days") noncurrent_version_transition_days = module.params.get("noncurrent_version_transition_days") @@ -215,20 +260,10 @@ def create_lifecycle_rule(client, module): rule_id = module.params.get("rule_id") status = module.params.get("status") storage_class = module.params.get("storage_class") - transition_date = module.params.get("transition_date") + transition_date = parse_date(module.params.get("transition_date")) transition_days = module.params.get("transition_days") transitions = module.params.get("transitions") purge_transitions = module.params.get("purge_transitions") - changed = False - - # Get the bucket's current lifecycle rules - try: - current_lifecycle = client.get_bucket_lifecycle_configuration(Bucket=name) - current_lifecycle_rules = current_lifecycle['Rules'] - except is_boto3_error_code('NoSuchLifecycleConfiguration'): - current_lifecycle_rules = [] - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except - module.fail_json_aws(e) rule = dict(Filter=dict(Prefix=prefix), Status=status.title()) if rule_id is not None: @@ -237,7 +272,7 @@ def create_lifecycle_rule(client, module): if expiration_days is not None: rule['Expiration'] = dict(Days=expiration_days) elif expiration_date is not None: - rule['Expiration'] = dict(Date=expiration_date) + rule['Expiration'] = dict(Date=expiration_date.isoformat()) if noncurrent_version_expiration_days is not None: rule['NoncurrentVersionExpiration'] = dict(NoncurrentDays=noncurrent_version_expiration_days) @@ -246,7 +281,7 @@ def create_lifecycle_rule(client, module): rule['Transitions'] = [dict(Days=transition_days, StorageClass=storage_class.upper()), ] elif transition_date is not None: - rule['Transitions'] = [dict(Date=transition_date, StorageClass=storage_class.upper()), ] + rule['Transitions'] = [dict(Date=transition_date.isoformat(), StorageClass=storage_class.upper()), ] if transitions is not None: if not rule.get('Transitions'): @@ -275,8 +310,17 @@ def create_lifecycle_rule(client, module): t_out['StorageClass'] = noncurrent_version_transition['storage_class'].upper() rule['NoncurrentVersionTransitions'].append(t_out) + return rule + + +def compare_and_update_configuration(client, module, current_lifecycle_rules, rule): + purge_transitions = module.params.get("purge_transitions") + rule_id = module.params.get("rule_id") + lifecycle_configuration = dict(Rules=[]) + changed = False appended = False + # If current_lifecycle_obj is not None then we have rules to compare, otherwise just add the rule if current_lifecycle_rules: # If rule ID exists, use that for comparison otherwise compare based on prefix @@ -300,13 +344,7 @@ def create_lifecycle_rule(client, module): lifecycle_configuration['Rules'].append(rule) changed = True - # Write lifecycle to bucket - try: - client.put_bucket_lifecycle_configuration(Bucket=name, LifecycleConfiguration=lifecycle_configuration) - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e) - - module.exit_json(changed=changed) + return changed, lifecycle_configuration def update_or_append_rule(new_rule, existing_rule, purge_transitions, lifecycle_obj): @@ -334,11 +372,35 @@ def update_or_append_rule(new_rule, existing_rule, purge_transitions, lifecycle_ return changed, appended -def compare_rule(rule_a, rule_b, purge_transitions): +def compare_and_remove_rule(current_lifecycle_rules, rule_id=None, prefix=None): + changed = False + lifecycle_configuration = dict(Rules=[]) + + # Check if rule exists + # If an ID exists, use that otherwise compare based on prefix + if rule_id is not None: + for existing_rule in current_lifecycle_rules: + if rule_id == existing_rule['ID']: + # We're not keeping the rule (i.e. deleting) so mark as changed + changed = True + else: + lifecycle_configuration['Rules'].append(existing_rule) + else: + for existing_rule in current_lifecycle_rules: + if prefix == existing_rule['Filter']['Prefix']: + # We're not keeping the rule (i.e. deleting) so mark as changed + changed = True + else: + lifecycle_configuration['Rules'].append(existing_rule) + + return changed, lifecycle_configuration + + +def compare_rule(new_rule, old_rule, purge_transitions): # Copy objects - rule1 = deepcopy(rule_a) - rule2 = deepcopy(rule_b) + rule1 = deepcopy(new_rule) + rule2 = deepcopy(old_rule) if purge_transitions: return rule1 == rule2 @@ -359,7 +421,7 @@ def compare_rule(rule_a, rule_b, purge_transitions): def merge_transitions(updated_rule, updating_rule): - # because of the legal s3 transitions, we know only one can exist for each storage class. + # because of the legal S3 transitions, we know only one can exist for each storage class. # So, our strategy is build some dicts, keyed on storage class and add the storage class transitions that are only # in updating_rule to updated_rule updated_transitions = {} @@ -373,54 +435,90 @@ def merge_transitions(updated_rule, updating_rule): updated_rule['Transitions'].append(transition) +def create_lifecycle_rule(client, module): + + name = module.params.get("name") + wait = module.params.get("wait") + changed = False + + old_lifecycle_rules = fetch_rules(client, module, name) + new_rule = build_rule(client, module) + (changed, lifecycle_configuration) = compare_and_update_configuration(client, module, + old_lifecycle_rules, + new_rule) + + # Write lifecycle to bucket + try: + client.put_bucket_lifecycle_configuration( + aws_retry=True, + Bucket=name, + LifecycleConfiguration=lifecycle_configuration) + except is_boto3_error_message('At least one action needs to be specified in a rule'): + # Amazon interpretted this as not changing anything + changed = False + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except + module.fail_json_aws(e, lifecycle_configuration=lifecycle_configuration, name=name, old_lifecycle_rules=old_lifecycle_rules) + + _changed = changed + _retries = 10 + while wait and _changed and _retries: + # We've seen examples where get_bucket_lifecycle_configuration returns + # the updated rules, then the old rules, then the updated rules again, + time.sleep(5) + _retries -= 1 + new_rules = fetch_rules(client, module, name) + (_changed, lifecycle_configuration) = compare_and_update_configuration(client, module, + new_rules, + new_rule) + + new_rules = fetch_rules(client, module, name) + + module.exit_json(changed=changed, new_rule=new_rule, rules=new_rules, + old_rules=old_lifecycle_rules, _retries=_retries, + _config=lifecycle_configuration) + + def destroy_lifecycle_rule(client, module): name = module.params.get("name") prefix = module.params.get("prefix") rule_id = module.params.get("rule_id") + wait = module.params.get("wait") changed = False if prefix is None: prefix = "" - # Get the bucket's current lifecycle rules - try: - current_lifecycle_rules = client.get_bucket_lifecycle_configuration(Bucket=name)['Rules'] - except is_boto3_error_code('NoSuchLifecycleConfiguration'): - current_lifecycle_rules = [] - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: # pylint: disable=duplicate-except - module.fail_json_aws(e) - - # Create lifecycle - lifecycle_obj = dict(Rules=[]) - - # Check if rule exists - # If an ID exists, use that otherwise compare based on prefix - if rule_id is not None: - for existing_rule in current_lifecycle_rules: - if rule_id == existing_rule['ID']: - # We're not keeping the rule (i.e. deleting) so mark as changed - changed = True - else: - lifecycle_obj['Rules'].append(existing_rule) - else: - for existing_rule in current_lifecycle_rules: - if prefix == existing_rule['Filter']['Prefix']: - # We're not keeping the rule (i.e. deleting) so mark as changed - changed = True - else: - lifecycle_obj['Rules'].append(existing_rule) + current_lifecycle_rules = fetch_rules(client, module, name) + changed, lifecycle_obj = compare_and_remove_rule(current_lifecycle_rules, rule_id, prefix) # Write lifecycle to bucket or, if there no rules left, delete lifecycle configuration try: if lifecycle_obj['Rules']: - client.put_bucket_lifecycle_configuration(Bucket=name, LifecycleConfiguration=lifecycle_obj) + client.put_bucket_lifecycle_configuration( + aws_retry=True, + Bucket=name, + LifecycleConfiguration=lifecycle_obj) elif current_lifecycle_rules: changed = True - client.delete_bucket_lifecycle(Bucket=name) + client.delete_bucket_lifecycle(aws_retry=True, Bucket=name) except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e) - module.exit_json(changed=changed) + + _changed = changed + _retries = 10 + while wait and _changed and _retries: + # We've seen examples where get_bucket_lifecycle_configuration returns + # the updated rules, then the old rules, then the updated rules again, + time.sleep(5) + _retries -= 1 + new_rules = fetch_rules(client, module, name) + (_changed, lifecycle_configuration) = compare_and_remove_rule(new_rules, rule_id, prefix) + + new_rules = fetch_rules(client, module, name) + + module.exit_json(changed=changed, rules=new_rules, old_rules=current_lifecycle_rules, + _retries=_retries) def main(): @@ -442,7 +540,8 @@ def main(): transition_days=dict(type='int'), transition_date=dict(), transitions=dict(type='list', elements='dict'), - purge_transitions=dict(default='yes', type='bool') + purge_transitions=dict(default='yes', type='bool'), + wait=dict(type='bool', default=False) ) module = AnsibleAWSModule(argument_spec=argument_spec, @@ -456,7 +555,7 @@ def main(): ['noncurrent_version_transition_days', 'noncurrent_version_transitions'], ],) - client = module.client('s3') + client = module.client('s3', retry_decorator=AWSRetry.jittered_backoff()) expiration_date = module.params.get("expiration_date") transition_date = module.params.get("transition_date") @@ -474,18 +573,18 @@ def main(): else: msg = "one of the following is required when 'state' is 'present': %s" % ', '.join(required_when_present) module.fail_json(msg=msg) - # If expiration_date set, check string is valid - if expiration_date is not None: - try: - datetime.datetime.strptime(expiration_date, "%Y-%m-%dT%H:%M:%S.000Z") - except ValueError: - module.fail_json(msg="expiration_date is not a valid ISO-8601 format. The time must be midnight and a timezone of GMT must be included") - - if transition_date is not None: - try: - datetime.datetime.strptime(transition_date, "%Y-%m-%dT%H:%M:%S.000Z") - except ValueError: - module.fail_json(msg="expiration_date is not a valid ISO-8601 format. The time must be midnight and a timezone of GMT must be included") + + # If dates have been set, make sure they're in a valid format + if expiration_date: + expiration_date = parse_date(expiration_date) + if expiration_date is None: + module.fail_json(msg="expiration_date is not a valid ISO-8601 format." + " The time must be midnight and a timezone of GMT must be included") + if transition_date: + transition_date = parse_date(transition_date) + if transition_date is None: + module.fail_json(msg="transition_date is not a valid ISO-8601 format." + " The time must be midnight and a timezone of GMT must be included") if state == 'present': create_lifecycle_rule(client, module) diff --git a/tests/integration/targets/s3_lifecycle/aliases b/tests/integration/targets/s3_lifecycle/aliases index 215b6b04389..a112c3d1bb2 100644 --- a/tests/integration/targets/s3_lifecycle/aliases +++ b/tests/integration/targets/s3_lifecycle/aliases @@ -1,5 +1,2 @@ -# reason: broken -# ansible/ansible#59311 -disabled - cloud/aws +shippable/aws/group1 diff --git a/tests/integration/targets/s3_lifecycle/defaults/main.yml b/tests/integration/targets/s3_lifecycle/defaults/main.yml new file mode 100644 index 00000000000..8c5fc34f3e9 --- /dev/null +++ b/tests/integration/targets/s3_lifecycle/defaults/main.yml @@ -0,0 +1 @@ +bucket_name: '{{ resource_prefix }}-s3-lifecycle' diff --git a/tests/integration/targets/s3_lifecycle/tasks/main.yml b/tests/integration/targets/s3_lifecycle/tasks/main.yml index 95a669a7b52..d2700b9ea57 100644 --- a/tests/integration/targets/s3_lifecycle/tasks/main.yml +++ b/tests/integration/targets/s3_lifecycle/tasks/main.yml @@ -8,24 +8,26 @@ aws_secret_key: '{{ aws_secret_key }}' security_token: '{{ security_token | default(omit) }}' region: '{{ aws_region }}' + s3_lifecycle: + wait: yes block: # ============================================================ - name: Create simple s3_bucket s3_bucket: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' state: present register: output - assert: that: - output.changed - - output.name == '{{ resource_prefix }}-testbucket-ansible' + - output.name == bucket_name - not output.requester_pays # ============================================================ - name: Create a lifecycle policy s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' expiration_days: 300 prefix: '' register: output @@ -36,7 +38,7 @@ # ============================================================ - name: Create a lifecycle policy (idempotency) s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' expiration_days: 300 register: output @@ -46,7 +48,7 @@ # ============================================================ - name: Create a second lifecycle policy s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' transition_days: 30 prefix: /something register: output @@ -57,7 +59,7 @@ # ============================================================ - name: Create a second lifecycle policy (idempotency) s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' transition_days: 30 prefix: /something register: output @@ -68,7 +70,7 @@ # ============================================================ - name: Disable the second lifecycle policy s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' status: disabled transition_days: 30 prefix: /something @@ -80,7 +82,7 @@ # ============================================================ - name: Disable the second lifecycle policy (idempotency) s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' status: disabled transition_days: 30 prefix: /something @@ -92,7 +94,7 @@ # ============================================================ - name: Re-enable the second lifecycle policy s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' status: enabled transition_days: 300 prefix: /something @@ -104,7 +106,7 @@ # ============================================================ - name: Re-enable the second lifecycle policy (idempotency) s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' status: enabled transition_days: 300 prefix: /something @@ -116,7 +118,7 @@ # ============================================================ - name: Delete the second lifecycle policy s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' state: absent prefix: /something register: output @@ -127,7 +129,7 @@ # ============================================================ - name: Delete the second lifecycle policy (idempotency) s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' state: absent prefix: /something register: output @@ -138,7 +140,7 @@ # ============================================================ - name: Create a second lifecycle policy, with infrequent access s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' transition_days: 30 storage_class: standard_ia prefix: /something @@ -150,7 +152,7 @@ # ============================================================ - name: Create a second lifecycle policy, with infrequent access (idempotency) s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' storage_class: standard_ia transition_days: 30 prefix: /something @@ -162,7 +164,7 @@ # ============================================================ - name: Create a second lifecycle policy, with glacier s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' transition_days: 300 prefix: /something register: output @@ -173,7 +175,7 @@ # ============================================================ - name: Create a second lifecycle policy, with glacier (idempotency) s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' transition_days: 300 prefix: /something register: output @@ -184,7 +186,7 @@ # ============================================================ - name: Create a lifecycle policy with infrequent access s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' transition_days: 30 storage_class: standard_ia prefix: /something @@ -192,7 +194,7 @@ - name: Create a second lifecycle policy, with glacier s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' transition_days: 300 prefix: /something purge_transitions: false @@ -200,7 +202,7 @@ - name: Create a lifecycle policy with infrequent access (idempotency) s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' storage_class: standard_ia transition_days: 30 prefix: /something @@ -213,7 +215,7 @@ - name: Create a second lifecycle policy, with glacier (idempotency) s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' transition_days: 300 prefix: /something purge_transitions: false @@ -225,7 +227,7 @@ # ============================================================ - name: Create a lifecycle policy, with noncurrent expiration s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' noncurrent_version_expiration_days: 300 prefix: /something register: output @@ -236,7 +238,7 @@ # ============================================================ - name: Create a lifecycle policy, with noncurrent expiration s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' noncurrent_version_expiration_days: 300 prefix: /something register: output @@ -247,7 +249,7 @@ # ============================================================ - name: Create a lifecycle policy, with noncurrent transition s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' noncurrent_version_transition_days: 300 prefix: /something register: output @@ -258,7 +260,7 @@ # ============================================================ - name: Create a lifecycle policy, with noncurrent transitions and expirations s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' noncurrent_version_transition_days: 300 prefix: /something register: output @@ -269,7 +271,7 @@ # ============================================================ - name: Create a lifecycle policy, with noncurrent transition s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' noncurrent_version_transition_days: 300 noncurrent_version_storage_class: standard_ia prefix: /something @@ -281,7 +283,7 @@ # ============================================================ - name: Create a lifecycle policy, with noncurrent transitions and expirations s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' noncurrent_version_storage_class: standard_ia noncurrent_version_transition_days: 300 prefix: /something @@ -293,7 +295,7 @@ # ============================================================ - name: Create a lifecycle policy, with noncurrent transitions s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' noncurrent_version_transitions: - transition_days: 30 storage_class: standard_ia @@ -310,7 +312,7 @@ # ============================================================ - name: Create a lifecycle policy, with noncurrent transitions s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + name: '{{ bucket_name }}' noncurrent_version_transitions: - transition_days: 30 storage_class: standard_ia @@ -324,61 +326,165 @@ - assert: that: - output is not changed + # ============================================================ # test all the examples # Configure a lifecycle rule on a bucket to expire (delete) items with a prefix of /logs/ after 30 days - - s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + - name: example 1 + s3_lifecycle: + name: '{{ bucket_name }}' + expiration_days: 30 + prefix: /logs/ + status: enabled + state: present + register: output + - assert: + that: + - output is changed + + - name: example 1 (idempotency) + s3_lifecycle: + name: '{{ bucket_name }}' expiration_days: 30 prefix: /logs/ status: enabled state: present + register: output + - assert: + that: + - output is not changed # Configure a lifecycle rule to transition all items with a prefix of /logs/ to glacier after 7 days and then delete after 90 days - - s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + - name: example 2 + s3_lifecycle: + name: '{{ bucket_name }}' + transition_days: 7 + expiration_days: 90 + prefix: /logs/ + status: enabled + state: present + register: output + - assert: + that: + - output is changed + + - name: example 2 (idempotency) + s3_lifecycle: + name: '{{ bucket_name }}' transition_days: 7 expiration_days: 90 prefix: /logs/ status: enabled state: present + register: output + - assert: + that: + - output is not changed # Configure a lifecycle rule to transition all items with a prefix of /logs/ to glacier on 31 Dec 2020 and then delete on 31 Dec 2030. # Note that midnight GMT must be specified. # Be sure to quote your date strings - - s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + - name: example 3 + s3_lifecycle: + name: '{{ bucket_name }}' + transition_date: "2020-12-30T00:00:00.000Z" + expiration_date: "2030-12-30T00:00:00.000Z" + prefix: /logs/ + status: enabled + state: present + register: output + - assert: + that: + - output is changed + + - name: example 3 (idempotency) + s3_lifecycle: + name: '{{ bucket_name }}' transition_date: "2020-12-30T00:00:00.000Z" expiration_date: "2030-12-30T00:00:00.000Z" prefix: /logs/ status: enabled state: present + register: output + - assert: + that: + - output is not changed # Disable the rule created above - - s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + - name: example 4 + s3_lifecycle: + name: '{{ bucket_name }}' prefix: /logs/ status: disabled state: present + register: output + - assert: + that: + - output is changed + + - name: example 4 (idempotency) + s3_lifecycle: + name: '{{ bucket_name }}' + prefix: /logs/ + status: disabled + state: present + register: output + - assert: + that: + - output is not changed # Delete the lifecycle rule created above - - s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + - name: example 5 + s3_lifecycle: + name: '{{ bucket_name }}' + prefix: /logs/ + state: absent + register: output + - assert: + that: + - output is changed + + - name: example 5 (idempotency) + s3_lifecycle: + name: '{{ bucket_name }}' prefix: /logs/ state: absent + register: output + - assert: + that: + - output is not changed # Configure a lifecycle rule to transition all backup files older than 31 days in /backups/ to standard infrequent access class. - - s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + - name: example 6 + s3_lifecycle: + name: '{{ bucket_name }}' prefix: /backups/ storage_class: standard_ia transition_days: 31 state: present status: enabled + register: output + - assert: + that: + - output is changed + + - name: example 6 (idempotency) + s3_lifecycle: + name: '{{ bucket_name }}' + prefix: /backups/ + storage_class: standard_ia + transition_days: 31 + state: present + status: enabled + register: output + - assert: + that: + - output is not changed # Configure a lifecycle rule to transition files to infrequent access after 30 days and glacier after 90 - - s3_lifecycle: - name: "{{ resource_prefix }}-testbucket-ansible" + - name: example 7 + s3_lifecycle: + name: '{{ bucket_name }}' prefix: /other_logs/ state: present status: enabled @@ -387,6 +493,27 @@ storage_class: standard_ia - transition_days: 90 storage_class: glacier + register: output + - assert: + that: + - output is changed + + - name: example 7 (idempotency) + s3_lifecycle: + name: '{{ bucket_name }}' + prefix: /other_logs/ + state: present + status: enabled + transitions: + - transition_days: 30 + storage_class: standard_ia + - transition_days: 90 + storage_class: glacier + register: output + - assert: + that: + - output is not changed + # ============================================================ always: - name: Ensure all buckets are deleted @@ -395,4 +522,4 @@ state: absent ignore_errors: yes with_items: - - "{{ resource_prefix }}-testbucket-ansible" + - '{{ bucket_name }}'