From b6fa7b297c429e912ce519dabf586adcfb810335 Mon Sep 17 00:00:00 2001 From: David James Date: Thu, 2 Feb 2023 02:11:25 +1100 Subject: [PATCH] s3_lifecycle - ability to set the number of newest noncurrent versions to retain (#1606) s3_lifecycle - ability to set the number of newest noncurrent versions to retain SUMMARY Adds the ability to set "Number of newer versions to retain" ISSUE TYPE Feature Pull Request COMPONENT NAME s3_lifecycle ADDITIONAL INFORMATION See: https://docs.aws.amazon.com/AmazonS3/latest/API/API_NoncurrentVersionExpiration.html Previously only the NoncurrentDays parameter was supported, this PR adds support for NewerNoncurrentVersions Reviewed-by: Markus Bergholz Reviewed-by: Mark Chappell (cherry picked from commit 3391b27d0c856d7b044de2a49da730cc6e5481f9) --- ...cycle_add_number_of_versions_to_retain.yml | 2 + plugins/modules/s3_lifecycle.py | 51 +++++++++++++----- .../targets/s3_lifecycle/meta/main.yml | 5 +- .../targets/s3_lifecycle/tasks/main.yml | 53 ++++++++++++++++++- 4 files changed, 96 insertions(+), 15 deletions(-) create mode 100644 changelogs/fragments/1606-s3_lifecycle_add_number_of_versions_to_retain.yml diff --git a/changelogs/fragments/1606-s3_lifecycle_add_number_of_versions_to_retain.yml b/changelogs/fragments/1606-s3_lifecycle_add_number_of_versions_to_retain.yml new file mode 100644 index 00000000000..2c1bceada01 --- /dev/null +++ b/changelogs/fragments/1606-s3_lifecycle_add_number_of_versions_to_retain.yml @@ -0,0 +1,2 @@ +minor_changes: +- s3_lifecycle - add parameter `noncurrent_version_keep_newer` to set the number of newest noncurrent versions to retain diff --git a/plugins/modules/s3_lifecycle.py b/plugins/modules/s3_lifecycle.py index 662a776cadf..59474d2518f 100644 --- a/plugins/modules/s3_lifecycle.py +++ b/plugins/modules/s3_lifecycle.py @@ -63,8 +63,17 @@ noncurrent_version_expiration_days: description: - The number of days after which non-current versions should be deleted. + - Must be set if I(noncurrent_version_keep_newer) is set. required: false type: int + noncurrent_version_keep_newer: + description: + - The minimum number of non-current versions to retain. + - Requires C(botocore >= 1.23.12) + - Requres I(noncurrent_version_expiration_days). + required: false + type: int + version_added: 5.3.0 noncurrent_version_storage_class: description: - The storage class to which non-current versions are transitioned. @@ -269,6 +278,7 @@ def build_rule(client, module): noncurrent_version_transition_days = module.params.get("noncurrent_version_transition_days") noncurrent_version_transitions = module.params.get("noncurrent_version_transitions") noncurrent_version_storage_class = module.params.get("noncurrent_version_storage_class") + noncurrent_version_keep_newer = module.params.get("noncurrent_version_keep_newer") prefix = module.params.get("prefix") or "" rule_id = module.params.get("rule_id") status = module.params.get("status") @@ -294,10 +304,12 @@ def build_rule(client, module): rule['Expiration'] = dict(Date=expiration_date.isoformat()) elif expire_object_delete_marker is not None: rule['Expiration'] = dict(ExpiredObjectDeleteMarker=expire_object_delete_marker) - + if noncurrent_version_expiration_days or noncurrent_version_keep_newer: + rule['NoncurrentVersionExpiration'] = dict() if noncurrent_version_expiration_days is not None: - rule['NoncurrentVersionExpiration'] = dict(NoncurrentDays=noncurrent_version_expiration_days) - + rule['NoncurrentVersionExpiration']['NoncurrentDays'] = noncurrent_version_expiration_days + if noncurrent_version_keep_newer is not None: + rule['NoncurrentVersionExpiration']['NewerNoncurrentVersions'] = noncurrent_version_keep_newer if transition_days is not None: rule['Transitions'] = [dict(Days=transition_days, StorageClass=storage_class.upper()), ] @@ -572,6 +584,7 @@ def main(): expiration_date=dict(), expire_object_delete_marker=dict(type='bool'), noncurrent_version_expiration_days=dict(type='int'), + noncurrent_version_keep_newer=dict(type='int'), noncurrent_version_storage_class=dict(default='glacier', type='str', choices=s3_storage_class), noncurrent_version_transition_days=dict(type='int'), noncurrent_version_transitions=dict(type='list', elements='dict'), @@ -587,16 +600,21 @@ def main(): wait=dict(type='bool', default=False) ) - module = AnsibleAWSModule(argument_spec=argument_spec, - mutually_exclusive=[ - ['expiration_days', 'expiration_date', 'expire_object_delete_marker'], - ['expiration_days', 'transition_date'], - ['transition_days', 'transition_date'], - ['transition_days', 'expiration_date'], - ['transition_days', 'transitions'], - ['transition_date', 'transitions'], - ['noncurrent_version_transition_days', 'noncurrent_version_transitions'], - ],) + module = AnsibleAWSModule( + argument_spec=argument_spec, + mutually_exclusive=[ + ["expiration_days", "expiration_date", "expire_object_delete_marker"], + ["expiration_days", "transition_date"], + ["transition_days", "transition_date"], + ["transition_days", "expiration_date"], + ["transition_days", "transitions"], + ["transition_date", "transitions"], + ["noncurrent_version_transition_days", "noncurrent_version_transitions"], + ], + required_by={ + "noncurrent_version_keep_newer": ["noncurrent_version_expiration_days"], + }, + ) client = module.client('s3', retry_decorator=AWSRetry.jittered_backoff()) @@ -604,12 +622,19 @@ def main(): transition_date = module.params.get("transition_date") state = module.params.get("state") + if module.params.get("noncurrent_version_keep_newer"): + module.require_botocore_at_least( + "1.23.12", + reason="to set number of versions to keep with noncurrent_version_keep_newer" + ) + if state == 'present' and module.params["status"] == "enabled": # allow deleting/disabling a rule by id/prefix required_when_present = ('abort_incomplete_multipart_upload_days', 'expiration_date', 'expiration_days', 'expire_object_delete_marker', 'transition_date', 'transition_days', 'transitions', 'noncurrent_version_expiration_days', + 'noncurrent_version_keep_newer', 'noncurrent_version_transition_days', 'noncurrent_version_transitions') for param in required_when_present: diff --git a/tests/integration/targets/s3_lifecycle/meta/main.yml b/tests/integration/targets/s3_lifecycle/meta/main.yml index 32cf5dda7ed..c0199066436 100644 --- a/tests/integration/targets/s3_lifecycle/meta/main.yml +++ b/tests/integration/targets/s3_lifecycle/meta/main.yml @@ -1 +1,4 @@ -dependencies: [] +dependencies: + - role: setup_botocore_pip + vars: + botocore_version: "1.23.12" diff --git a/tests/integration/targets/s3_lifecycle/tasks/main.yml b/tests/integration/targets/s3_lifecycle/tasks/main.yml index 4f0fd527860..71f183e8f38 100644 --- a/tests/integration/targets/s3_lifecycle/tasks/main.yml +++ b/tests/integration/targets/s3_lifecycle/tasks/main.yml @@ -423,7 +423,6 @@ that: - output is changed - # ============================================================ - name: Create a lifecycle policy, with expired_object_delete_marker (idempotency) s3_lifecycle: name: '{{ bucket_name }}' @@ -435,6 +434,58 @@ that: - output is not changed + # ============================================================ + - name: Update lifecycle policy, with noncurrent_version_expiration_days + s3_lifecycle: + name: '{{ bucket_name }}' + noncurrent_version_expiration_days: 5 + prefix: /something + register: output + + - assert: + that: + - output is changed + + - name: Update lifecycle policy, with noncurrent_version_expiration_days (idempotency) + s3_lifecycle: + name: '{{ bucket_name }}' + noncurrent_version_expiration_days: 5 + prefix: /something + register: output + + - assert: + that: + - output is not changed + + # ============================================================ + - name: Update lifecycle policy, with noncurrent_version_keep_newer + s3_lifecycle: + name: '{{ bucket_name }}' + noncurrent_version_expiration_days: 10 + noncurrent_version_keep_newer: 6 + prefix: /something + register: output + vars: + ansible_python_interpreter: "{{ botocore_virtualenv_interpreter }}" + + - assert: + that: + - output is changed + + - name: Update lifecycle policy, with noncurrent_version_keep_newer (idempotency) + s3_lifecycle: + name: '{{ bucket_name }}' + noncurrent_version_expiration_days: 10 + noncurrent_version_keep_newer: 6 + prefix: /something + register: output + vars: + ansible_python_interpreter: "{{ botocore_virtualenv_interpreter }}" + + - assert: + that: + - output is not changed + # ============================================================ # test all the examples # Configure a lifecycle rule on a bucket to expire (delete) items with a prefix of /logs/ after 30 days