From d443ae520dc74b4e2593ee35ac310c6ca4a3eed0 Mon Sep 17 00:00:00 2001 From: Matthew Davis <7035647+mdavis-xyz@users.noreply.github.com> Date: Fri, 14 Jan 2022 23:06:46 +1100 Subject: [PATCH] Add abort multipart upload and expire obj del markers to s3 lifecycle (#794) Add abort multipart upload and expire obj del markers to s3 lifecycle Depends-On: ansible/ansible-zuul-jobs#1247 SUMMARY Fixes #365 #796 ISSUE TYPE Feature Pull Request COMPONENT NAME s3_lifecycle ADDITIONAL INFORMATION I have not run integration tests yet because of #793. I'm unsure about how to name and structure the new arguments. Do I nest them to match the API, or flatten them to match existing arguments? Reviewed-by: Alina Buzachis Reviewed-by: Matthew Davis Reviewed-by: Mark Chappell Reviewed-by: None Reviewed-by: Markus Bergholz --- s3_lifecycle.py | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/s3_lifecycle.py b/s3_lifecycle.py index 9cec1402eb1..c12ce6b0897 100644 --- a/s3_lifecycle.py +++ b/s3_lifecycle.py @@ -23,16 +23,30 @@ - Name of the S3 bucket. required: true type: str + abort_incomplete_multipart_upload_days: + description: + - Specifies the days since the initiation of an incomplete multipart upload that Amazon S3 will wait before permanently removing all parts of the upload. + type: int + version_added: 2.2.0 expiration_date: description: - Indicates the lifetime of the objects that are subject to the rule by the date they will expire. - The value must be ISO-8601 format, the time must be midnight and a GMT timezone must be specified. + - This cannot be specified with I(expire_object_delete_marker) type: str expiration_days: description: - Indicates the lifetime, in days, of the objects that are subject to the rule. - The value must be a non-zero positive integer. + - This cannot be specified with I(expire_object_delete_marker) type: int + expire_object_delete_marker: + description: + - Indicates whether Amazon S3 will remove a delete marker with no noncurrent versions. + - If set to C(true), the delete marker will be expired; if set to C(false) the policy takes no action. + - This cannot be specified with I(expiration_days) or I(expiration_date). + type: bool + version_added: 2.2.0 prefix: description: - Prefix identifying one or more objects to which the rule applies. @@ -250,8 +264,10 @@ def fetch_rules(client, module, name): def build_rule(client, module): name = module.params.get("name") + abort_incomplete_multipart_upload_days = module.params.get("abort_incomplete_multipart_upload_days") expiration_date = parse_date(module.params.get("expiration_date")) expiration_days = module.params.get("expiration_days") + expire_object_delete_marker = module.params.get("expire_object_delete_marker") noncurrent_version_expiration_days = module.params.get("noncurrent_version_expiration_days") noncurrent_version_transition_days = module.params.get("noncurrent_version_transition_days") noncurrent_version_transitions = module.params.get("noncurrent_version_transitions") @@ -268,11 +284,19 @@ def build_rule(client, module): rule = dict(Filter=dict(Prefix=prefix), Status=status.title()) if rule_id is not None: rule['ID'] = rule_id + + if abort_incomplete_multipart_upload_days: + rule['AbortIncompleteMultipartUpload'] = { + 'DaysAfterInitiation': abort_incomplete_multipart_upload_days + } + # Create expiration if expiration_days is not None: rule['Expiration'] = dict(Days=expiration_days) elif expiration_date is not None: rule['Expiration'] = dict(Date=expiration_date.isoformat()) + elif expire_object_delete_marker is not None: + rule['Expiration'] = dict(ExpiredObjectDeleteMarker=expire_object_delete_marker) if noncurrent_version_expiration_days is not None: rule['NoncurrentVersionExpiration'] = dict(NoncurrentDays=noncurrent_version_expiration_days) @@ -525,8 +549,10 @@ def main(): s3_storage_class = ['glacier', 'onezone_ia', 'standard_ia', 'intelligent_tiering', 'deep_archive'] argument_spec = dict( name=dict(required=True, type='str'), + abort_incomplete_multipart_upload_days=dict(type='int'), expiration_days=dict(type='int'), expiration_date=dict(), + expire_object_delete_marker=dict(type='bool'), noncurrent_version_expiration_days=dict(type='int'), noncurrent_version_storage_class=dict(default='glacier', type='str', choices=s3_storage_class), noncurrent_version_transition_days=dict(type='int'), @@ -546,7 +572,7 @@ def main(): module = AnsibleAWSModule(argument_spec=argument_spec, mutually_exclusive=[ - ['expiration_days', 'expiration_date'], + ['expiration_days', 'expiration_date', 'expire_object_delete_marker'], ['expiration_days', 'transition_date'], ['transition_days', 'transition_date'], ['transition_days', 'expiration_date'], @@ -563,8 +589,10 @@ def main(): if state == 'present' and module.params["status"] == "enabled": # allow deleting/disabling a rule by id/prefix - required_when_present = ('expiration_date', 'expiration_days', 'transition_date', - 'transition_days', 'transitions', 'noncurrent_version_expiration_days', + required_when_present = ('abort_incomplete_multipart_upload_days', + 'expiration_date', 'expiration_days', 'expire_object_delete_marker', + 'transition_date', 'transition_days', 'transitions', + 'noncurrent_version_expiration_days', 'noncurrent_version_transition_days', 'noncurrent_version_transitions') for param in required_when_present: