Add abort multipart upload and expire obj del markers to s3 lifecycle (…

…#794)

Add abort multipart upload and expire obj del markers to s3 lifecycle

Depends-On: ansible/ansible-zuul-jobs#1247
SUMMARY
Fixes #365 #796
ISSUE TYPE

Feature Pull Request

COMPONENT NAME
s3_lifecycle
ADDITIONAL INFORMATION
I have not run integration tests yet because of #793.
I'm unsure about how to name and structure the new arguments.
Do I nest them to match the API, or flatten them to match existing arguments?

Reviewed-by: Alina Buzachis <None>
Reviewed-by: Matthew Davis <None>
Reviewed-by: Mark Chappell <None>
Reviewed-by: None <None>
Reviewed-by: Markus Bergholz <[email protected]>
(cherry picked from commit ed3a7a0)

changelogs/fragments/794-s3_lifecycle_abort_expire.yml

@@ -0,0 +1,2 @@
+minor_changes:
+- s3_lifecycle - Add ``abort_incomplete_multipart_upload_days`` and ``expire_object_delete_marker`` parameters (https://github.com/ansible-collections/community.aws/pull/794).

plugins/modules/s3_lifecycle.py

@@ -23,16 +23,30 @@
       - Name of the S3 bucket.
     required: true
     type: str
+  abort_incomplete_multipart_upload_days:
+    description:
+      - Specifies the days since the initiation of an incomplete multipart upload that Amazon S3 will wait before permanently removing all parts of the upload.
+    type: int
+    version_added: 2.2.0
   expiration_date:
     description:
       - Indicates the lifetime of the objects that are subject to the rule by the date they will expire.
       - The value must be ISO-8601 format, the time must be midnight and a GMT timezone must be specified.
+      - This cannot be specified with I(expire_object_delete_marker)
     type: str
   expiration_days:
     description:
       - Indicates the lifetime, in days, of the objects that are subject to the rule.
       - The value must be a non-zero positive integer.
+      - This cannot be specified with I(expire_object_delete_marker)
     type: int
+  expire_object_delete_marker:
+    description:
+      - Indicates whether Amazon S3 will remove a delete marker with no noncurrent versions.
+      - If set to C(true), the delete marker will be expired; if set to C(false) the policy takes no action.
+      - This cannot be specified with I(expiration_days) or I(expiration_date).
+    type: bool
+    version_added: 2.2.0
   prefix:
     description:
       - Prefix identifying one or more objects to which the rule applies.
@@ -250,8 +264,10 @@ def fetch_rules(client, module, name):
 
 def build_rule(client, module):
     name = module.params.get("name")
+    abort_incomplete_multipart_upload_days = module.params.get("abort_incomplete_multipart_upload_days")
     expiration_date = parse_date(module.params.get("expiration_date"))
     expiration_days = module.params.get("expiration_days")
+    expire_object_delete_marker = module.params.get("expire_object_delete_marker")
     noncurrent_version_expiration_days = module.params.get("noncurrent_version_expiration_days")
     noncurrent_version_transition_days = module.params.get("noncurrent_version_transition_days")
     noncurrent_version_transitions = module.params.get("noncurrent_version_transitions")
@@ -268,11 +284,19 @@ def build_rule(client, module):
     rule = dict(Filter=dict(Prefix=prefix), Status=status.title())
     if rule_id is not None:
         rule['ID'] = rule_id
+
+    if abort_incomplete_multipart_upload_days:
+        rule['AbortIncompleteMultipartUpload'] = {
+            'DaysAfterInitiation': abort_incomplete_multipart_upload_days
+        }
+
     # Create expiration
     if expiration_days is not None:
         rule['Expiration'] = dict(Days=expiration_days)
     elif expiration_date is not None:
         rule['Expiration'] = dict(Date=expiration_date.isoformat())
+    elif expire_object_delete_marker is not None:
+        rule['Expiration'] = dict(ExpiredObjectDeleteMarker=expire_object_delete_marker)
 
     if noncurrent_version_expiration_days is not None:
         rule['NoncurrentVersionExpiration'] = dict(NoncurrentDays=noncurrent_version_expiration_days)
@@ -525,8 +549,10 @@ def main():
     s3_storage_class = ['glacier', 'onezone_ia', 'standard_ia', 'intelligent_tiering', 'deep_archive']
     argument_spec = dict(
         name=dict(required=True, type='str'),
+        abort_incomplete_multipart_upload_days=dict(type='int'),
         expiration_days=dict(type='int'),
         expiration_date=dict(),
+        expire_object_delete_marker=dict(type='bool'),
         noncurrent_version_expiration_days=dict(type='int'),
         noncurrent_version_storage_class=dict(default='glacier', type='str', choices=s3_storage_class),
         noncurrent_version_transition_days=dict(type='int'),
@@ -546,7 +572,7 @@ def main():
 
     module = AnsibleAWSModule(argument_spec=argument_spec,
                               mutually_exclusive=[
-                                  ['expiration_days', 'expiration_date'],
+                                  ['expiration_days', 'expiration_date', 'expire_object_delete_marker'],
                                   ['expiration_days', 'transition_date'],
                                   ['transition_days', 'transition_date'],
                                   ['transition_days', 'expiration_date'],
@@ -563,8 +589,10 @@ def main():
 
     if state == 'present' and module.params["status"] == "enabled":  # allow deleting/disabling a rule by id/prefix
 
-        required_when_present = ('expiration_date', 'expiration_days', 'transition_date',
-                                 'transition_days', 'transitions', 'noncurrent_version_expiration_days',
+        required_when_present = ('abort_incomplete_multipart_upload_days',
+                                 'expiration_date', 'expiration_days', 'expire_object_delete_marker',
+                                 'transition_date', 'transition_days', 'transitions',
+                                 'noncurrent_version_expiration_days',
                                  'noncurrent_version_transition_days',
                                  'noncurrent_version_transitions')
         for param in required_when_present:

tests/integration/targets/s3_lifecycle/tasks/main.yml

@@ -327,6 +327,54 @@
         that:
           - output is not changed
 
+  # ============================================================
+    - name: Create a lifecycle policy, with abort_incomplete_multipart_upload
+      s3_lifecycle:
+        name: '{{ bucket_name }}'
+        abort_incomplete_multipart_upload_days: 1
+        prefix: /something
+      register: output
+
+    - assert:
+        that:
+          - output is changed
+
+  # ============================================================
+    - name: Create a lifecycle policy, with abort_incomplete_multipart_upload (idempotency)
+      s3_lifecycle:
+        name: '{{ bucket_name }}'
+        abort_incomplete_multipart_upload_days: 1
+        prefix: /something
+      register: output
+
+    - assert:
+        that:
+          - output is not changed
+
+  # ============================================================
+    - name: Create a lifecycle policy, with expired_object_delete_marker
+      s3_lifecycle:
+        name: '{{ bucket_name }}'
+        expire_object_delete_marker: True
+        prefix: /something
+      register: output
+
+    - assert:
+        that:
+          - output is changed
+
+  # ============================================================
+    - name: Create a lifecycle policy, with expired_object_delete_marker  (idempotency)
+      s3_lifecycle:
+        name: '{{ bucket_name }}'
+        expire_object_delete_marker: True
+        prefix: /something
+      register: output
+
+    - assert:
+        that:
+          - output is not changed
+
   # ============================================================
   # test all the examples
   # Configure a lifecycle rule on a bucket to expire (delete) items with a prefix of /logs/ after 30 days