s3_lifecycle: reassure that configuration is complete before returning.

Get bucket lifecycle configuration a few times to make sure it is stable.

It was observed that shortly (~30s) after setting the rules a method get-bucket-lifecycle-configuration
returns alternatively new and old rules in a random manner.
Similar issue reported for boto3 library:
boto/boto3#2491

changelogs/fragments/1085-s3_lifecycle-reassure-that-configuration-is-complete-before-returning.yml

@@ -0,0 +1,2 @@
+bugfixes:
+  - s3_lifecycle: reassure that configuration is complete before returning

plugins/modules/s3_lifecycle.py

@@ -485,15 +485,23 @@ def create_lifecycle_rule(client, module):
 
     _changed = changed
     _retries = 10
-    while wait and _changed and _retries:
+    _not_changed_cnt = 6
+    while wait and _changed and _retries and _not_changed_cnt:
         # We've seen examples where get_bucket_lifecycle_configuration returns
-        # the updated rules, then the old rules, then the updated rules again,
+        # the updated rules, then the old rules, then the updated rules again and
+        # again couple of times.
+        # Thus try to read the rule few times in a row to check if it has changed.
         time.sleep(5)
         _retries -= 1
         new_rules = fetch_rules(client, module, name)
         (_changed, lifecycle_configuration) = compare_and_update_configuration(client, module,
                                                                                new_rules,
                                                                                new_rule)
+        if not _changed:
+            _not_changed_cnt -= 1
+            _changed = True
+        else:
+            _not_changed_cnt = 6
 
     new_rules = fetch_rules(client, module, name)
 
@@ -531,13 +539,21 @@ def destroy_lifecycle_rule(client, module):
 
     _changed = changed
     _retries = 10
-    while wait and _changed and _retries:
+    _not_changed_cnt = 6
+    while wait and _changed and _retries and _not_changed_cnt:
         # We've seen examples where get_bucket_lifecycle_configuration returns
-        # the updated rules, then the old rules, then the updated rules again,
+        # the updated rules, then the old rules, then the updated rules again and
+        # again couple of times.
+        # Thus try to read the rule few times in a row to check if it has changed.
         time.sleep(5)
         _retries -= 1
         new_rules = fetch_rules(client, module, name)
         (_changed, lifecycle_configuration) = compare_and_remove_rule(new_rules, rule_id, prefix)
+        if not _changed:
+            _not_changed_cnt -= 1
+            _changed = True
+        else:
+            _not_changed_cnt = 6
 
     new_rules = fetch_rules(client, module, name)
 

tests/integration/targets/s3_lifecycle/tasks/main.yml

@@ -24,6 +24,42 @@
           - output.changed
           - output.name == bucket_name
           - not output.requester_pays
+  # ============================================================
+    - name: Create a number of policies and check if all of them are created
+      community.aws.s3_lifecycle:
+        name: "{{ bucket }}"
+        rule_id: "{{ item }}"
+        expiration_days: 10
+        prefix: "{{ item }}"
+        status: enabled
+        state: present
+        wait: yes
+      register: output
+      loop:
+        - rule_1
+        - rule_2
+        - rule_3
+    - assert:
+        that:
+          - (output.results | last).rules | length == 3
+  # ============================================================
+    - name: Remove previously created policies and check if all of them are removed
+      community.aws.s3_lifecycle:
+        name: "{{ bucket }}"
+        rule_id: "{{ item }}"
+        expiration_days: 10
+        prefix: "{{ item }}"
+        status: enabled
+        state: absent
+        wait: yes
+      register: output
+      loop:
+        - rule_1
+        - rule_2
+        - rule_3
+    - assert:
+        that:
+          - (output.results | last).rules | length == 0
   # ============================================================
     - name: Create a lifecycle policy
       s3_lifecycle: