Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support min max object size in s3_lifecycle module #2205

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 43 additions & 10 deletions plugins/modules/s3_lifecycle.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,16 @@
and replaced with the new transition(s)
default: true
type: bool
maximum_object_size:
description:
- The maximum object size to which the rule applies.
required: false
type: int
minimum_object_size:
description:
- The minimum object size to which the rule applies.
required: false
type: int
noncurrent_version_expiration_days:
description:
- The number of days after which non-current versions should be deleted.
Expand Down Expand Up @@ -271,13 +281,30 @@ def fetch_rules(client, module, name):
module.fail_json_aws(e)
return current_lifecycle_rules


# Helper function to deeply compare filters
def filters_are_equal(filter1, filter2):
if filter1 == filter2:
return True
if not filter1 or not filter2:
return False
# Treat empty string as equal to a filter not being set
return (
filter1.get("Prefix", "") == filter2.get("Prefix", "")
and filter1.get("ObjectSizeGreaterThan") == filter2.get("ObjectSizeGreaterThan")
and filter1.get("ObjectSizeLessThan") == filter2.get("ObjectSizeLessThan")
and filter1.get("And", {}).get("Prefix", "") == filter2.get("And", {}).get("Prefix", "")
and filter1.get("And", {}).get("ObjectSizeGreaterThan") == filter2.get("And", {}).get("ObjectSizeGreaterThan")
and filter1.get("And", {}).get("ObjectSizeLessThan") == filter2.get("And", {}).get("ObjectSizeLessThan")
)

def build_rule(client, module):
name = module.params.get("name")
abort_incomplete_multipart_upload_days = module.params.get("abort_incomplete_multipart_upload_days")
expiration_date = parse_date(module.params.get("expiration_date"))
expiration_days = module.params.get("expiration_days")
expire_object_delete_marker = module.params.get("expire_object_delete_marker")
maximum_object_size = module.params.get("maximum_object_size")
minimum_object_size = module.params.get("minimum_object_size")
noncurrent_version_expiration_days = module.params.get("noncurrent_version_expiration_days")
noncurrent_version_transition_days = module.params.get("noncurrent_version_transition_days")
noncurrent_version_transitions = module.params.get("noncurrent_version_transitions")
Expand All @@ -292,7 +319,15 @@ def build_rule(client, module):
transitions = module.params.get("transitions")
purge_transitions = module.params.get("purge_transitions")

rule = dict(Filter=dict(Prefix=prefix), Status=status.title())
if maximum_object_size is not None or minimum_object_size is not None:
and_dict = dict(Prefix=prefix)
if minimum_object_size is not None:
and_dict["ObjectSizeGreaterThan"] = minimum_object_size
if maximum_object_size is not None:
and_dict["ObjectSizeLessThan"] = maximum_object_size
rule = dict(Filter=dict(And=and_dict), Status=status.title())
else:
rule = dict(Filter=dict(Prefix=prefix), Status=status.title())
if rule_id is not None:
rule["ID"] = rule_id

Expand Down Expand Up @@ -362,18 +397,14 @@ def compare_and_update_configuration(client, module, current_lifecycle_rules, ru
lifecycle_configuration = dict(Rules=[])
changed = False
appended = False

# If current_lifecycle_obj is not None then we have rules to compare, otherwise just add the rule
if current_lifecycle_rules:
# If rule ID exists, use that for comparison otherwise compare based on prefix
for existing_rule in current_lifecycle_rules:
if rule.get("ID") == existing_rule.get("ID") and rule["Filter"].get("Prefix", "") != existing_rule.get(
"Filter", {}
).get("Prefix", ""):
if rule.get("ID") == existing_rule.get("ID") and not filters_are_equal(rule.get("Filter"), existing_rule.get("Filter")):
existing_rule.pop("ID")
elif rule_id is None and rule["Filter"].get("Prefix", "") == existing_rule.get("Filter", {}).get(
"Prefix", ""
):
elif rule_id is None and filters_are_equal(rule.get("Filter"), existing_rule.get("Filter")):
existing_rule.pop("ID")
if rule.get("ID") == existing_rule.get("ID"):
changed_, appended_ = update_or_append_rule(
Expand Down Expand Up @@ -598,6 +629,8 @@ def main():
expiration_days=dict(type="int"),
expiration_date=dict(),
expire_object_delete_marker=dict(type="bool"),
maximum_object_size=dict(type=int),
minimum_object_size=dict(type=int),
Comment on lines +632 to +633
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
maximum_object_size=dict(type=int),
minimum_object_size=dict(type=int),
maximum_object_size=dict(type="int"),
minimum_object_size=dict(type="int"),

noncurrent_version_expiration_days=dict(type="int"),
noncurrent_version_keep_newer=dict(type="int"),
noncurrent_version_storage_class=dict(default="glacier", type="str", choices=s3_storage_class),
Expand Down Expand Up @@ -681,4 +714,4 @@ def main():


if __name__ == "__main__":
main()
main()
89 changes: 89 additions & 0 deletions tests/integration/targets/s3_lifecycle/tasks/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -694,6 +694,95 @@
that:
- output is changed

# Check create and delete lifecycle policy with minimum and maximum object size (with prefix)
- name: Create rule with minimum and maximum object size
s3_lifecycle:
name: "{{ bucket_name }}"
rule_id: minimum-object-size-prefix
prefix: /something/
minimum_object_size: 100
maximum_object_size: 1000
state: present
status: enabled
expiration_days: 30
register: output
- assert:
that:
- output is changed

- name: Create rule with minimum object size (idempotency)
s3_lifecycle:
name: "{{ bucket_name }}"
rule_id: minimum-object-size-prefix
prefix: /something/
minimum_object_size: 100
maximum_object_size: 1000
state: present
status: enabled
expiration_days: 30
register: output
- assert:
that:
- output is not changed

- name: Delete rule with minimum and maximum object size
s3_lifecycle:
name: "{{ bucket_name }}"
rule_id: minimum-object-size-prefix
prefix: /something/
minimum_object_size: 100
maximum_object_size: 1000
state: absent
status: enabled
expiration_days: 30
register: output
- assert:
that:
- output is changed

# Check create and delete lifecycle policy with minimum and maximum object size (no prefix)
- name: Create rule with minimum and maximum object size
s3_lifecycle:
name: "{{ bucket_name }}"
rule_id: minimum-object-size-noprefix
minimum_object_size: 100
maximum_object_size: 1000
state: present
status: enabled
expiration_days: 30
register: output
- assert:
that:
- output is changed

- name: Create rule with minimum object size (idempotency)
s3_lifecycle:
name: "{{ bucket_name }}"
rule_id: minimum-object-size-noprefix
minimum_object_size: 100
maximum_object_size: 1000
state: present
status: enabled
expiration_days: 30
register: output
- assert:
that:
- output is not changed

- name: Delete rule with minimum and maximum object size
s3_lifecycle:
name: "{{ bucket_name }}"
rule_id: minimum-object-size-noprefix
minimum_object_size: 100
maximum_object_size: 1000
state: absent
status: enabled
expiration_days: 30
register: output
- assert:
that:
- output is changed

# ============================================================
always:
- name: Ensure all buckets are deleted
Expand Down
Loading