From 5ead61c55b70ae143e058311490f527837ca049b Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Fri, 9 Oct 2020 17:23:29 +0200 Subject: [PATCH 01/63] WIP - adding new functionality to aws_s3_bucket_info module --- plugins/modules/aws_s3_bucket_info.py | 246 +++++++++++++++++++++++++- 1 file changed, 237 insertions(+), 9 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 40de3650c9c..62a8754cb46 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -1,6 +1,8 @@ #!/usr/bin/python -# Copyright (c) 2017 Ansible Project -# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +""" +Copyright (c) 2017 Ansible Project +GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +""" from __future__ import absolute_import, division, print_function __metaclass__ = type @@ -10,6 +12,7 @@ --- module: aws_s3_bucket_info version_added: 1.0.0 +author: "Gerben Geijteman (@hyperized)" short_description: Lists S3 buckets in AWS requirements: - boto3 >= 1.4.4 @@ -18,7 +21,101 @@ - Lists S3 buckets in AWS - This module was called C(aws_s3_bucket_facts) before Ansible 2.9, returning C(ansible_facts). Note that the M(community.aws.aws_s3_bucket_info) module no longer returns C(ansible_facts)! -author: "Gerben Geijteman (@hyperized)" +options: + name: + description: + - Get info about specified bucket + type: str + default: "" + name_filter: + description: + - Get info about buckets name matching defined string + type: str + default: "" + bucket_facts: + description: + - Retrieve requested S3 bucket detailed information + suboptions: + bucket_accelerate_configuration: + description: Retrive S3 bucket accelerate configuration + type: bool + default: False + bucket_location: + description: Retrive S3 bucket location + type: bool + default: False + bucket_replication: + description: Retrive S3 bucket replication + type: bool + default: False + bucket_acl: + description: Retrive S3 bucket ACLs + type: bool + default: False + bucket_logging: + description: Retrive S3 bucket logging + type: bool + default: False + bucket_request_payment: + description: Retrive S3 bucket request payment + type: bool + default: False + bucket_analytics_configuration: + description: Retrive S3 bucket analytics configuration + type: bool + default: False + bucket_metrics_configuration: + description: Retrive S3 bucket metrics configuration + type: bool + default: False + bucket_tagging: + description: Retrive S3 bucket tagging + type: bool + default: False + bucket_cors: + description: Retrive S3 bucket CORS configuration + type: bool + default: False + bucket_notification_configuration: + description: Retrive S3 bucket notification configuration + type: bool + default: False + bucket_encryption: + description: Retrive S3 bucket encryption + type: bool + default: False + bucket_ownership_controls: + description: Retrive S3 ownership controls + type: bool + default: False + bucket_website: + description: Retrive S3 bucket website + type: bool + default: False + bucket_inventory_configuration: + description: Retrive S3 bucket inventory configuration + type: bool + default: False + bucket_policy: + description: Retrive S3 bucket policy + type: bool + default: False + bucket_policy_status: + description: Retrive S3 bucket policy status + type: bool + default: False + bucket_lifecycle_configuration: + description: Retrive S3 bucket lifecycle configuration + type: bool + default: False + type: dict + transform_location: + description: + - S3 bucket location for default us-east-1 is normally reported as 'null' + - setting this option to 'true' will return 'us-east-1' instead + type: bool + default: False + extends_documentation_fragment: - amazon.aws.aws - amazon.aws.ec2 @@ -59,19 +156,109 @@ from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule -def get_bucket_list(module, connection): +def get_bucket_list(module, connection, name="", name_filter=""): """ Return result of list_buckets json encoded + Filter only buckets matching 'name' or name_filter if defined :param module: :param connection: :return: """ + buckets = [] + filtered_buckets = [] + final_buckets = [] + # Get all buckets try: buckets = camel_dict_to_snake_dict(connection.list_buckets())['buckets'] except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg="Failed to list buckets") - return buckets + # Filter buckets if requested + if name_filter: + for bucket in buckets: + if name_filter in bucket['name']: + filtered_buckets.append(bucket) + elif name: + for bucket in buckets: + if name == bucket['name']: + filtered_buckets.append(bucket) + + # Return proper list (filtered or all) + if filtered_buckets: + final_buckets = filtered_buckets + else: + final_buckets = buckets + return(final_buckets) + + +def get_buckets_facts(connection, buckets, requested_facts, transform_location): + """ + Retrive additional information about S3 buckets + """ + full_bucket_list = [] + # Iterate over all buckets and append retrived facts to bucket + for bucket in buckets: + bucket.update(get_bucket_details(connection, bucket['name'], requested_facts, transform_location)) + full_bucket_list.append(bucket) + + return(full_bucket_list) + + +def get_bucket_details(connection, name, requested_facts, transform_location): + """ + Execute all enabled S3API get calls for selected bucket + """ + all_facts = {} + + for key in requested_facts: + if requested_facts[key]: + if key == 'bucket_location': + all_facts[key] = get_bucket_location(name, connection, transform_location) + else: + all_facts[key] = get_bucket_property(name, connection, key) + + return(all_facts) + + +def get_bucket_location(name, connection, transform_location=False): + """ + Get bucket location and optionally transform 'null' to 'us-east-1' + """ + try: + data = connection.get_bucket_location(Bucket=name) + except botocore.exceptions.ClientError as err_msg: + data = {'error': err_msg} + + if transform_location: + try: + if not data['LocationConstraint']: + data['LocationConstraint'] = 'us-east-1' + except KeyError: + data['transform_failed'] = True + + try: + data.pop('ResponseMetadata') + return(data) + except KeyError: + return(data) + + +def get_bucket_property(name, connection, get_api_name): + """ + Get bucket property + """ + api_call = "get_" + get_api_name + api_function = getattr(connection, api_call) + try: + data = api_function(Bucket=name) + except botocore.exceptions.ClientError as err_msg: + data = {'error': err_msg} + + try: + data.pop('ResponseMetadata') + return(data) + except KeyError: + return(data) def main(): @@ -80,24 +267,65 @@ def main(): :return: """ + argument_spec = dict( + name=dict(type=str, default=""), + name_filter=dict(type=str, default=""), + bucket_facts=dict(type='dict', options=dict( + bucket_accelerate_configuration=dict(type=bool, default=False), + bucket_acl=dict(type=bool, default=False), + bucket_analytics_configuration=dict(type=bool, default=False), + bucket_cors=dict(type=bool, default=False), + bucket_encryption=dict(type=bool, default=False), + bucket_inventory_configuration=dict(type=bool, default=False), + bucket_lifecycle_configuration=dict(type=bool, default=False), + bucket_location=dict(type=bool, default=False), + bucket_logging=dict(type=bool, default=False), + bucket_metrics_configuration=dict(type=bool, default=False), + bucket_notification_configuration=dict(type=bool, default=False), + bucket_ownership_controls=dict(type=bool, default=False), + bucket_policy=dict(type=bool, default=False), + bucket_policy_status=dict(type=bool, default=False), + bucket_replication=dict(type=bool, default=False), + bucket_request_payment=dict(type=bool, default=False), + bucket_tagging=dict(type=bool, default=False), + bucket_website=dict(type=bool, default=False), + )), + transform_location=dict(type='bool', default=False) + ) + + # Ensure we have an empty dict result = {} # Including ec2 argument spec - module = AnsibleAWSModule(argument_spec={}, supports_check_mode=True) + module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True) is_old_facts = module._name == 'aws_s3_bucket_facts' if is_old_facts: module.deprecate("The 'aws_s3_bucket_facts' module has been renamed to 'aws_s3_bucket_info', " "and the renamed one no longer returns ansible_facts", date='2021-12-01', collection_name='community.aws') + # Get parameters + name = module.params.get("name") + name_filter = module.params.get("name_filter") + requested_facts = module.params.get("bucket_facts") + transform_location = module.params.get("bucket_facts") + # Set up connection + connection = {} try: connection = module.client('s3') - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + except (connection.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: module.fail_json_aws(e, msg='Failed to connect to AWS') - # Gather results - result['buckets'] = get_bucket_list(module, connection) + # Get basic bucket list (name + creation date) + bucket_list = get_bucket_list(module, connection, name, name_filter) + + # Gather detailed information about buckets if requested + bucket_facts = module.params.get("bucket_facts") + if bucket_facts: + result['buckets'] = get_buckets_facts(connection, bucket_list, requested_facts, transform_location) + else: + result['buckets'] = bucket_list # Send exit if is_old_facts: From d03148702ad8576360fe31760a6642a7600df357 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 16:06:17 +0200 Subject: [PATCH 02/63] Fixing exceptions adding integration tests --- plugins/modules/aws_s3_bucket_info.py | 123 ++++++++++++------ .../targets/aws_s3_bucket_info/aliases | 2 + .../aws_s3_bucket_info/defaults/main.yml | 6 + .../targets/aws_s3_bucket_info/meta/main.yml | 3 + .../targets/aws_s3_bucket_info/tasks/main.yml | 77 +++++++++++ 5 files changed, 173 insertions(+), 38 deletions(-) create mode 100644 tests/integration/targets/aws_s3_bucket_info/aliases create mode 100644 tests/integration/targets/aws_s3_bucket_info/defaults/main.yml create mode 100644 tests/integration/targets/aws_s3_bucket_info/meta/main.yml create mode 100644 tests/integration/targets/aws_s3_bucket_info/tasks/main.yml diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 62a8754cb46..451eba88a0b 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -24,22 +24,20 @@ options: name: description: - - Get info about specified bucket + - Get info only about specified bucket type: str default: "" name_filter: description: - - Get info about buckets name matching defined string + - Get info only about buckets name matching defined string type: str default: "" bucket_facts: description: - Retrieve requested S3 bucket detailed information + - Each bucket_X option executes one API call, hence many options=true will case slower module execution + - You can limit buckets by using I(name) or I(name_filter) option suboptions: - bucket_accelerate_configuration: - description: Retrive S3 bucket accelerate configuration - type: bool - default: False bucket_location: description: Retrive S3 bucket location type: bool @@ -64,10 +62,6 @@ description: Retrive S3 bucket analytics configuration type: bool default: False - bucket_metrics_configuration: - description: Retrive S3 bucket metrics configuration - type: bool - default: False bucket_tagging: description: Retrive S3 bucket tagging type: bool @@ -92,10 +86,6 @@ description: Retrive S3 bucket website type: bool default: False - bucket_inventory_configuration: - description: Retrive S3 bucket inventory configuration - type: bool - default: False bucket_policy: description: Retrive S3 bucket policy type: bool @@ -108,11 +98,16 @@ description: Retrive S3 bucket lifecycle configuration type: bool default: False + public_access_block: + description: Retrive S3 bucket public access block + type: bool + default: False type: dict transform_location: description: - S3 bucket location for default us-east-1 is normally reported as 'null' - setting this option to 'true' will return 'us-east-1' instead + - affects only queries with I(bucket_facts) > I(bucket_location) = true type: bool default: False @@ -131,6 +126,32 @@ - community.aws.aws_s3_bucket_info: register: result +# Retrieve detailed bucket information +- community.aws.aws_s3_bucket_info: + # Show only buckets with name matching + name_filter: your.testing + # Choose facts to retrieve + bucket_facts: + # bucket_accelerate_configuration: true + bucket_acl: true + bucket_cors: true + bucket_encryption: true + # bucket_lifecycle_configuration: true + bucket_location: true + # bucket_logging: true + # bucket_notification_configuration: true + # bucket_ownership_controls: true + # bucket_policy: true + # bucket_policy_status: true + # bucket_replication: true + # bucket_request_payment: true + # bucket_tagging: true + # bucket_website: true + # public_access_block: true + transform_location: true + register: result + +# Print out result - name: List buckets ansible.builtin.debug: msg: "{{ result['buckets'] }}" @@ -143,7 +164,20 @@ sample: - creation_date: '2017-07-06 15:05:12 +00:00' name: my_bucket - type: list + # bucket facts if requested + bucket_location: dictionary data + bucket_cors: dictionary data + # ...etc + +# if name options was specified +bucket_name: + description: "Name of the bucket requested" + sample: "my_bucket" + +# if name_filter was specified +bucket_name_filter: + description: "String to match bucket name" + sample: "buckets_prefix" ''' try: @@ -184,7 +218,7 @@ def get_bucket_list(module, connection, name="", name_filter=""): filtered_buckets.append(bucket) # Return proper list (filtered or all) - if filtered_buckets: + if name or name_filter: final_buckets = filtered_buckets else: final_buckets = buckets @@ -213,47 +247,53 @@ def get_bucket_details(connection, name, requested_facts, transform_location): for key in requested_facts: if requested_facts[key]: if key == 'bucket_location': - all_facts[key] = get_bucket_location(name, connection, transform_location) + all_facts[key] = {} + try: + all_facts[key] = get_bucket_location(name, connection, transform_location) + # we just pass on error - error means that resources is undefined + except botocore.exceptions.ClientError: + pass else: - all_facts[key] = get_bucket_property(name, connection, key) + all_facts[key] = {} + try: + all_facts[key] = get_bucket_property(name, connection, key) + # we just pass on error - error means that resources is undefined + except botocore.exceptions.ClientError: + pass return(all_facts) - +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_location(name, connection, transform_location=False): """ Get bucket location and optionally transform 'null' to 'us-east-1' """ - try: - data = connection.get_bucket_location(Bucket=name) - except botocore.exceptions.ClientError as err_msg: - data = {'error': err_msg} + data = connection.get_bucket_location(Bucket=name) + # Replace 'null' with 'us-east-1'? if transform_location: try: if not data['LocationConstraint']: data['LocationConstraint'] = 'us-east-1' except KeyError: - data['transform_failed'] = True - + pass + # Strip response metadata (not needed) try: data.pop('ResponseMetadata') return(data) except KeyError: return(data) - +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ Get bucket property """ api_call = "get_" + get_api_name api_function = getattr(connection, api_call) - try: - data = api_function(Bucket=name) - except botocore.exceptions.ClientError as err_msg: - data = {'error': err_msg} + data = api_function(Bucket=name) + # Strip response metadata (not needed) try: data.pop('ResponseMetadata') return(data) @@ -266,22 +306,18 @@ def main(): Get list of S3 buckets :return: """ - argument_spec = dict( name=dict(type=str, default=""), name_filter=dict(type=str, default=""), bucket_facts=dict(type='dict', options=dict( bucket_accelerate_configuration=dict(type=bool, default=False), bucket_acl=dict(type=bool, default=False), - bucket_analytics_configuration=dict(type=bool, default=False), bucket_cors=dict(type=bool, default=False), bucket_encryption=dict(type=bool, default=False), - bucket_inventory_configuration=dict(type=bool, default=False), bucket_lifecycle_configuration=dict(type=bool, default=False), + bucket_notification_configuration=dict(type=bool, default=False), bucket_location=dict(type=bool, default=False), bucket_logging=dict(type=bool, default=False), - bucket_metrics_configuration=dict(type=bool, default=False), - bucket_notification_configuration=dict(type=bool, default=False), bucket_ownership_controls=dict(type=bool, default=False), bucket_policy=dict(type=bool, default=False), bucket_policy_status=dict(type=bool, default=False), @@ -289,16 +325,21 @@ def main(): bucket_request_payment=dict(type=bool, default=False), bucket_tagging=dict(type=bool, default=False), bucket_website=dict(type=bool, default=False), + public_access_block=dict(type=bool, default=False), )), transform_location=dict(type='bool', default=False) ) - # Ensure we have an empty dict result = {} + # Define mutually exclusive options + mutually_exclusive = [ + ['name', 'name_filter'] + ] + # Including ec2 argument spec - module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True) + module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True, mutually_exclusive=mutually_exclusive) is_old_facts = module._name == 'aws_s3_bucket_facts' if is_old_facts: module.deprecate("The 'aws_s3_bucket_facts' module has been renamed to 'aws_s3_bucket_info', " @@ -320,6 +361,12 @@ def main(): # Get basic bucket list (name + creation date) bucket_list = get_bucket_list(module, connection, name, name_filter) + # Add information about name/name_filter to result + if name: + result['bucket_name'] = name + elif name_filter: + result['bucket_name_filter'] = name_filter + # Gather detailed information about buckets if requested bucket_facts = module.params.get("bucket_facts") if bucket_facts: @@ -333,6 +380,6 @@ def main(): else: module.exit_json(msg="Retrieved s3 info.", **result) - +## MAIN ## if __name__ == '__main__': main() diff --git a/tests/integration/targets/aws_s3_bucket_info/aliases b/tests/integration/targets/aws_s3_bucket_info/aliases new file mode 100644 index 00000000000..6e3860bee23 --- /dev/null +++ b/tests/integration/targets/aws_s3_bucket_info/aliases @@ -0,0 +1,2 @@ +cloud/aws +shippable/aws/group2 diff --git a/tests/integration/targets/aws_s3_bucket_info/defaults/main.yml b/tests/integration/targets/aws_s3_bucket_info/defaults/main.yml new file mode 100644 index 00000000000..4f4376a251c --- /dev/null +++ b/tests/integration/targets/aws_s3_bucket_info/defaults/main.yml @@ -0,0 +1,6 @@ +--- +name_pattern: "testbucket-ansible-integration" + +testing_buckets: + - "{{ resource_prefix }}-{{ name_pattern }}-1" + - "{{ resource_prefix }}-{{ name_pattern }}-2" diff --git a/tests/integration/targets/aws_s3_bucket_info/meta/main.yml b/tests/integration/targets/aws_s3_bucket_info/meta/main.yml new file mode 100644 index 00000000000..1f64f1169a9 --- /dev/null +++ b/tests/integration/targets/aws_s3_bucket_info/meta/main.yml @@ -0,0 +1,3 @@ +dependencies: + - prepare_tests + - setup_ec2 diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml new file mode 100644 index 00000000000..9e07a021372 --- /dev/null +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -0,0 +1,77 @@ +--- +- name: Test community.aws.aws_s3_bucket_info + module_defaults: + group/aws: + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + security_token: "{{ security_token | default(omit) }}" + region: "{{ aws_region }}" + block: + - name: Create simple s3_buckets + s3_bucket: + name: "{{ item }}" + state: present + register: output + loop: "{{ testing_buckets }}" + + - name: Get simple S3 bucket list + aws_s3_bucket_info: + register: bucket_list + + - name: Assert result.changed == False and bucket list was retrieved + assert: + that: + - bucket_list.changed == False + - bucket_list.buckets + + - name: Get complex S3 bucket list + aws_s3_bucket_info: + name_filter: "{{ name_pattern }}" + bucket_facts: + bucket_accelerate_configuration: true + bucket_acl: true + bucket_cors: true + bucket_encryption: true + bucket_lifecycle_configuration: true + bucket_location: true + bucket_logging: true + bucket_notification_configuration: true + bucket_ownership_controls: true + bucket_policy: true + bucket_policy_status: true + bucket_replication: true + bucket_request_payment: true + bucket_tagging: true + bucket_website: true + public_access_block: true + transform_location: true + register: bucket_list + + - name: Assert that buckets list contains requested facts + assert: + that: + - item.name is search(name_pattern) + - item.bucket_accelerate_configuration is defined + - item.bucket_acl is defined + - item.bucket_cors is defined + - item.bucket_encryption is defined + - item.bucket_lifecycle_configuration is defined + - item.bucket_location is defined + - item.bucket_logging is defined + - item.bucket_notification_configuration is defined + - item.bucket_ownership_controls is defined + - item.bucket_policy is defined + - item.bucket_policy_status is defined + - item.bucket_replication is defined + - item.bucket_request_payment is defined + - item.bucket_tagging is defined + - item.bucket_website is defined + - item.public_access_block is defined + loop: "{{ bucket_list.buckets }}" + + always: + - name: Delete simple s3_buckets + s3_bucket: + name: "{{ item }}" + state: present + loop: "{{ testing_buckets }}" From d52b7bc6547eeeb6eadf7c670446c16103308e68 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 17:02:45 +0200 Subject: [PATCH 03/63] Code linting - shippable --- plugins/modules/aws_s3_bucket_info.py | 56 +++++++++++++++------------ 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 451eba88a0b..335e1b30392 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -36,8 +36,12 @@ description: - Retrieve requested S3 bucket detailed information - Each bucket_X option executes one API call, hence many options=true will case slower module execution - - You can limit buckets by using I(name) or I(name_filter) option + - You can limit buckets by using I(name) or I(name_filter) option suboptions: + bucket_accelerate_configuration: + description: Retrive S3 accelerate configuration + type: bool + default: False bucket_location: description: Retrive S3 bucket location type: bool @@ -58,10 +62,6 @@ description: Retrive S3 bucket request payment type: bool default: False - bucket_analytics_configuration: - description: Retrive S3 bucket analytics configuration - type: bool - default: False bucket_tagging: description: Retrive S3 bucket tagging type: bool @@ -168,16 +168,19 @@ bucket_location: dictionary data bucket_cors: dictionary data # ...etc + type: list # if name options was specified -bucket_name: +bucket_name: description: "Name of the bucket requested" sample: "my_bucket" + type: str # if name_filter was specified -bucket_name_filter: +bucket_name_filter: description: "String to match bucket name" sample: "buckets_prefix" + type: str ''' try: @@ -263,6 +266,7 @@ def get_bucket_details(connection, name, requested_facts, transform_location): return(all_facts) + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_location(name, connection, transform_location=False): """ @@ -284,6 +288,7 @@ def get_bucket_location(name, connection, transform_location=False): except KeyError: return(data) + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ @@ -310,23 +315,23 @@ def main(): name=dict(type=str, default=""), name_filter=dict(type=str, default=""), bucket_facts=dict(type='dict', options=dict( - bucket_accelerate_configuration=dict(type=bool, default=False), - bucket_acl=dict(type=bool, default=False), - bucket_cors=dict(type=bool, default=False), - bucket_encryption=dict(type=bool, default=False), - bucket_lifecycle_configuration=dict(type=bool, default=False), - bucket_notification_configuration=dict(type=bool, default=False), - bucket_location=dict(type=bool, default=False), - bucket_logging=dict(type=bool, default=False), - bucket_ownership_controls=dict(type=bool, default=False), - bucket_policy=dict(type=bool, default=False), - bucket_policy_status=dict(type=bool, default=False), - bucket_replication=dict(type=bool, default=False), - bucket_request_payment=dict(type=bool, default=False), - bucket_tagging=dict(type=bool, default=False), - bucket_website=dict(type=bool, default=False), - public_access_block=dict(type=bool, default=False), - )), + bucket_accelerate_configuration=dict(type='bool', default=False), + bucket_acl=dict(type='bool', default=False), + bucket_cors=dict(type='bool', default=False), + bucket_encryption=dict(type='bool', default=False), + bucket_lifecycle_configuration=dict(type='bool', default=False), + bucket_location=dict(type='bool', default=False), + bucket_logging=dict(type='bool', default=False), + bucket_notification_configuration=dict(type='bool', default=False), + bucket_ownership_controls=dict(type='bool', default=False), + bucket_policy=dict(type='bool', default=False), + bucket_policy_status=dict(type='bool', default=False), + bucket_replication=dict(type='bool', default=False), + bucket_request_payment=dict(type='bool', default=False), + bucket_tagging=dict(type='bool', default=False), + bucket_website=dict(type='bool', default=False), + public_access_block=dict(type='bool', default=False), + )), transform_location=dict(type='bool', default=False) ) @@ -380,6 +385,7 @@ def main(): else: module.exit_json(msg="Retrieved s3 info.", **result) -## MAIN ## + +# MAIN if __name__ == '__main__': main() From 4995fee38e03fc45ce443e676dab7fca38fdf687 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 17:09:01 +0200 Subject: [PATCH 04/63] Fixing option type --- plugins/modules/aws_s3_bucket_info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 335e1b30392..c4122589a41 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -312,8 +312,8 @@ def main(): :return: """ argument_spec = dict( - name=dict(type=str, default=""), - name_filter=dict(type=str, default=""), + name=dict(type='str', default=""), + name_filter=dict(type='str', default=""), bucket_facts=dict(type='dict', options=dict( bucket_accelerate_configuration=dict(type='bool', default=False), bucket_acl=dict(type='bool', default=False), From 9d5f9badb1a27d6f847e597cd48eca03ade792b4 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 17:14:16 +0200 Subject: [PATCH 05/63] Fixing RETURN / Shippable --- plugins/modules/aws_s3_bucket_info.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index c4122589a41..1e8fb02e273 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -169,18 +169,6 @@ bucket_cors: dictionary data # ...etc type: list - -# if name options was specified -bucket_name: - description: "Name of the bucket requested" - sample: "my_bucket" - type: str - -# if name_filter was specified -bucket_name_filter: - description: "String to match bucket name" - sample: "buckets_prefix" - type: str ''' try: From ff514e19e87da92261e7e186bbe67ca236f3130f Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 18 Nov 2020 09:19:40 +0100 Subject: [PATCH 06/63] Sync with upstream + .gitignore VSCode --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 6058f0fa338..3b4462815fc 100644 --- a/.gitignore +++ b/.gitignore @@ -387,4 +387,7 @@ $RECYCLE.BIN/ # Antsibull-changelog changelogs/.plugin-cache.yaml +# Visual Studio Code +.vscode/* + # End of https://www.gitignore.io/api/git,linux,pydev,python,windows,pycharm+all,jupyternotebook,vim,webstorm,emacs,dotenv From d5ea3f5b583a918249a39f23db2d4f35a7686178 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 18 Nov 2020 10:32:14 +0100 Subject: [PATCH 07/63] Extending integration test --- .../targets/aws_s3_bucket_info/tasks/main.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 9e07a021372..dd5fc672407 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -11,6 +11,9 @@ s3_bucket: name: "{{ item }}" state: present + tags: + integration: "verified" + facts: "defined" register: output loop: "{{ testing_buckets }}" @@ -47,7 +50,7 @@ transform_location: true register: bucket_list - - name: Assert that buckets list contains requested facts + - name: Assert that buckets list contains requested bucket facts assert: that: - item.name is search(name_pattern) @@ -68,6 +71,20 @@ - item.bucket_website is defined - item.public_access_block is defined loop: "{{ bucket_list.buckets }}" + loop_control: + label: "{{ item.name }}" + + - name: Assert that retrieved bucket facts contains valid data + assert: + that: + - item.bucket_acl.Owner is defined + - item.bucket_location.LocationConstraint is search(aws_region) + - item.bucket_tagging.TagSet is defined + - item.bucket_tagging.TagSet is search("defined") + - item.bucket_tagging.TagSet is search("verified") + loop: "{{ bucket_list.buckets }}" + loop_control: + label: "{{ item.name }}" always: - name: Delete simple s3_buckets From 865604d4d9eb19f6271ef784910218b66897b13d Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 18 Nov 2020 10:41:28 +0100 Subject: [PATCH 08/63] Adding version_added to new module options --- plugins/modules/aws_s3_bucket_info.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 1e8fb02e273..2f2de2dc033 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -32,6 +32,7 @@ - Get info only about buckets name matching defined string type: str default: "" + version_added: 1.3.0 bucket_facts: description: - Retrieve requested S3 bucket detailed information @@ -103,6 +104,7 @@ type: bool default: False type: dict + version_added: 1.3.0 transform_location: description: - S3 bucket location for default us-east-1 is normally reported as 'null' @@ -110,6 +112,7 @@ - affects only queries with I(bucket_facts) > I(bucket_location) = true type: bool default: False + version_added: 1.3.0 extends_documentation_fragment: - amazon.aws.aws From f605e75c5e98423e76dcb0488b72f8b4b63017ba Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 18 Nov 2020 16:00:56 +0100 Subject: [PATCH 09/63] Prepare 1.3.0 changelog --- changelogs/changelog.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/changelogs/changelog.yaml b/changelogs/changelog.yaml index 25102ab8cc6..6732b27f5e8 100644 --- a/changelogs/changelog.yaml +++ b/changelogs/changelog.yaml @@ -785,6 +785,7 @@ releases: - redshift - add support for setting tags. - s3_lifecycle - Add support for intelligent tiering and deep archive storage classes (https://github.com/ansible-collections/community.aws/issues/270) + - aws_s3_bucket_info - new module options ``name:``, ``name_filter``, ``bucket_facts`` and ``transform_location`` (https://github.com/ansible-collections/community.aws/pull/260). fragments: - 244-rds_instance-no_log.yml - 264-fix-elemt-type-for-containers-in-ecs_taskdefinition.yml From 908a336561c1fc49c78c4b432312d6e0a6f5632a Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Thu, 19 Nov 2020 14:52:39 +0100 Subject: [PATCH 10/63] Use boto3_tag_list_to_ansible_dict for bucket tags --- plugins/modules/aws_s3_bucket_info.py | 31 +++++++++++++++++-- .../targets/aws_s3_bucket_info/tasks/main.yml | 5 ++- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 2f2de2dc033..4992fac15bb 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -195,6 +195,7 @@ def get_bucket_list(module, connection, name="", name_filter=""): buckets = [] filtered_buckets = [] final_buckets = [] + # Get all buckets try: buckets = camel_dict_to_snake_dict(connection.list_buckets())['buckets'] @@ -247,6 +248,13 @@ def get_bucket_details(connection, name, requested_facts, transform_location): # we just pass on error - error means that resources is undefined except botocore.exceptions.ClientError: pass + elif key == 'bucket_tagging': + all_facts[key] = {} + try: + all_facts[key] = get_bucket_tagging(name, connection) + # we just pass on error - error means that resources is undefined + except botocore.exceptions.ClientError: + pass else: all_facts[key] = {} try: @@ -280,6 +288,25 @@ def get_bucket_location(name, connection, transform_location=False): return(data) +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) +def get_bucket_tagging(name, connection): + """ + Get bucket tags and transform them using `boto3_tag_list_to_ansible_dict` function + """ + data = connection.get_bucket_tagging(Bucket=name) + + try: + bucket_tags = boto3_tag_list_to_ansible_dict(data['TagSet']) + return(bucket_tags) + except KeyError: + # Strip response metadata (not needed) + try: + data.pop('ResponseMetadata') + return(data) + except KeyError: + return(data) + + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ @@ -351,8 +378,8 @@ def main(): connection = {} try: connection = module.client('s3') - except (connection.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg='Failed to connect to AWS') + except (connection.exceptions.ClientError, botocore.exceptions.BotoCoreError) as err_code: + module.fail_json_aws(err_code, msg='Failed to connect to AWS') # Get basic bucket list (name + creation date) bucket_list = get_bucket_list(module, connection, name, name_filter) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index dd5fc672407..a7510edb5de 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -79,9 +79,8 @@ that: - item.bucket_acl.Owner is defined - item.bucket_location.LocationConstraint is search(aws_region) - - item.bucket_tagging.TagSet is defined - - item.bucket_tagging.TagSet is search("defined") - - item.bucket_tagging.TagSet is search("verified") + - item.bucket_tagging.facts is search("defined") + - item.bucket_tagging.integration is search("verified") loop: "{{ bucket_list.buckets }}" loop_control: label: "{{ item.name }}" From 202edd11769505e3314cddf932566509270ad6b8 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Thu, 19 Nov 2020 14:57:08 +0100 Subject: [PATCH 11/63] Fixing integration test (delete buckets) --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index a7510edb5de..0fcf081ae6a 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -89,5 +89,5 @@ - name: Delete simple s3_buckets s3_bucket: name: "{{ item }}" - state: present + state: absent loop: "{{ testing_buckets }}" From 158fb65279e835ff955ed69ffbe29fcfcf8c2ca3 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Fri, 20 Nov 2020 10:59:32 +0100 Subject: [PATCH 12/63] Adjust integration test (python 2.7) --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 0fcf081ae6a..f8b9dd81408 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -78,9 +78,9 @@ assert: that: - item.bucket_acl.Owner is defined - - item.bucket_location.LocationConstraint is search(aws_region) - - item.bucket_tagging.facts is search("defined") - - item.bucket_tagging.integration is search("verified") + - item.bucket_location.LocationConstraint is defined + - item.bucket_tagging.facts is defined + - item.bucket_tagging.integration is defined loop: "{{ bucket_list.buckets }}" loop_control: label: "{{ item.name }}" From 479904f91d005ca19e94328c71bd05a206df118c Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Fri, 20 Nov 2020 12:52:59 +0100 Subject: [PATCH 13/63] fix integratiuon test -2 --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index f8b9dd81408..17b8d422976 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -78,7 +78,6 @@ assert: that: - item.bucket_acl.Owner is defined - - item.bucket_location.LocationConstraint is defined - item.bucket_tagging.facts is defined - item.bucket_tagging.integration is defined loop: "{{ bucket_list.buckets }}" From aa0cab34f4dfa81aa49d0f402baf77921afa8f1b Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Thu, 17 Dec 2020 12:40:42 +0100 Subject: [PATCH 14/63] Adjusting changelogs/changelog.yaml --- changelogs/changelog.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/changelogs/changelog.yaml b/changelogs/changelog.yaml index 6732b27f5e8..882a31a01e8 100644 --- a/changelogs/changelog.yaml +++ b/changelogs/changelog.yaml @@ -785,7 +785,8 @@ releases: - redshift - add support for setting tags. - s3_lifecycle - Add support for intelligent tiering and deep archive storage classes (https://github.com/ansible-collections/community.aws/issues/270) - - aws_s3_bucket_info - new module options ``name:``, ``name_filter``, ``bucket_facts`` and ``transform_location`` (https://github.com/ansible-collections/community.aws/pull/260). + - aws_s3_bucket_info - new module options ``name:``, ``name_filter``, ``bucket_facts`` and ``transform_location`` + (https://github.com/ansible-collections/community.aws/pull/260). fragments: - 244-rds_instance-no_log.yml - 264-fix-elemt-type-for-containers-in-ecs_taskdefinition.yml From 5dc71dbed7dc0869ab263c68a8ef6dd11655f56e Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Fri, 9 Oct 2020 17:23:29 +0200 Subject: [PATCH 15/63] WIP - adding new functionality to aws_s3_bucket_info module --- plugins/modules/aws_s3_bucket_info.py | 138 ++++++++++++++------------ 1 file changed, 72 insertions(+), 66 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 4992fac15bb..d099c465f12 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -24,23 +24,20 @@ options: name: description: - - Get info only about specified bucket + - Get info about specified bucket type: str default: "" name_filter: description: - - Get info only about buckets name matching defined string + - Get info about buckets name matching defined string type: str default: "" - version_added: 1.3.0 bucket_facts: description: - Retrieve requested S3 bucket detailed information - - Each bucket_X option executes one API call, hence many options=true will case slower module execution - - You can limit buckets by using I(name) or I(name_filter) option suboptions: bucket_accelerate_configuration: - description: Retrive S3 accelerate configuration + description: Retrive S3 bucket accelerate configuration type: bool default: False bucket_location: @@ -63,6 +60,14 @@ description: Retrive S3 bucket request payment type: bool default: False + bucket_analytics_configuration: + description: Retrive S3 bucket analytics configuration + type: bool + default: False + bucket_metrics_configuration: + description: Retrive S3 bucket metrics configuration + type: bool + default: False bucket_tagging: description: Retrive S3 bucket tagging type: bool @@ -87,6 +92,10 @@ description: Retrive S3 bucket website type: bool default: False + bucket_inventory_configuration: + description: Retrive S3 bucket inventory configuration + type: bool + default: False bucket_policy: description: Retrive S3 bucket policy type: bool @@ -99,20 +108,13 @@ description: Retrive S3 bucket lifecycle configuration type: bool default: False - public_access_block: - description: Retrive S3 bucket public access block - type: bool - default: False type: dict - version_added: 1.3.0 transform_location: description: - S3 bucket location for default us-east-1 is normally reported as 'null' - setting this option to 'true' will return 'us-east-1' instead - - affects only queries with I(bucket_facts) > I(bucket_location) = true type: bool default: False - version_added: 1.3.0 extends_documentation_fragment: - amazon.aws.aws @@ -195,7 +197,6 @@ def get_bucket_list(module, connection, name="", name_filter=""): buckets = [] filtered_buckets = [] final_buckets = [] - # Get all buckets try: buckets = camel_dict_to_snake_dict(connection.list_buckets())['buckets'] @@ -220,6 +221,24 @@ def get_bucket_list(module, connection, name="", name_filter=""): return(final_buckets) + # Filter buckets if requested + if name_filter: + for bucket in buckets: + if name_filter in bucket['name']: + filtered_buckets.append(bucket) + elif name: + for bucket in buckets: + if name == bucket['name']: + filtered_buckets.append(bucket) + + # Return proper list (filtered or all) + if filtered_buckets: + final_buckets = filtered_buckets + else: + final_buckets = buckets + return(final_buckets) + + def get_buckets_facts(connection, buckets, requested_facts, transform_location): """ Retrive additional information about S3 buckets @@ -242,45 +261,29 @@ def get_bucket_details(connection, name, requested_facts, transform_location): for key in requested_facts: if requested_facts[key]: if key == 'bucket_location': - all_facts[key] = {} - try: - all_facts[key] = get_bucket_location(name, connection, transform_location) - # we just pass on error - error means that resources is undefined - except botocore.exceptions.ClientError: - pass - elif key == 'bucket_tagging': - all_facts[key] = {} - try: - all_facts[key] = get_bucket_tagging(name, connection) - # we just pass on error - error means that resources is undefined - except botocore.exceptions.ClientError: - pass + all_facts[key] = get_bucket_location(name, connection, transform_location) else: - all_facts[key] = {} - try: - all_facts[key] = get_bucket_property(name, connection, key) - # we just pass on error - error means that resources is undefined - except botocore.exceptions.ClientError: - pass + all_facts[key] = get_bucket_property(name, connection, key) return(all_facts) -@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_location(name, connection, transform_location=False): """ Get bucket location and optionally transform 'null' to 'us-east-1' """ - data = connection.get_bucket_location(Bucket=name) + try: + data = connection.get_bucket_location(Bucket=name) + except botocore.exceptions.ClientError as err_msg: + data = {'error': err_msg} - # Replace 'null' with 'us-east-1'? if transform_location: try: if not data['LocationConstraint']: data['LocationConstraint'] = 'us-east-1' except KeyError: - pass - # Strip response metadata (not needed) + data['transform_failed'] = True + try: data.pop('ResponseMetadata') return(data) @@ -288,35 +291,17 @@ def get_bucket_location(name, connection, transform_location=False): return(data) -@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) -def get_bucket_tagging(name, connection): - """ - Get bucket tags and transform them using `boto3_tag_list_to_ansible_dict` function - """ - data = connection.get_bucket_tagging(Bucket=name) - - try: - bucket_tags = boto3_tag_list_to_ansible_dict(data['TagSet']) - return(bucket_tags) - except KeyError: - # Strip response metadata (not needed) - try: - data.pop('ResponseMetadata') - return(data) - except KeyError: - return(data) - - -@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ Get bucket property """ api_call = "get_" + get_api_name api_function = getattr(connection, api_call) - data = api_function(Bucket=name) + try: + data = api_function(Bucket=name) + except botocore.exceptions.ClientError as err_msg: + data = {'error': err_msg} - # Strip response metadata (not needed) try: data.pop('ResponseMetadata') return(data) @@ -353,6 +338,33 @@ def main(): transform_location=dict(type='bool', default=False) ) + argument_spec = dict( + name=dict(type=str, default=""), + name_filter=dict(type=str, default=""), + bucket_facts=dict(type='dict', options=dict( + bucket_accelerate_configuration=dict(type=bool, default=False), + bucket_acl=dict(type=bool, default=False), + bucket_analytics_configuration=dict(type=bool, default=False), + bucket_cors=dict(type=bool, default=False), + bucket_encryption=dict(type=bool, default=False), + bucket_inventory_configuration=dict(type=bool, default=False), + bucket_lifecycle_configuration=dict(type=bool, default=False), + bucket_location=dict(type=bool, default=False), + bucket_logging=dict(type=bool, default=False), + bucket_metrics_configuration=dict(type=bool, default=False), + bucket_notification_configuration=dict(type=bool, default=False), + bucket_ownership_controls=dict(type=bool, default=False), + bucket_policy=dict(type=bool, default=False), + bucket_policy_status=dict(type=bool, default=False), + bucket_replication=dict(type=bool, default=False), + bucket_request_payment=dict(type=bool, default=False), + bucket_tagging=dict(type=bool, default=False), + bucket_website=dict(type=bool, default=False), + )), + transform_location=dict(type='bool', default=False) + ) + + # Ensure we have an empty dict result = {} @@ -384,12 +396,6 @@ def main(): # Get basic bucket list (name + creation date) bucket_list = get_bucket_list(module, connection, name, name_filter) - # Add information about name/name_filter to result - if name: - result['bucket_name'] = name - elif name_filter: - result['bucket_name_filter'] = name_filter - # Gather detailed information about buckets if requested bucket_facts = module.params.get("bucket_facts") if bucket_facts: From 924df2d552fb743d8fd51108895b102d9d9a0b7d Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 16:06:17 +0200 Subject: [PATCH 16/63] Fixing exceptions adding integration tests --- plugins/modules/aws_s3_bucket_info.py | 110 ++++++++---------- .../targets/aws_s3_bucket_info/tasks/main.yml | 19 +-- 2 files changed, 51 insertions(+), 78 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index d099c465f12..a1ee37d5b3e 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -24,22 +24,20 @@ options: name: description: - - Get info about specified bucket + - Get info only about specified bucket type: str default: "" name_filter: description: - - Get info about buckets name matching defined string + - Get info only about buckets name matching defined string type: str default: "" bucket_facts: description: - Retrieve requested S3 bucket detailed information + - Each bucket_X option executes one API call, hence many options=true will case slower module execution + - You can limit buckets by using I(name) or I(name_filter) option suboptions: - bucket_accelerate_configuration: - description: Retrive S3 bucket accelerate configuration - type: bool - default: False bucket_location: description: Retrive S3 bucket location type: bool @@ -64,10 +62,6 @@ description: Retrive S3 bucket analytics configuration type: bool default: False - bucket_metrics_configuration: - description: Retrive S3 bucket metrics configuration - type: bool - default: False bucket_tagging: description: Retrive S3 bucket tagging type: bool @@ -92,10 +86,6 @@ description: Retrive S3 bucket website type: bool default: False - bucket_inventory_configuration: - description: Retrive S3 bucket inventory configuration - type: bool - default: False bucket_policy: description: Retrive S3 bucket policy type: bool @@ -108,11 +98,16 @@ description: Retrive S3 bucket lifecycle configuration type: bool default: False + public_access_block: + description: Retrive S3 bucket public access block + type: bool + default: False type: dict transform_location: description: - S3 bucket location for default us-east-1 is normally reported as 'null' - setting this option to 'true' will return 'us-east-1' instead + - affects only queries with I(bucket_facts) > I(bucket_location) = true type: bool default: False @@ -173,7 +168,16 @@ bucket_location: dictionary data bucket_cors: dictionary data # ...etc - type: list + +# if name options was specified +bucket_name: + description: "Name of the bucket requested" + sample: "my_bucket" + +# if name_filter was specified +bucket_name_filter: + description: "String to match bucket name" + sample: "buckets_prefix" ''' try: @@ -232,7 +236,7 @@ def get_bucket_list(module, connection, name="", name_filter=""): filtered_buckets.append(bucket) # Return proper list (filtered or all) - if filtered_buckets: + if name or name_filter: final_buckets = filtered_buckets else: final_buckets = buckets @@ -261,47 +265,53 @@ def get_bucket_details(connection, name, requested_facts, transform_location): for key in requested_facts: if requested_facts[key]: if key == 'bucket_location': - all_facts[key] = get_bucket_location(name, connection, transform_location) + all_facts[key] = {} + try: + all_facts[key] = get_bucket_location(name, connection, transform_location) + # we just pass on error - error means that resources is undefined + except botocore.exceptions.ClientError: + pass else: - all_facts[key] = get_bucket_property(name, connection, key) + all_facts[key] = {} + try: + all_facts[key] = get_bucket_property(name, connection, key) + # we just pass on error - error means that resources is undefined + except botocore.exceptions.ClientError: + pass return(all_facts) - +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_location(name, connection, transform_location=False): """ Get bucket location and optionally transform 'null' to 'us-east-1' """ - try: - data = connection.get_bucket_location(Bucket=name) - except botocore.exceptions.ClientError as err_msg: - data = {'error': err_msg} + data = connection.get_bucket_location(Bucket=name) + # Replace 'null' with 'us-east-1'? if transform_location: try: if not data['LocationConstraint']: data['LocationConstraint'] = 'us-east-1' except KeyError: - data['transform_failed'] = True - + pass + # Strip response metadata (not needed) try: data.pop('ResponseMetadata') return(data) except KeyError: return(data) - +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ Get bucket property """ api_call = "get_" + get_api_name api_function = getattr(connection, api_call) - try: - data = api_function(Bucket=name) - except botocore.exceptions.ClientError as err_msg: - data = {'error': err_msg} + data = api_function(Bucket=name) + # Strip response metadata (not needed) try: data.pop('ResponseMetadata') return(data) @@ -314,45 +324,18 @@ def main(): Get list of S3 buckets :return: """ - argument_spec = dict( - name=dict(type='str', default=""), - name_filter=dict(type='str', default=""), - bucket_facts=dict(type='dict', options=dict( - bucket_accelerate_configuration=dict(type='bool', default=False), - bucket_acl=dict(type='bool', default=False), - bucket_cors=dict(type='bool', default=False), - bucket_encryption=dict(type='bool', default=False), - bucket_lifecycle_configuration=dict(type='bool', default=False), - bucket_location=dict(type='bool', default=False), - bucket_logging=dict(type='bool', default=False), - bucket_notification_configuration=dict(type='bool', default=False), - bucket_ownership_controls=dict(type='bool', default=False), - bucket_policy=dict(type='bool', default=False), - bucket_policy_status=dict(type='bool', default=False), - bucket_replication=dict(type='bool', default=False), - bucket_request_payment=dict(type='bool', default=False), - bucket_tagging=dict(type='bool', default=False), - bucket_website=dict(type='bool', default=False), - public_access_block=dict(type='bool', default=False), - )), - transform_location=dict(type='bool', default=False) - ) - argument_spec = dict( name=dict(type=str, default=""), name_filter=dict(type=str, default=""), bucket_facts=dict(type='dict', options=dict( bucket_accelerate_configuration=dict(type=bool, default=False), bucket_acl=dict(type=bool, default=False), - bucket_analytics_configuration=dict(type=bool, default=False), bucket_cors=dict(type=bool, default=False), bucket_encryption=dict(type=bool, default=False), - bucket_inventory_configuration=dict(type=bool, default=False), bucket_lifecycle_configuration=dict(type=bool, default=False), + bucket_notification_configuration=dict(type=bool, default=False), bucket_location=dict(type=bool, default=False), bucket_logging=dict(type=bool, default=False), - bucket_metrics_configuration=dict(type=bool, default=False), - bucket_notification_configuration=dict(type=bool, default=False), bucket_ownership_controls=dict(type=bool, default=False), bucket_policy=dict(type=bool, default=False), bucket_policy_status=dict(type=bool, default=False), @@ -360,11 +343,11 @@ def main(): bucket_request_payment=dict(type=bool, default=False), bucket_tagging=dict(type=bool, default=False), bucket_website=dict(type=bool, default=False), + public_access_block=dict(type=bool, default=False), )), transform_location=dict(type='bool', default=False) ) - # Ensure we have an empty dict result = {} @@ -396,6 +379,12 @@ def main(): # Get basic bucket list (name + creation date) bucket_list = get_bucket_list(module, connection, name, name_filter) + # Add information about name/name_filter to result + if name: + result['bucket_name'] = name + elif name_filter: + result['bucket_name_filter'] = name_filter + # Gather detailed information about buckets if requested bucket_facts = module.params.get("bucket_facts") if bucket_facts: @@ -409,7 +398,6 @@ def main(): else: module.exit_json(msg="Retrieved s3 info.", **result) - -# MAIN +## MAIN ## if __name__ == '__main__': main() diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 17b8d422976..9e07a021372 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -11,9 +11,6 @@ s3_bucket: name: "{{ item }}" state: present - tags: - integration: "verified" - facts: "defined" register: output loop: "{{ testing_buckets }}" @@ -50,7 +47,7 @@ transform_location: true register: bucket_list - - name: Assert that buckets list contains requested bucket facts + - name: Assert that buckets list contains requested facts assert: that: - item.name is search(name_pattern) @@ -71,22 +68,10 @@ - item.bucket_website is defined - item.public_access_block is defined loop: "{{ bucket_list.buckets }}" - loop_control: - label: "{{ item.name }}" - - - name: Assert that retrieved bucket facts contains valid data - assert: - that: - - item.bucket_acl.Owner is defined - - item.bucket_tagging.facts is defined - - item.bucket_tagging.integration is defined - loop: "{{ bucket_list.buckets }}" - loop_control: - label: "{{ item.name }}" always: - name: Delete simple s3_buckets s3_bucket: name: "{{ item }}" - state: absent + state: present loop: "{{ testing_buckets }}" From 646a93ee848f12c0242a45ef4c938046abac6131 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 17:02:45 +0200 Subject: [PATCH 17/63] Code linting - shippable --- plugins/modules/aws_s3_bucket_info.py | 56 +++++++++++++++------------ 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index a1ee37d5b3e..ef4797aef01 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -36,8 +36,12 @@ description: - Retrieve requested S3 bucket detailed information - Each bucket_X option executes one API call, hence many options=true will case slower module execution - - You can limit buckets by using I(name) or I(name_filter) option + - You can limit buckets by using I(name) or I(name_filter) option suboptions: + bucket_accelerate_configuration: + description: Retrive S3 accelerate configuration + type: bool + default: False bucket_location: description: Retrive S3 bucket location type: bool @@ -58,10 +62,6 @@ description: Retrive S3 bucket request payment type: bool default: False - bucket_analytics_configuration: - description: Retrive S3 bucket analytics configuration - type: bool - default: False bucket_tagging: description: Retrive S3 bucket tagging type: bool @@ -168,16 +168,19 @@ bucket_location: dictionary data bucket_cors: dictionary data # ...etc + type: list # if name options was specified -bucket_name: +bucket_name: description: "Name of the bucket requested" sample: "my_bucket" + type: str # if name_filter was specified -bucket_name_filter: +bucket_name_filter: description: "String to match bucket name" sample: "buckets_prefix" + type: str ''' try: @@ -281,6 +284,7 @@ def get_bucket_details(connection, name, requested_facts, transform_location): return(all_facts) + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_location(name, connection, transform_location=False): """ @@ -302,6 +306,7 @@ def get_bucket_location(name, connection, transform_location=False): except KeyError: return(data) + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ @@ -328,23 +333,23 @@ def main(): name=dict(type=str, default=""), name_filter=dict(type=str, default=""), bucket_facts=dict(type='dict', options=dict( - bucket_accelerate_configuration=dict(type=bool, default=False), - bucket_acl=dict(type=bool, default=False), - bucket_cors=dict(type=bool, default=False), - bucket_encryption=dict(type=bool, default=False), - bucket_lifecycle_configuration=dict(type=bool, default=False), - bucket_notification_configuration=dict(type=bool, default=False), - bucket_location=dict(type=bool, default=False), - bucket_logging=dict(type=bool, default=False), - bucket_ownership_controls=dict(type=bool, default=False), - bucket_policy=dict(type=bool, default=False), - bucket_policy_status=dict(type=bool, default=False), - bucket_replication=dict(type=bool, default=False), - bucket_request_payment=dict(type=bool, default=False), - bucket_tagging=dict(type=bool, default=False), - bucket_website=dict(type=bool, default=False), - public_access_block=dict(type=bool, default=False), - )), + bucket_accelerate_configuration=dict(type='bool', default=False), + bucket_acl=dict(type='bool', default=False), + bucket_cors=dict(type='bool', default=False), + bucket_encryption=dict(type='bool', default=False), + bucket_lifecycle_configuration=dict(type='bool', default=False), + bucket_location=dict(type='bool', default=False), + bucket_logging=dict(type='bool', default=False), + bucket_notification_configuration=dict(type='bool', default=False), + bucket_ownership_controls=dict(type='bool', default=False), + bucket_policy=dict(type='bool', default=False), + bucket_policy_status=dict(type='bool', default=False), + bucket_replication=dict(type='bool', default=False), + bucket_request_payment=dict(type='bool', default=False), + bucket_tagging=dict(type='bool', default=False), + bucket_website=dict(type='bool', default=False), + public_access_block=dict(type='bool', default=False), + )), transform_location=dict(type='bool', default=False) ) @@ -398,6 +403,7 @@ def main(): else: module.exit_json(msg="Retrieved s3 info.", **result) -## MAIN ## + +# MAIN if __name__ == '__main__': main() From 2dade705ddaca3b4c84cec65dc81b50e22527849 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 17:09:01 +0200 Subject: [PATCH 18/63] Fixing option type --- plugins/modules/aws_s3_bucket_info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index ef4797aef01..bb6e8af82ac 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -330,8 +330,8 @@ def main(): :return: """ argument_spec = dict( - name=dict(type=str, default=""), - name_filter=dict(type=str, default=""), + name=dict(type='str', default=""), + name_filter=dict(type='str', default=""), bucket_facts=dict(type='dict', options=dict( bucket_accelerate_configuration=dict(type='bool', default=False), bucket_acl=dict(type='bool', default=False), From 444fd57c0371fb818fd430273d70aadaa8e7122e Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 17:14:16 +0200 Subject: [PATCH 19/63] Fixing RETURN / Shippable --- plugins/modules/aws_s3_bucket_info.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index bb6e8af82ac..4c705d2c3fc 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -169,18 +169,6 @@ bucket_cors: dictionary data # ...etc type: list - -# if name options was specified -bucket_name: - description: "Name of the bucket requested" - sample: "my_bucket" - type: str - -# if name_filter was specified -bucket_name_filter: - description: "String to match bucket name" - sample: "buckets_prefix" - type: str ''' try: From 1b6cda149c96a722b21fa39e3d53e072d6bf1461 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 18 Nov 2020 10:32:14 +0100 Subject: [PATCH 20/63] Extending integration test --- .../targets/aws_s3_bucket_info/tasks/main.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 9e07a021372..dd5fc672407 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -11,6 +11,9 @@ s3_bucket: name: "{{ item }}" state: present + tags: + integration: "verified" + facts: "defined" register: output loop: "{{ testing_buckets }}" @@ -47,7 +50,7 @@ transform_location: true register: bucket_list - - name: Assert that buckets list contains requested facts + - name: Assert that buckets list contains requested bucket facts assert: that: - item.name is search(name_pattern) @@ -68,6 +71,20 @@ - item.bucket_website is defined - item.public_access_block is defined loop: "{{ bucket_list.buckets }}" + loop_control: + label: "{{ item.name }}" + + - name: Assert that retrieved bucket facts contains valid data + assert: + that: + - item.bucket_acl.Owner is defined + - item.bucket_location.LocationConstraint is search(aws_region) + - item.bucket_tagging.TagSet is defined + - item.bucket_tagging.TagSet is search("defined") + - item.bucket_tagging.TagSet is search("verified") + loop: "{{ bucket_list.buckets }}" + loop_control: + label: "{{ item.name }}" always: - name: Delete simple s3_buckets From 6a0d5c3425c7ab2f1bca019634026c3ac60a12bb Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 18 Nov 2020 10:41:28 +0100 Subject: [PATCH 21/63] Adding version_added to new module options --- plugins/modules/aws_s3_bucket_info.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 4c705d2c3fc..b4c31428667 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -32,6 +32,7 @@ - Get info only about buckets name matching defined string type: str default: "" + version_added: 1.3.0 bucket_facts: description: - Retrieve requested S3 bucket detailed information @@ -103,6 +104,7 @@ type: bool default: False type: dict + version_added: 1.3.0 transform_location: description: - S3 bucket location for default us-east-1 is normally reported as 'null' @@ -110,6 +112,7 @@ - affects only queries with I(bucket_facts) > I(bucket_location) = true type: bool default: False + version_added: 1.3.0 extends_documentation_fragment: - amazon.aws.aws From 909da2033c659ccfb70b2e00055476f0017f5231 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 18 Nov 2020 16:00:56 +0100 Subject: [PATCH 22/63] Prepare 1.3.0 changelog --- changelogs/changelog.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelogs/changelog.yaml b/changelogs/changelog.yaml index 882a31a01e8..27a4b2c7910 100644 --- a/changelogs/changelog.yaml +++ b/changelogs/changelog.yaml @@ -804,3 +804,5 @@ releases: name: s3_metrics_configuration namespace: '' release_date: '2020-12-10' + minor_changes: + - aws_s3_bucket_info - new module options ``name:``, ``name_filter``, ``bucket_facts`` and ``transform_location`` (https://github.com/ansible-collections/community.aws/pull/260). From 83e1e22b6ae417a4006ae6a5844022cb08d647fe Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Thu, 19 Nov 2020 14:52:39 +0100 Subject: [PATCH 23/63] Use boto3_tag_list_to_ansible_dict for bucket tags --- plugins/modules/aws_s3_bucket_info.py | 45 +++++++++++-------- .../targets/aws_s3_bucket_info/tasks/main.yml | 5 +-- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index b4c31428667..4992fac15bb 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -195,6 +195,7 @@ def get_bucket_list(module, connection, name="", name_filter=""): buckets = [] filtered_buckets = [] final_buckets = [] + # Get all buckets try: buckets = camel_dict_to_snake_dict(connection.list_buckets())['buckets'] @@ -219,24 +220,6 @@ def get_bucket_list(module, connection, name="", name_filter=""): return(final_buckets) - # Filter buckets if requested - if name_filter: - for bucket in buckets: - if name_filter in bucket['name']: - filtered_buckets.append(bucket) - elif name: - for bucket in buckets: - if name == bucket['name']: - filtered_buckets.append(bucket) - - # Return proper list (filtered or all) - if name or name_filter: - final_buckets = filtered_buckets - else: - final_buckets = buckets - return(final_buckets) - - def get_buckets_facts(connection, buckets, requested_facts, transform_location): """ Retrive additional information about S3 buckets @@ -265,6 +248,13 @@ def get_bucket_details(connection, name, requested_facts, transform_location): # we just pass on error - error means that resources is undefined except botocore.exceptions.ClientError: pass + elif key == 'bucket_tagging': + all_facts[key] = {} + try: + all_facts[key] = get_bucket_tagging(name, connection) + # we just pass on error - error means that resources is undefined + except botocore.exceptions.ClientError: + pass else: all_facts[key] = {} try: @@ -298,6 +288,25 @@ def get_bucket_location(name, connection, transform_location=False): return(data) +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) +def get_bucket_tagging(name, connection): + """ + Get bucket tags and transform them using `boto3_tag_list_to_ansible_dict` function + """ + data = connection.get_bucket_tagging(Bucket=name) + + try: + bucket_tags = boto3_tag_list_to_ansible_dict(data['TagSet']) + return(bucket_tags) + except KeyError: + # Strip response metadata (not needed) + try: + data.pop('ResponseMetadata') + return(data) + except KeyError: + return(data) + + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index dd5fc672407..a7510edb5de 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -79,9 +79,8 @@ that: - item.bucket_acl.Owner is defined - item.bucket_location.LocationConstraint is search(aws_region) - - item.bucket_tagging.TagSet is defined - - item.bucket_tagging.TagSet is search("defined") - - item.bucket_tagging.TagSet is search("verified") + - item.bucket_tagging.facts is search("defined") + - item.bucket_tagging.integration is search("verified") loop: "{{ bucket_list.buckets }}" loop_control: label: "{{ item.name }}" From 9fd02b82ea27681c085f4d1713543e4be84b7d7c Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Thu, 19 Nov 2020 14:57:08 +0100 Subject: [PATCH 24/63] Fixing integration test (delete buckets) --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index a7510edb5de..0fcf081ae6a 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -89,5 +89,5 @@ - name: Delete simple s3_buckets s3_bucket: name: "{{ item }}" - state: present + state: absent loop: "{{ testing_buckets }}" From 45706678d607a8ae8f6f5ba247a73d8ca8f01aba Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Fri, 20 Nov 2020 10:59:32 +0100 Subject: [PATCH 25/63] Adjust integration test (python 2.7) --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 0fcf081ae6a..f8b9dd81408 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -78,9 +78,9 @@ assert: that: - item.bucket_acl.Owner is defined - - item.bucket_location.LocationConstraint is search(aws_region) - - item.bucket_tagging.facts is search("defined") - - item.bucket_tagging.integration is search("verified") + - item.bucket_location.LocationConstraint is defined + - item.bucket_tagging.facts is defined + - item.bucket_tagging.integration is defined loop: "{{ bucket_list.buckets }}" loop_control: label: "{{ item.name }}" From 5fa69eabacbb0cff36cffd7069333fda23ec94ec Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Fri, 20 Nov 2020 12:52:59 +0100 Subject: [PATCH 26/63] fix integratiuon test -2 --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index f8b9dd81408..17b8d422976 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -78,7 +78,6 @@ assert: that: - item.bucket_acl.Owner is defined - - item.bucket_location.LocationConstraint is defined - item.bucket_tagging.facts is defined - item.bucket_tagging.integration is defined loop: "{{ bucket_list.buckets }}" From dd33d69cfc3f49876f8ae330f8cddefa23a1edb5 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Thu, 17 Dec 2020 12:40:42 +0100 Subject: [PATCH 27/63] Adjusting changelogs/changelog.yaml --- changelogs/changelog.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/changelogs/changelog.yaml b/changelogs/changelog.yaml index 27a4b2c7910..882a31a01e8 100644 --- a/changelogs/changelog.yaml +++ b/changelogs/changelog.yaml @@ -804,5 +804,3 @@ releases: name: s3_metrics_configuration namespace: '' release_date: '2020-12-10' - minor_changes: - - aws_s3_bucket_info - new module options ``name:``, ``name_filter``, ``bucket_facts`` and ``transform_location`` (https://github.com/ansible-collections/community.aws/pull/260). From 4191e8ecda3b7b7bc146b4642d7b4d41dda374bd Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Fri, 9 Oct 2020 17:23:29 +0200 Subject: [PATCH 28/63] WIP - adding new functionality to aws_s3_bucket_info module --- plugins/modules/aws_s3_bucket_info.py | 144 ++++++++++++++------------ 1 file changed, 75 insertions(+), 69 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 4992fac15bb..5f559ca08bb 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -24,23 +24,20 @@ options: name: description: - - Get info only about specified bucket + - Get info about specified bucket type: str default: "" name_filter: description: - - Get info only about buckets name matching defined string + - Get info about buckets name matching defined string type: str default: "" - version_added: 1.3.0 bucket_facts: description: - Retrieve requested S3 bucket detailed information - - Each bucket_X option executes one API call, hence many options=true will case slower module execution - - You can limit buckets by using I(name) or I(name_filter) option suboptions: bucket_accelerate_configuration: - description: Retrive S3 accelerate configuration + description: Retrive S3 bucket accelerate configuration type: bool default: False bucket_location: @@ -63,6 +60,14 @@ description: Retrive S3 bucket request payment type: bool default: False + bucket_analytics_configuration: + description: Retrive S3 bucket analytics configuration + type: bool + default: False + bucket_metrics_configuration: + description: Retrive S3 bucket metrics configuration + type: bool + default: False bucket_tagging: description: Retrive S3 bucket tagging type: bool @@ -87,6 +92,10 @@ description: Retrive S3 bucket website type: bool default: False + bucket_inventory_configuration: + description: Retrive S3 bucket inventory configuration + type: bool + default: False bucket_policy: description: Retrive S3 bucket policy type: bool @@ -99,20 +108,13 @@ description: Retrive S3 bucket lifecycle configuration type: bool default: False - public_access_block: - description: Retrive S3 bucket public access block - type: bool - default: False type: dict - version_added: 1.3.0 transform_location: description: - S3 bucket location for default us-east-1 is normally reported as 'null' - setting this option to 'true' will return 'us-east-1' instead - - affects only queries with I(bucket_facts) > I(bucket_location) = true type: bool default: False - version_added: 1.3.0 extends_documentation_fragment: - amazon.aws.aws @@ -195,7 +197,6 @@ def get_bucket_list(module, connection, name="", name_filter=""): buckets = [] filtered_buckets = [] final_buckets = [] - # Get all buckets try: buckets = camel_dict_to_snake_dict(connection.list_buckets())['buckets'] @@ -220,6 +221,24 @@ def get_bucket_list(module, connection, name="", name_filter=""): return(final_buckets) + # Filter buckets if requested + if name_filter: + for bucket in buckets: + if name_filter in bucket['name']: + filtered_buckets.append(bucket) + elif name: + for bucket in buckets: + if name == bucket['name']: + filtered_buckets.append(bucket) + + # Return proper list (filtered or all) + if filtered_buckets: + final_buckets = filtered_buckets + else: + final_buckets = buckets + return(final_buckets) + + def get_buckets_facts(connection, buckets, requested_facts, transform_location): """ Retrive additional information about S3 buckets @@ -242,45 +261,29 @@ def get_bucket_details(connection, name, requested_facts, transform_location): for key in requested_facts: if requested_facts[key]: if key == 'bucket_location': - all_facts[key] = {} - try: - all_facts[key] = get_bucket_location(name, connection, transform_location) - # we just pass on error - error means that resources is undefined - except botocore.exceptions.ClientError: - pass - elif key == 'bucket_tagging': - all_facts[key] = {} - try: - all_facts[key] = get_bucket_tagging(name, connection) - # we just pass on error - error means that resources is undefined - except botocore.exceptions.ClientError: - pass + all_facts[key] = get_bucket_location(name, connection, transform_location) else: - all_facts[key] = {} - try: - all_facts[key] = get_bucket_property(name, connection, key) - # we just pass on error - error means that resources is undefined - except botocore.exceptions.ClientError: - pass + all_facts[key] = get_bucket_property(name, connection, key) return(all_facts) -@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_location(name, connection, transform_location=False): """ Get bucket location and optionally transform 'null' to 'us-east-1' """ - data = connection.get_bucket_location(Bucket=name) + try: + data = connection.get_bucket_location(Bucket=name) + except botocore.exceptions.ClientError as err_msg: + data = {'error': err_msg} - # Replace 'null' with 'us-east-1'? if transform_location: try: if not data['LocationConstraint']: data['LocationConstraint'] = 'us-east-1' except KeyError: - pass - # Strip response metadata (not needed) + data['transform_failed'] = True + try: data.pop('ResponseMetadata') return(data) @@ -288,35 +291,17 @@ def get_bucket_location(name, connection, transform_location=False): return(data) -@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) -def get_bucket_tagging(name, connection): - """ - Get bucket tags and transform them using `boto3_tag_list_to_ansible_dict` function - """ - data = connection.get_bucket_tagging(Bucket=name) - - try: - bucket_tags = boto3_tag_list_to_ansible_dict(data['TagSet']) - return(bucket_tags) - except KeyError: - # Strip response metadata (not needed) - try: - data.pop('ResponseMetadata') - return(data) - except KeyError: - return(data) - - -@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ Get bucket property """ api_call = "get_" + get_api_name api_function = getattr(connection, api_call) - data = api_function(Bucket=name) + try: + data = api_function(Bucket=name) + except botocore.exceptions.ClientError as err_msg: + data = {'error': err_msg} - # Strip response metadata (not needed) try: data.pop('ResponseMetadata') return(data) @@ -353,6 +338,33 @@ def main(): transform_location=dict(type='bool', default=False) ) + argument_spec = dict( + name=dict(type=str, default=""), + name_filter=dict(type=str, default=""), + bucket_facts=dict(type='dict', options=dict( + bucket_accelerate_configuration=dict(type=bool, default=False), + bucket_acl=dict(type=bool, default=False), + bucket_analytics_configuration=dict(type=bool, default=False), + bucket_cors=dict(type=bool, default=False), + bucket_encryption=dict(type=bool, default=False), + bucket_inventory_configuration=dict(type=bool, default=False), + bucket_lifecycle_configuration=dict(type=bool, default=False), + bucket_location=dict(type=bool, default=False), + bucket_logging=dict(type=bool, default=False), + bucket_metrics_configuration=dict(type=bool, default=False), + bucket_notification_configuration=dict(type=bool, default=False), + bucket_ownership_controls=dict(type=bool, default=False), + bucket_policy=dict(type=bool, default=False), + bucket_policy_status=dict(type=bool, default=False), + bucket_replication=dict(type=bool, default=False), + bucket_request_payment=dict(type=bool, default=False), + bucket_tagging=dict(type=bool, default=False), + bucket_website=dict(type=bool, default=False), + )), + transform_location=dict(type='bool', default=False) + ) + + # Ensure we have an empty dict result = {} @@ -362,7 +374,7 @@ def main(): ] # Including ec2 argument spec - module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True, mutually_exclusive=mutually_exclusive) + module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True) is_old_facts = module._name == 'aws_s3_bucket_facts' if is_old_facts: module.deprecate("The 'aws_s3_bucket_facts' module has been renamed to 'aws_s3_bucket_info', " @@ -378,18 +390,12 @@ def main(): connection = {} try: connection = module.client('s3') - except (connection.exceptions.ClientError, botocore.exceptions.BotoCoreError) as err_code: - module.fail_json_aws(err_code, msg='Failed to connect to AWS') + except (connection.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg='Failed to connect to AWS') # Get basic bucket list (name + creation date) bucket_list = get_bucket_list(module, connection, name, name_filter) - # Add information about name/name_filter to result - if name: - result['bucket_name'] = name - elif name_filter: - result['bucket_name_filter'] = name_filter - # Gather detailed information about buckets if requested bucket_facts = module.params.get("bucket_facts") if bucket_facts: From 7b995c4e84a6ec163413436788d4c877246284fa Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 16:06:17 +0200 Subject: [PATCH 29/63] Fixing exceptions adding integration tests --- plugins/modules/aws_s3_bucket_info.py | 112 ++++++++---------- .../targets/aws_s3_bucket_info/tasks/main.yml | 19 +-- 2 files changed, 52 insertions(+), 79 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 5f559ca08bb..96fbdbf71e3 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -24,22 +24,20 @@ options: name: description: - - Get info about specified bucket + - Get info only about specified bucket type: str default: "" name_filter: description: - - Get info about buckets name matching defined string + - Get info only about buckets name matching defined string type: str default: "" bucket_facts: description: - Retrieve requested S3 bucket detailed information + - Each bucket_X option executes one API call, hence many options=true will case slower module execution + - You can limit buckets by using I(name) or I(name_filter) option suboptions: - bucket_accelerate_configuration: - description: Retrive S3 bucket accelerate configuration - type: bool - default: False bucket_location: description: Retrive S3 bucket location type: bool @@ -64,10 +62,6 @@ description: Retrive S3 bucket analytics configuration type: bool default: False - bucket_metrics_configuration: - description: Retrive S3 bucket metrics configuration - type: bool - default: False bucket_tagging: description: Retrive S3 bucket tagging type: bool @@ -92,10 +86,6 @@ description: Retrive S3 bucket website type: bool default: False - bucket_inventory_configuration: - description: Retrive S3 bucket inventory configuration - type: bool - default: False bucket_policy: description: Retrive S3 bucket policy type: bool @@ -108,11 +98,16 @@ description: Retrive S3 bucket lifecycle configuration type: bool default: False + public_access_block: + description: Retrive S3 bucket public access block + type: bool + default: False type: dict transform_location: description: - S3 bucket location for default us-east-1 is normally reported as 'null' - setting this option to 'true' will return 'us-east-1' instead + - affects only queries with I(bucket_facts) > I(bucket_location) = true type: bool default: False @@ -173,7 +168,16 @@ bucket_location: dictionary data bucket_cors: dictionary data # ...etc - type: list + +# if name options was specified +bucket_name: + description: "Name of the bucket requested" + sample: "my_bucket" + +# if name_filter was specified +bucket_name_filter: + description: "String to match bucket name" + sample: "buckets_prefix" ''' try: @@ -232,7 +236,7 @@ def get_bucket_list(module, connection, name="", name_filter=""): filtered_buckets.append(bucket) # Return proper list (filtered or all) - if filtered_buckets: + if name or name_filter: final_buckets = filtered_buckets else: final_buckets = buckets @@ -261,47 +265,53 @@ def get_bucket_details(connection, name, requested_facts, transform_location): for key in requested_facts: if requested_facts[key]: if key == 'bucket_location': - all_facts[key] = get_bucket_location(name, connection, transform_location) + all_facts[key] = {} + try: + all_facts[key] = get_bucket_location(name, connection, transform_location) + # we just pass on error - error means that resources is undefined + except botocore.exceptions.ClientError: + pass else: - all_facts[key] = get_bucket_property(name, connection, key) + all_facts[key] = {} + try: + all_facts[key] = get_bucket_property(name, connection, key) + # we just pass on error - error means that resources is undefined + except botocore.exceptions.ClientError: + pass return(all_facts) - +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_location(name, connection, transform_location=False): """ Get bucket location and optionally transform 'null' to 'us-east-1' """ - try: - data = connection.get_bucket_location(Bucket=name) - except botocore.exceptions.ClientError as err_msg: - data = {'error': err_msg} + data = connection.get_bucket_location(Bucket=name) + # Replace 'null' with 'us-east-1'? if transform_location: try: if not data['LocationConstraint']: data['LocationConstraint'] = 'us-east-1' except KeyError: - data['transform_failed'] = True - + pass + # Strip response metadata (not needed) try: data.pop('ResponseMetadata') return(data) except KeyError: return(data) - +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ Get bucket property """ api_call = "get_" + get_api_name api_function = getattr(connection, api_call) - try: - data = api_function(Bucket=name) - except botocore.exceptions.ClientError as err_msg: - data = {'error': err_msg} + data = api_function(Bucket=name) + # Strip response metadata (not needed) try: data.pop('ResponseMetadata') return(data) @@ -314,45 +324,18 @@ def main(): Get list of S3 buckets :return: """ - argument_spec = dict( - name=dict(type='str', default=""), - name_filter=dict(type='str', default=""), - bucket_facts=dict(type='dict', options=dict( - bucket_accelerate_configuration=dict(type='bool', default=False), - bucket_acl=dict(type='bool', default=False), - bucket_cors=dict(type='bool', default=False), - bucket_encryption=dict(type='bool', default=False), - bucket_lifecycle_configuration=dict(type='bool', default=False), - bucket_location=dict(type='bool', default=False), - bucket_logging=dict(type='bool', default=False), - bucket_notification_configuration=dict(type='bool', default=False), - bucket_ownership_controls=dict(type='bool', default=False), - bucket_policy=dict(type='bool', default=False), - bucket_policy_status=dict(type='bool', default=False), - bucket_replication=dict(type='bool', default=False), - bucket_request_payment=dict(type='bool', default=False), - bucket_tagging=dict(type='bool', default=False), - bucket_website=dict(type='bool', default=False), - public_access_block=dict(type='bool', default=False), - )), - transform_location=dict(type='bool', default=False) - ) - argument_spec = dict( name=dict(type=str, default=""), name_filter=dict(type=str, default=""), bucket_facts=dict(type='dict', options=dict( bucket_accelerate_configuration=dict(type=bool, default=False), bucket_acl=dict(type=bool, default=False), - bucket_analytics_configuration=dict(type=bool, default=False), bucket_cors=dict(type=bool, default=False), bucket_encryption=dict(type=bool, default=False), - bucket_inventory_configuration=dict(type=bool, default=False), bucket_lifecycle_configuration=dict(type=bool, default=False), + bucket_notification_configuration=dict(type=bool, default=False), bucket_location=dict(type=bool, default=False), bucket_logging=dict(type=bool, default=False), - bucket_metrics_configuration=dict(type=bool, default=False), - bucket_notification_configuration=dict(type=bool, default=False), bucket_ownership_controls=dict(type=bool, default=False), bucket_policy=dict(type=bool, default=False), bucket_policy_status=dict(type=bool, default=False), @@ -360,11 +343,11 @@ def main(): bucket_request_payment=dict(type=bool, default=False), bucket_tagging=dict(type=bool, default=False), bucket_website=dict(type=bool, default=False), + public_access_block=dict(type=bool, default=False), )), transform_location=dict(type='bool', default=False) ) - # Ensure we have an empty dict result = {} @@ -374,7 +357,7 @@ def main(): ] # Including ec2 argument spec - module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True) + module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True, mutually_exclusive=mutually_exclusive) is_old_facts = module._name == 'aws_s3_bucket_facts' if is_old_facts: module.deprecate("The 'aws_s3_bucket_facts' module has been renamed to 'aws_s3_bucket_info', " @@ -396,6 +379,12 @@ def main(): # Get basic bucket list (name + creation date) bucket_list = get_bucket_list(module, connection, name, name_filter) + # Add information about name/name_filter to result + if name: + result['bucket_name'] = name + elif name_filter: + result['bucket_name_filter'] = name_filter + # Gather detailed information about buckets if requested bucket_facts = module.params.get("bucket_facts") if bucket_facts: @@ -409,7 +398,6 @@ def main(): else: module.exit_json(msg="Retrieved s3 info.", **result) - -# MAIN +## MAIN ## if __name__ == '__main__': main() diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 17b8d422976..9e07a021372 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -11,9 +11,6 @@ s3_bucket: name: "{{ item }}" state: present - tags: - integration: "verified" - facts: "defined" register: output loop: "{{ testing_buckets }}" @@ -50,7 +47,7 @@ transform_location: true register: bucket_list - - name: Assert that buckets list contains requested bucket facts + - name: Assert that buckets list contains requested facts assert: that: - item.name is search(name_pattern) @@ -71,22 +68,10 @@ - item.bucket_website is defined - item.public_access_block is defined loop: "{{ bucket_list.buckets }}" - loop_control: - label: "{{ item.name }}" - - - name: Assert that retrieved bucket facts contains valid data - assert: - that: - - item.bucket_acl.Owner is defined - - item.bucket_tagging.facts is defined - - item.bucket_tagging.integration is defined - loop: "{{ bucket_list.buckets }}" - loop_control: - label: "{{ item.name }}" always: - name: Delete simple s3_buckets s3_bucket: name: "{{ item }}" - state: absent + state: present loop: "{{ testing_buckets }}" From 56ce0019c0545141defde4c0b55a1c144d429f55 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 17:02:45 +0200 Subject: [PATCH 30/63] Code linting - shippable --- plugins/modules/aws_s3_bucket_info.py | 56 +++++++++++++++------------ 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 96fbdbf71e3..85123a606ed 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -36,8 +36,12 @@ description: - Retrieve requested S3 bucket detailed information - Each bucket_X option executes one API call, hence many options=true will case slower module execution - - You can limit buckets by using I(name) or I(name_filter) option + - You can limit buckets by using I(name) or I(name_filter) option suboptions: + bucket_accelerate_configuration: + description: Retrive S3 accelerate configuration + type: bool + default: False bucket_location: description: Retrive S3 bucket location type: bool @@ -58,10 +62,6 @@ description: Retrive S3 bucket request payment type: bool default: False - bucket_analytics_configuration: - description: Retrive S3 bucket analytics configuration - type: bool - default: False bucket_tagging: description: Retrive S3 bucket tagging type: bool @@ -168,16 +168,19 @@ bucket_location: dictionary data bucket_cors: dictionary data # ...etc + type: list # if name options was specified -bucket_name: +bucket_name: description: "Name of the bucket requested" sample: "my_bucket" + type: str # if name_filter was specified -bucket_name_filter: +bucket_name_filter: description: "String to match bucket name" sample: "buckets_prefix" + type: str ''' try: @@ -281,6 +284,7 @@ def get_bucket_details(connection, name, requested_facts, transform_location): return(all_facts) + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_location(name, connection, transform_location=False): """ @@ -302,6 +306,7 @@ def get_bucket_location(name, connection, transform_location=False): except KeyError: return(data) + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ @@ -328,23 +333,23 @@ def main(): name=dict(type=str, default=""), name_filter=dict(type=str, default=""), bucket_facts=dict(type='dict', options=dict( - bucket_accelerate_configuration=dict(type=bool, default=False), - bucket_acl=dict(type=bool, default=False), - bucket_cors=dict(type=bool, default=False), - bucket_encryption=dict(type=bool, default=False), - bucket_lifecycle_configuration=dict(type=bool, default=False), - bucket_notification_configuration=dict(type=bool, default=False), - bucket_location=dict(type=bool, default=False), - bucket_logging=dict(type=bool, default=False), - bucket_ownership_controls=dict(type=bool, default=False), - bucket_policy=dict(type=bool, default=False), - bucket_policy_status=dict(type=bool, default=False), - bucket_replication=dict(type=bool, default=False), - bucket_request_payment=dict(type=bool, default=False), - bucket_tagging=dict(type=bool, default=False), - bucket_website=dict(type=bool, default=False), - public_access_block=dict(type=bool, default=False), - )), + bucket_accelerate_configuration=dict(type='bool', default=False), + bucket_acl=dict(type='bool', default=False), + bucket_cors=dict(type='bool', default=False), + bucket_encryption=dict(type='bool', default=False), + bucket_lifecycle_configuration=dict(type='bool', default=False), + bucket_location=dict(type='bool', default=False), + bucket_logging=dict(type='bool', default=False), + bucket_notification_configuration=dict(type='bool', default=False), + bucket_ownership_controls=dict(type='bool', default=False), + bucket_policy=dict(type='bool', default=False), + bucket_policy_status=dict(type='bool', default=False), + bucket_replication=dict(type='bool', default=False), + bucket_request_payment=dict(type='bool', default=False), + bucket_tagging=dict(type='bool', default=False), + bucket_website=dict(type='bool', default=False), + public_access_block=dict(type='bool', default=False), + )), transform_location=dict(type='bool', default=False) ) @@ -398,6 +403,7 @@ def main(): else: module.exit_json(msg="Retrieved s3 info.", **result) -## MAIN ## + +# MAIN if __name__ == '__main__': main() From 9cb2d3eee269e2401ad838e3060c01bebbe6d0dc Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 17:09:01 +0200 Subject: [PATCH 31/63] Fixing option type --- plugins/modules/aws_s3_bucket_info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 85123a606ed..7deebe6c4c2 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -330,8 +330,8 @@ def main(): :return: """ argument_spec = dict( - name=dict(type=str, default=""), - name_filter=dict(type=str, default=""), + name=dict(type='str', default=""), + name_filter=dict(type='str', default=""), bucket_facts=dict(type='dict', options=dict( bucket_accelerate_configuration=dict(type='bool', default=False), bucket_acl=dict(type='bool', default=False), From ad99a49e776626fd922fa963ef8bbb2de06b1b66 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 17:14:16 +0200 Subject: [PATCH 32/63] Fixing RETURN / Shippable --- plugins/modules/aws_s3_bucket_info.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 7deebe6c4c2..3d4c2356573 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -169,18 +169,6 @@ bucket_cors: dictionary data # ...etc type: list - -# if name options was specified -bucket_name: - description: "Name of the bucket requested" - sample: "my_bucket" - type: str - -# if name_filter was specified -bucket_name_filter: - description: "String to match bucket name" - sample: "buckets_prefix" - type: str ''' try: From 2a9af044785e34f8232d68ea7dec907b1f66d91e Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 18 Nov 2020 10:32:14 +0100 Subject: [PATCH 33/63] Extending integration test --- .../targets/aws_s3_bucket_info/tasks/main.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 9e07a021372..dd5fc672407 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -11,6 +11,9 @@ s3_bucket: name: "{{ item }}" state: present + tags: + integration: "verified" + facts: "defined" register: output loop: "{{ testing_buckets }}" @@ -47,7 +50,7 @@ transform_location: true register: bucket_list - - name: Assert that buckets list contains requested facts + - name: Assert that buckets list contains requested bucket facts assert: that: - item.name is search(name_pattern) @@ -68,6 +71,20 @@ - item.bucket_website is defined - item.public_access_block is defined loop: "{{ bucket_list.buckets }}" + loop_control: + label: "{{ item.name }}" + + - name: Assert that retrieved bucket facts contains valid data + assert: + that: + - item.bucket_acl.Owner is defined + - item.bucket_location.LocationConstraint is search(aws_region) + - item.bucket_tagging.TagSet is defined + - item.bucket_tagging.TagSet is search("defined") + - item.bucket_tagging.TagSet is search("verified") + loop: "{{ bucket_list.buckets }}" + loop_control: + label: "{{ item.name }}" always: - name: Delete simple s3_buckets From ed835077aae1ce6d65151e02de8c43c5c162019e Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 18 Nov 2020 10:41:28 +0100 Subject: [PATCH 34/63] Adding version_added to new module options --- plugins/modules/aws_s3_bucket_info.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 3d4c2356573..a60daa52134 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -32,6 +32,7 @@ - Get info only about buckets name matching defined string type: str default: "" + version_added: 1.3.0 bucket_facts: description: - Retrieve requested S3 bucket detailed information @@ -103,6 +104,7 @@ type: bool default: False type: dict + version_added: 1.3.0 transform_location: description: - S3 bucket location for default us-east-1 is normally reported as 'null' @@ -110,6 +112,7 @@ - affects only queries with I(bucket_facts) > I(bucket_location) = true type: bool default: False + version_added: 1.3.0 extends_documentation_fragment: - amazon.aws.aws From 31ba9538ecd874a436ad8876563110422f27b9b5 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Thu, 19 Nov 2020 14:52:39 +0100 Subject: [PATCH 35/63] Use boto3_tag_list_to_ansible_dict for bucket tags --- plugins/modules/aws_s3_bucket_info.py | 49 +++++++++++-------- .../targets/aws_s3_bucket_info/tasks/main.yml | 5 +- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index a60daa52134..4992fac15bb 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -195,6 +195,7 @@ def get_bucket_list(module, connection, name="", name_filter=""): buckets = [] filtered_buckets = [] final_buckets = [] + # Get all buckets try: buckets = camel_dict_to_snake_dict(connection.list_buckets())['buckets'] @@ -219,24 +220,6 @@ def get_bucket_list(module, connection, name="", name_filter=""): return(final_buckets) - # Filter buckets if requested - if name_filter: - for bucket in buckets: - if name_filter in bucket['name']: - filtered_buckets.append(bucket) - elif name: - for bucket in buckets: - if name == bucket['name']: - filtered_buckets.append(bucket) - - # Return proper list (filtered or all) - if name or name_filter: - final_buckets = filtered_buckets - else: - final_buckets = buckets - return(final_buckets) - - def get_buckets_facts(connection, buckets, requested_facts, transform_location): """ Retrive additional information about S3 buckets @@ -265,6 +248,13 @@ def get_bucket_details(connection, name, requested_facts, transform_location): # we just pass on error - error means that resources is undefined except botocore.exceptions.ClientError: pass + elif key == 'bucket_tagging': + all_facts[key] = {} + try: + all_facts[key] = get_bucket_tagging(name, connection) + # we just pass on error - error means that resources is undefined + except botocore.exceptions.ClientError: + pass else: all_facts[key] = {} try: @@ -298,6 +288,25 @@ def get_bucket_location(name, connection, transform_location=False): return(data) +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) +def get_bucket_tagging(name, connection): + """ + Get bucket tags and transform them using `boto3_tag_list_to_ansible_dict` function + """ + data = connection.get_bucket_tagging(Bucket=name) + + try: + bucket_tags = boto3_tag_list_to_ansible_dict(data['TagSet']) + return(bucket_tags) + except KeyError: + # Strip response metadata (not needed) + try: + data.pop('ResponseMetadata') + return(data) + except KeyError: + return(data) + + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ @@ -369,8 +378,8 @@ def main(): connection = {} try: connection = module.client('s3') - except (connection.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg='Failed to connect to AWS') + except (connection.exceptions.ClientError, botocore.exceptions.BotoCoreError) as err_code: + module.fail_json_aws(err_code, msg='Failed to connect to AWS') # Get basic bucket list (name + creation date) bucket_list = get_bucket_list(module, connection, name, name_filter) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index dd5fc672407..a7510edb5de 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -79,9 +79,8 @@ that: - item.bucket_acl.Owner is defined - item.bucket_location.LocationConstraint is search(aws_region) - - item.bucket_tagging.TagSet is defined - - item.bucket_tagging.TagSet is search("defined") - - item.bucket_tagging.TagSet is search("verified") + - item.bucket_tagging.facts is search("defined") + - item.bucket_tagging.integration is search("verified") loop: "{{ bucket_list.buckets }}" loop_control: label: "{{ item.name }}" From 6fd3592eb5f545393fd29e289e1563d6280edd7e Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Thu, 19 Nov 2020 14:57:08 +0100 Subject: [PATCH 36/63] Fixing integration test (delete buckets) --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index a7510edb5de..0fcf081ae6a 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -89,5 +89,5 @@ - name: Delete simple s3_buckets s3_bucket: name: "{{ item }}" - state: present + state: absent loop: "{{ testing_buckets }}" From a4b240a18805421fb2c06a38c9424149abb91ec6 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Fri, 20 Nov 2020 10:59:32 +0100 Subject: [PATCH 37/63] Adjust integration test (python 2.7) --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 0fcf081ae6a..f8b9dd81408 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -78,9 +78,9 @@ assert: that: - item.bucket_acl.Owner is defined - - item.bucket_location.LocationConstraint is search(aws_region) - - item.bucket_tagging.facts is search("defined") - - item.bucket_tagging.integration is search("verified") + - item.bucket_location.LocationConstraint is defined + - item.bucket_tagging.facts is defined + - item.bucket_tagging.integration is defined loop: "{{ bucket_list.buckets }}" loop_control: label: "{{ item.name }}" From dc1921de75e26bea0f5ba6240b4c9b1496a6760a Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Fri, 20 Nov 2020 12:52:59 +0100 Subject: [PATCH 38/63] fix integratiuon test -2 --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index f8b9dd81408..17b8d422976 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -78,7 +78,6 @@ assert: that: - item.bucket_acl.Owner is defined - - item.bucket_location.LocationConstraint is defined - item.bucket_tagging.facts is defined - item.bucket_tagging.integration is defined loop: "{{ bucket_list.buckets }}" From 9c607ae220474cf267fb9def72ce3bc744259745 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Fri, 9 Oct 2020 17:23:29 +0200 Subject: [PATCH 39/63] WIP - adding new functionality to aws_s3_bucket_info module --- plugins/modules/aws_s3_bucket_info.py | 144 ++++++++++++++------------ 1 file changed, 75 insertions(+), 69 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 4992fac15bb..5f559ca08bb 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -24,23 +24,20 @@ options: name: description: - - Get info only about specified bucket + - Get info about specified bucket type: str default: "" name_filter: description: - - Get info only about buckets name matching defined string + - Get info about buckets name matching defined string type: str default: "" - version_added: 1.3.0 bucket_facts: description: - Retrieve requested S3 bucket detailed information - - Each bucket_X option executes one API call, hence many options=true will case slower module execution - - You can limit buckets by using I(name) or I(name_filter) option suboptions: bucket_accelerate_configuration: - description: Retrive S3 accelerate configuration + description: Retrive S3 bucket accelerate configuration type: bool default: False bucket_location: @@ -63,6 +60,14 @@ description: Retrive S3 bucket request payment type: bool default: False + bucket_analytics_configuration: + description: Retrive S3 bucket analytics configuration + type: bool + default: False + bucket_metrics_configuration: + description: Retrive S3 bucket metrics configuration + type: bool + default: False bucket_tagging: description: Retrive S3 bucket tagging type: bool @@ -87,6 +92,10 @@ description: Retrive S3 bucket website type: bool default: False + bucket_inventory_configuration: + description: Retrive S3 bucket inventory configuration + type: bool + default: False bucket_policy: description: Retrive S3 bucket policy type: bool @@ -99,20 +108,13 @@ description: Retrive S3 bucket lifecycle configuration type: bool default: False - public_access_block: - description: Retrive S3 bucket public access block - type: bool - default: False type: dict - version_added: 1.3.0 transform_location: description: - S3 bucket location for default us-east-1 is normally reported as 'null' - setting this option to 'true' will return 'us-east-1' instead - - affects only queries with I(bucket_facts) > I(bucket_location) = true type: bool default: False - version_added: 1.3.0 extends_documentation_fragment: - amazon.aws.aws @@ -195,7 +197,6 @@ def get_bucket_list(module, connection, name="", name_filter=""): buckets = [] filtered_buckets = [] final_buckets = [] - # Get all buckets try: buckets = camel_dict_to_snake_dict(connection.list_buckets())['buckets'] @@ -220,6 +221,24 @@ def get_bucket_list(module, connection, name="", name_filter=""): return(final_buckets) + # Filter buckets if requested + if name_filter: + for bucket in buckets: + if name_filter in bucket['name']: + filtered_buckets.append(bucket) + elif name: + for bucket in buckets: + if name == bucket['name']: + filtered_buckets.append(bucket) + + # Return proper list (filtered or all) + if filtered_buckets: + final_buckets = filtered_buckets + else: + final_buckets = buckets + return(final_buckets) + + def get_buckets_facts(connection, buckets, requested_facts, transform_location): """ Retrive additional information about S3 buckets @@ -242,45 +261,29 @@ def get_bucket_details(connection, name, requested_facts, transform_location): for key in requested_facts: if requested_facts[key]: if key == 'bucket_location': - all_facts[key] = {} - try: - all_facts[key] = get_bucket_location(name, connection, transform_location) - # we just pass on error - error means that resources is undefined - except botocore.exceptions.ClientError: - pass - elif key == 'bucket_tagging': - all_facts[key] = {} - try: - all_facts[key] = get_bucket_tagging(name, connection) - # we just pass on error - error means that resources is undefined - except botocore.exceptions.ClientError: - pass + all_facts[key] = get_bucket_location(name, connection, transform_location) else: - all_facts[key] = {} - try: - all_facts[key] = get_bucket_property(name, connection, key) - # we just pass on error - error means that resources is undefined - except botocore.exceptions.ClientError: - pass + all_facts[key] = get_bucket_property(name, connection, key) return(all_facts) -@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_location(name, connection, transform_location=False): """ Get bucket location and optionally transform 'null' to 'us-east-1' """ - data = connection.get_bucket_location(Bucket=name) + try: + data = connection.get_bucket_location(Bucket=name) + except botocore.exceptions.ClientError as err_msg: + data = {'error': err_msg} - # Replace 'null' with 'us-east-1'? if transform_location: try: if not data['LocationConstraint']: data['LocationConstraint'] = 'us-east-1' except KeyError: - pass - # Strip response metadata (not needed) + data['transform_failed'] = True + try: data.pop('ResponseMetadata') return(data) @@ -288,35 +291,17 @@ def get_bucket_location(name, connection, transform_location=False): return(data) -@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) -def get_bucket_tagging(name, connection): - """ - Get bucket tags and transform them using `boto3_tag_list_to_ansible_dict` function - """ - data = connection.get_bucket_tagging(Bucket=name) - - try: - bucket_tags = boto3_tag_list_to_ansible_dict(data['TagSet']) - return(bucket_tags) - except KeyError: - # Strip response metadata (not needed) - try: - data.pop('ResponseMetadata') - return(data) - except KeyError: - return(data) - - -@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ Get bucket property """ api_call = "get_" + get_api_name api_function = getattr(connection, api_call) - data = api_function(Bucket=name) + try: + data = api_function(Bucket=name) + except botocore.exceptions.ClientError as err_msg: + data = {'error': err_msg} - # Strip response metadata (not needed) try: data.pop('ResponseMetadata') return(data) @@ -353,6 +338,33 @@ def main(): transform_location=dict(type='bool', default=False) ) + argument_spec = dict( + name=dict(type=str, default=""), + name_filter=dict(type=str, default=""), + bucket_facts=dict(type='dict', options=dict( + bucket_accelerate_configuration=dict(type=bool, default=False), + bucket_acl=dict(type=bool, default=False), + bucket_analytics_configuration=dict(type=bool, default=False), + bucket_cors=dict(type=bool, default=False), + bucket_encryption=dict(type=bool, default=False), + bucket_inventory_configuration=dict(type=bool, default=False), + bucket_lifecycle_configuration=dict(type=bool, default=False), + bucket_location=dict(type=bool, default=False), + bucket_logging=dict(type=bool, default=False), + bucket_metrics_configuration=dict(type=bool, default=False), + bucket_notification_configuration=dict(type=bool, default=False), + bucket_ownership_controls=dict(type=bool, default=False), + bucket_policy=dict(type=bool, default=False), + bucket_policy_status=dict(type=bool, default=False), + bucket_replication=dict(type=bool, default=False), + bucket_request_payment=dict(type=bool, default=False), + bucket_tagging=dict(type=bool, default=False), + bucket_website=dict(type=bool, default=False), + )), + transform_location=dict(type='bool', default=False) + ) + + # Ensure we have an empty dict result = {} @@ -362,7 +374,7 @@ def main(): ] # Including ec2 argument spec - module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True, mutually_exclusive=mutually_exclusive) + module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True) is_old_facts = module._name == 'aws_s3_bucket_facts' if is_old_facts: module.deprecate("The 'aws_s3_bucket_facts' module has been renamed to 'aws_s3_bucket_info', " @@ -378,18 +390,12 @@ def main(): connection = {} try: connection = module.client('s3') - except (connection.exceptions.ClientError, botocore.exceptions.BotoCoreError) as err_code: - module.fail_json_aws(err_code, msg='Failed to connect to AWS') + except (connection.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg='Failed to connect to AWS') # Get basic bucket list (name + creation date) bucket_list = get_bucket_list(module, connection, name, name_filter) - # Add information about name/name_filter to result - if name: - result['bucket_name'] = name - elif name_filter: - result['bucket_name_filter'] = name_filter - # Gather detailed information about buckets if requested bucket_facts = module.params.get("bucket_facts") if bucket_facts: From aca189d93f0223707824f65359daf787f71c178e Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 16:06:17 +0200 Subject: [PATCH 40/63] Fixing exceptions adding integration tests --- plugins/modules/aws_s3_bucket_info.py | 112 ++++++++---------- .../targets/aws_s3_bucket_info/tasks/main.yml | 19 +-- 2 files changed, 52 insertions(+), 79 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 5f559ca08bb..96fbdbf71e3 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -24,22 +24,20 @@ options: name: description: - - Get info about specified bucket + - Get info only about specified bucket type: str default: "" name_filter: description: - - Get info about buckets name matching defined string + - Get info only about buckets name matching defined string type: str default: "" bucket_facts: description: - Retrieve requested S3 bucket detailed information + - Each bucket_X option executes one API call, hence many options=true will case slower module execution + - You can limit buckets by using I(name) or I(name_filter) option suboptions: - bucket_accelerate_configuration: - description: Retrive S3 bucket accelerate configuration - type: bool - default: False bucket_location: description: Retrive S3 bucket location type: bool @@ -64,10 +62,6 @@ description: Retrive S3 bucket analytics configuration type: bool default: False - bucket_metrics_configuration: - description: Retrive S3 bucket metrics configuration - type: bool - default: False bucket_tagging: description: Retrive S3 bucket tagging type: bool @@ -92,10 +86,6 @@ description: Retrive S3 bucket website type: bool default: False - bucket_inventory_configuration: - description: Retrive S3 bucket inventory configuration - type: bool - default: False bucket_policy: description: Retrive S3 bucket policy type: bool @@ -108,11 +98,16 @@ description: Retrive S3 bucket lifecycle configuration type: bool default: False + public_access_block: + description: Retrive S3 bucket public access block + type: bool + default: False type: dict transform_location: description: - S3 bucket location for default us-east-1 is normally reported as 'null' - setting this option to 'true' will return 'us-east-1' instead + - affects only queries with I(bucket_facts) > I(bucket_location) = true type: bool default: False @@ -173,7 +168,16 @@ bucket_location: dictionary data bucket_cors: dictionary data # ...etc - type: list + +# if name options was specified +bucket_name: + description: "Name of the bucket requested" + sample: "my_bucket" + +# if name_filter was specified +bucket_name_filter: + description: "String to match bucket name" + sample: "buckets_prefix" ''' try: @@ -232,7 +236,7 @@ def get_bucket_list(module, connection, name="", name_filter=""): filtered_buckets.append(bucket) # Return proper list (filtered or all) - if filtered_buckets: + if name or name_filter: final_buckets = filtered_buckets else: final_buckets = buckets @@ -261,47 +265,53 @@ def get_bucket_details(connection, name, requested_facts, transform_location): for key in requested_facts: if requested_facts[key]: if key == 'bucket_location': - all_facts[key] = get_bucket_location(name, connection, transform_location) + all_facts[key] = {} + try: + all_facts[key] = get_bucket_location(name, connection, transform_location) + # we just pass on error - error means that resources is undefined + except botocore.exceptions.ClientError: + pass else: - all_facts[key] = get_bucket_property(name, connection, key) + all_facts[key] = {} + try: + all_facts[key] = get_bucket_property(name, connection, key) + # we just pass on error - error means that resources is undefined + except botocore.exceptions.ClientError: + pass return(all_facts) - +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_location(name, connection, transform_location=False): """ Get bucket location and optionally transform 'null' to 'us-east-1' """ - try: - data = connection.get_bucket_location(Bucket=name) - except botocore.exceptions.ClientError as err_msg: - data = {'error': err_msg} + data = connection.get_bucket_location(Bucket=name) + # Replace 'null' with 'us-east-1'? if transform_location: try: if not data['LocationConstraint']: data['LocationConstraint'] = 'us-east-1' except KeyError: - data['transform_failed'] = True - + pass + # Strip response metadata (not needed) try: data.pop('ResponseMetadata') return(data) except KeyError: return(data) - +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ Get bucket property """ api_call = "get_" + get_api_name api_function = getattr(connection, api_call) - try: - data = api_function(Bucket=name) - except botocore.exceptions.ClientError as err_msg: - data = {'error': err_msg} + data = api_function(Bucket=name) + # Strip response metadata (not needed) try: data.pop('ResponseMetadata') return(data) @@ -314,45 +324,18 @@ def main(): Get list of S3 buckets :return: """ - argument_spec = dict( - name=dict(type='str', default=""), - name_filter=dict(type='str', default=""), - bucket_facts=dict(type='dict', options=dict( - bucket_accelerate_configuration=dict(type='bool', default=False), - bucket_acl=dict(type='bool', default=False), - bucket_cors=dict(type='bool', default=False), - bucket_encryption=dict(type='bool', default=False), - bucket_lifecycle_configuration=dict(type='bool', default=False), - bucket_location=dict(type='bool', default=False), - bucket_logging=dict(type='bool', default=False), - bucket_notification_configuration=dict(type='bool', default=False), - bucket_ownership_controls=dict(type='bool', default=False), - bucket_policy=dict(type='bool', default=False), - bucket_policy_status=dict(type='bool', default=False), - bucket_replication=dict(type='bool', default=False), - bucket_request_payment=dict(type='bool', default=False), - bucket_tagging=dict(type='bool', default=False), - bucket_website=dict(type='bool', default=False), - public_access_block=dict(type='bool', default=False), - )), - transform_location=dict(type='bool', default=False) - ) - argument_spec = dict( name=dict(type=str, default=""), name_filter=dict(type=str, default=""), bucket_facts=dict(type='dict', options=dict( bucket_accelerate_configuration=dict(type=bool, default=False), bucket_acl=dict(type=bool, default=False), - bucket_analytics_configuration=dict(type=bool, default=False), bucket_cors=dict(type=bool, default=False), bucket_encryption=dict(type=bool, default=False), - bucket_inventory_configuration=dict(type=bool, default=False), bucket_lifecycle_configuration=dict(type=bool, default=False), + bucket_notification_configuration=dict(type=bool, default=False), bucket_location=dict(type=bool, default=False), bucket_logging=dict(type=bool, default=False), - bucket_metrics_configuration=dict(type=bool, default=False), - bucket_notification_configuration=dict(type=bool, default=False), bucket_ownership_controls=dict(type=bool, default=False), bucket_policy=dict(type=bool, default=False), bucket_policy_status=dict(type=bool, default=False), @@ -360,11 +343,11 @@ def main(): bucket_request_payment=dict(type=bool, default=False), bucket_tagging=dict(type=bool, default=False), bucket_website=dict(type=bool, default=False), + public_access_block=dict(type=bool, default=False), )), transform_location=dict(type='bool', default=False) ) - # Ensure we have an empty dict result = {} @@ -374,7 +357,7 @@ def main(): ] # Including ec2 argument spec - module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True) + module = AnsibleAWSModule(argument_spec=argument_spec, supports_check_mode=True, mutually_exclusive=mutually_exclusive) is_old_facts = module._name == 'aws_s3_bucket_facts' if is_old_facts: module.deprecate("The 'aws_s3_bucket_facts' module has been renamed to 'aws_s3_bucket_info', " @@ -396,6 +379,12 @@ def main(): # Get basic bucket list (name + creation date) bucket_list = get_bucket_list(module, connection, name, name_filter) + # Add information about name/name_filter to result + if name: + result['bucket_name'] = name + elif name_filter: + result['bucket_name_filter'] = name_filter + # Gather detailed information about buckets if requested bucket_facts = module.params.get("bucket_facts") if bucket_facts: @@ -409,7 +398,6 @@ def main(): else: module.exit_json(msg="Retrieved s3 info.", **result) - -# MAIN +## MAIN ## if __name__ == '__main__': main() diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 17b8d422976..9e07a021372 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -11,9 +11,6 @@ s3_bucket: name: "{{ item }}" state: present - tags: - integration: "verified" - facts: "defined" register: output loop: "{{ testing_buckets }}" @@ -50,7 +47,7 @@ transform_location: true register: bucket_list - - name: Assert that buckets list contains requested bucket facts + - name: Assert that buckets list contains requested facts assert: that: - item.name is search(name_pattern) @@ -71,22 +68,10 @@ - item.bucket_website is defined - item.public_access_block is defined loop: "{{ bucket_list.buckets }}" - loop_control: - label: "{{ item.name }}" - - - name: Assert that retrieved bucket facts contains valid data - assert: - that: - - item.bucket_acl.Owner is defined - - item.bucket_tagging.facts is defined - - item.bucket_tagging.integration is defined - loop: "{{ bucket_list.buckets }}" - loop_control: - label: "{{ item.name }}" always: - name: Delete simple s3_buckets s3_bucket: name: "{{ item }}" - state: absent + state: present loop: "{{ testing_buckets }}" From 6107bc21edf733b9ff1c7c2fea1926b51c4bd342 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 17:02:45 +0200 Subject: [PATCH 41/63] Code linting - shippable --- plugins/modules/aws_s3_bucket_info.py | 56 +++++++++++++++------------ 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 96fbdbf71e3..85123a606ed 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -36,8 +36,12 @@ description: - Retrieve requested S3 bucket detailed information - Each bucket_X option executes one API call, hence many options=true will case slower module execution - - You can limit buckets by using I(name) or I(name_filter) option + - You can limit buckets by using I(name) or I(name_filter) option suboptions: + bucket_accelerate_configuration: + description: Retrive S3 accelerate configuration + type: bool + default: False bucket_location: description: Retrive S3 bucket location type: bool @@ -58,10 +62,6 @@ description: Retrive S3 bucket request payment type: bool default: False - bucket_analytics_configuration: - description: Retrive S3 bucket analytics configuration - type: bool - default: False bucket_tagging: description: Retrive S3 bucket tagging type: bool @@ -168,16 +168,19 @@ bucket_location: dictionary data bucket_cors: dictionary data # ...etc + type: list # if name options was specified -bucket_name: +bucket_name: description: "Name of the bucket requested" sample: "my_bucket" + type: str # if name_filter was specified -bucket_name_filter: +bucket_name_filter: description: "String to match bucket name" sample: "buckets_prefix" + type: str ''' try: @@ -281,6 +284,7 @@ def get_bucket_details(connection, name, requested_facts, transform_location): return(all_facts) + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_location(name, connection, transform_location=False): """ @@ -302,6 +306,7 @@ def get_bucket_location(name, connection, transform_location=False): except KeyError: return(data) + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ @@ -328,23 +333,23 @@ def main(): name=dict(type=str, default=""), name_filter=dict(type=str, default=""), bucket_facts=dict(type='dict', options=dict( - bucket_accelerate_configuration=dict(type=bool, default=False), - bucket_acl=dict(type=bool, default=False), - bucket_cors=dict(type=bool, default=False), - bucket_encryption=dict(type=bool, default=False), - bucket_lifecycle_configuration=dict(type=bool, default=False), - bucket_notification_configuration=dict(type=bool, default=False), - bucket_location=dict(type=bool, default=False), - bucket_logging=dict(type=bool, default=False), - bucket_ownership_controls=dict(type=bool, default=False), - bucket_policy=dict(type=bool, default=False), - bucket_policy_status=dict(type=bool, default=False), - bucket_replication=dict(type=bool, default=False), - bucket_request_payment=dict(type=bool, default=False), - bucket_tagging=dict(type=bool, default=False), - bucket_website=dict(type=bool, default=False), - public_access_block=dict(type=bool, default=False), - )), + bucket_accelerate_configuration=dict(type='bool', default=False), + bucket_acl=dict(type='bool', default=False), + bucket_cors=dict(type='bool', default=False), + bucket_encryption=dict(type='bool', default=False), + bucket_lifecycle_configuration=dict(type='bool', default=False), + bucket_location=dict(type='bool', default=False), + bucket_logging=dict(type='bool', default=False), + bucket_notification_configuration=dict(type='bool', default=False), + bucket_ownership_controls=dict(type='bool', default=False), + bucket_policy=dict(type='bool', default=False), + bucket_policy_status=dict(type='bool', default=False), + bucket_replication=dict(type='bool', default=False), + bucket_request_payment=dict(type='bool', default=False), + bucket_tagging=dict(type='bool', default=False), + bucket_website=dict(type='bool', default=False), + public_access_block=dict(type='bool', default=False), + )), transform_location=dict(type='bool', default=False) ) @@ -398,6 +403,7 @@ def main(): else: module.exit_json(msg="Retrieved s3 info.", **result) -## MAIN ## + +# MAIN if __name__ == '__main__': main() From 269d77516f832b2551a25543265b54b2cd2bbdae Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 17:09:01 +0200 Subject: [PATCH 42/63] Fixing option type --- plugins/modules/aws_s3_bucket_info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 85123a606ed..7deebe6c4c2 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -330,8 +330,8 @@ def main(): :return: """ argument_spec = dict( - name=dict(type=str, default=""), - name_filter=dict(type=str, default=""), + name=dict(type='str', default=""), + name_filter=dict(type='str', default=""), bucket_facts=dict(type='dict', options=dict( bucket_accelerate_configuration=dict(type='bool', default=False), bucket_acl=dict(type='bool', default=False), From 9b8deb5de4465114c46ae8b7dc761f221feafec3 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Mon, 12 Oct 2020 17:14:16 +0200 Subject: [PATCH 43/63] Fixing RETURN / Shippable --- plugins/modules/aws_s3_bucket_info.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 7deebe6c4c2..3d4c2356573 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -169,18 +169,6 @@ bucket_cors: dictionary data # ...etc type: list - -# if name options was specified -bucket_name: - description: "Name of the bucket requested" - sample: "my_bucket" - type: str - -# if name_filter was specified -bucket_name_filter: - description: "String to match bucket name" - sample: "buckets_prefix" - type: str ''' try: From cb2c885239ce30df590bb5c0e7eb400e7902c7fa Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 18 Nov 2020 10:32:14 +0100 Subject: [PATCH 44/63] Extending integration test --- .../targets/aws_s3_bucket_info/tasks/main.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 9e07a021372..dd5fc672407 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -11,6 +11,9 @@ s3_bucket: name: "{{ item }}" state: present + tags: + integration: "verified" + facts: "defined" register: output loop: "{{ testing_buckets }}" @@ -47,7 +50,7 @@ transform_location: true register: bucket_list - - name: Assert that buckets list contains requested facts + - name: Assert that buckets list contains requested bucket facts assert: that: - item.name is search(name_pattern) @@ -68,6 +71,20 @@ - item.bucket_website is defined - item.public_access_block is defined loop: "{{ bucket_list.buckets }}" + loop_control: + label: "{{ item.name }}" + + - name: Assert that retrieved bucket facts contains valid data + assert: + that: + - item.bucket_acl.Owner is defined + - item.bucket_location.LocationConstraint is search(aws_region) + - item.bucket_tagging.TagSet is defined + - item.bucket_tagging.TagSet is search("defined") + - item.bucket_tagging.TagSet is search("verified") + loop: "{{ bucket_list.buckets }}" + loop_control: + label: "{{ item.name }}" always: - name: Delete simple s3_buckets From 67ee9ac7343d81ff40f2407f1fcd98a6967fa090 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 18 Nov 2020 10:41:28 +0100 Subject: [PATCH 45/63] Adding version_added to new module options --- plugins/modules/aws_s3_bucket_info.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 3d4c2356573..a60daa52134 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -32,6 +32,7 @@ - Get info only about buckets name matching defined string type: str default: "" + version_added: 1.3.0 bucket_facts: description: - Retrieve requested S3 bucket detailed information @@ -103,6 +104,7 @@ type: bool default: False type: dict + version_added: 1.3.0 transform_location: description: - S3 bucket location for default us-east-1 is normally reported as 'null' @@ -110,6 +112,7 @@ - affects only queries with I(bucket_facts) > I(bucket_location) = true type: bool default: False + version_added: 1.3.0 extends_documentation_fragment: - amazon.aws.aws From 76d4fdae07df5619a44e52af107117db25445214 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Thu, 19 Nov 2020 14:52:39 +0100 Subject: [PATCH 46/63] Use boto3_tag_list_to_ansible_dict for bucket tags --- plugins/modules/aws_s3_bucket_info.py | 49 +++++++++++-------- .../targets/aws_s3_bucket_info/tasks/main.yml | 5 +- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index a60daa52134..4992fac15bb 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -195,6 +195,7 @@ def get_bucket_list(module, connection, name="", name_filter=""): buckets = [] filtered_buckets = [] final_buckets = [] + # Get all buckets try: buckets = camel_dict_to_snake_dict(connection.list_buckets())['buckets'] @@ -219,24 +220,6 @@ def get_bucket_list(module, connection, name="", name_filter=""): return(final_buckets) - # Filter buckets if requested - if name_filter: - for bucket in buckets: - if name_filter in bucket['name']: - filtered_buckets.append(bucket) - elif name: - for bucket in buckets: - if name == bucket['name']: - filtered_buckets.append(bucket) - - # Return proper list (filtered or all) - if name or name_filter: - final_buckets = filtered_buckets - else: - final_buckets = buckets - return(final_buckets) - - def get_buckets_facts(connection, buckets, requested_facts, transform_location): """ Retrive additional information about S3 buckets @@ -265,6 +248,13 @@ def get_bucket_details(connection, name, requested_facts, transform_location): # we just pass on error - error means that resources is undefined except botocore.exceptions.ClientError: pass + elif key == 'bucket_tagging': + all_facts[key] = {} + try: + all_facts[key] = get_bucket_tagging(name, connection) + # we just pass on error - error means that resources is undefined + except botocore.exceptions.ClientError: + pass else: all_facts[key] = {} try: @@ -298,6 +288,25 @@ def get_bucket_location(name, connection, transform_location=False): return(data) +@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) +def get_bucket_tagging(name, connection): + """ + Get bucket tags and transform them using `boto3_tag_list_to_ansible_dict` function + """ + data = connection.get_bucket_tagging(Bucket=name) + + try: + bucket_tags = boto3_tag_list_to_ansible_dict(data['TagSet']) + return(bucket_tags) + except KeyError: + # Strip response metadata (not needed) + try: + data.pop('ResponseMetadata') + return(data) + except KeyError: + return(data) + + @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ @@ -369,8 +378,8 @@ def main(): connection = {} try: connection = module.client('s3') - except (connection.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg='Failed to connect to AWS') + except (connection.exceptions.ClientError, botocore.exceptions.BotoCoreError) as err_code: + module.fail_json_aws(err_code, msg='Failed to connect to AWS') # Get basic bucket list (name + creation date) bucket_list = get_bucket_list(module, connection, name, name_filter) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index dd5fc672407..a7510edb5de 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -79,9 +79,8 @@ that: - item.bucket_acl.Owner is defined - item.bucket_location.LocationConstraint is search(aws_region) - - item.bucket_tagging.TagSet is defined - - item.bucket_tagging.TagSet is search("defined") - - item.bucket_tagging.TagSet is search("verified") + - item.bucket_tagging.facts is search("defined") + - item.bucket_tagging.integration is search("verified") loop: "{{ bucket_list.buckets }}" loop_control: label: "{{ item.name }}" From 1c4602fa7d79332c6806001b8b2aaf8bc8c95eaa Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Thu, 19 Nov 2020 14:57:08 +0100 Subject: [PATCH 47/63] Fixing integration test (delete buckets) --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index a7510edb5de..0fcf081ae6a 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -89,5 +89,5 @@ - name: Delete simple s3_buckets s3_bucket: name: "{{ item }}" - state: present + state: absent loop: "{{ testing_buckets }}" From 965236aa65f0319e3413badd6724a9340f84107b Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Fri, 20 Nov 2020 10:59:32 +0100 Subject: [PATCH 48/63] Adjust integration test (python 2.7) --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 0fcf081ae6a..f8b9dd81408 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -78,9 +78,9 @@ assert: that: - item.bucket_acl.Owner is defined - - item.bucket_location.LocationConstraint is search(aws_region) - - item.bucket_tagging.facts is search("defined") - - item.bucket_tagging.integration is search("verified") + - item.bucket_location.LocationConstraint is defined + - item.bucket_tagging.facts is defined + - item.bucket_tagging.integration is defined loop: "{{ bucket_list.buckets }}" loop_control: label: "{{ item.name }}" From 1abb51cec17e68fac04b3cdcb3f7d3de34fc5f40 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Fri, 20 Nov 2020 12:52:59 +0100 Subject: [PATCH 49/63] fix integratiuon test -2 --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index f8b9dd81408..17b8d422976 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -78,7 +78,6 @@ assert: that: - item.bucket_acl.Owner is defined - - item.bucket_location.LocationConstraint is defined - item.bucket_tagging.facts is defined - item.bucket_tagging.integration is defined loop: "{{ bucket_list.buckets }}" From ca7974a3818251f686c7af1e9bd0ee6c570b634a Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 27 Jan 2021 11:52:20 +0100 Subject: [PATCH 50/63] Merging requested PR changes #1 Co-authored-by: Mark Chappell --- plugins/modules/aws_s3_bucket_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 4992fac15bb..79617c9c91d 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -24,7 +24,7 @@ options: name: description: - - Get info only about specified bucket + - Name of bucket to query. type: str default: "" name_filter: From bee203c5c4625e88bd9c07409caf67d60dfa77dd Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 27 Jan 2021 11:53:17 +0100 Subject: [PATCH 51/63] Bump version_added 1.3.0->1.4.0 Co-authored-by: Mark Chappell --- plugins/modules/aws_s3_bucket_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 79617c9c91d..feff0c736c3 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -112,7 +112,7 @@ - affects only queries with I(bucket_facts) > I(bucket_location) = true type: bool default: False - version_added: 1.3.0 + version_added: 1.4.0 extends_documentation_fragment: - amazon.aws.aws From de04a978568c0820cf269bfdf455f9fa807bf150 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 27 Jan 2021 11:54:51 +0100 Subject: [PATCH 52/63] Fix module docstring Co-authored-by: Mark Chappell --- plugins/modules/aws_s3_bucket_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index feff0c736c3..0fb2fdfcc77 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -29,7 +29,7 @@ default: "" name_filter: description: - - Get info only about buckets name matching defined string + - Get information only about buckets name containing a string. type: str default: "" version_added: 1.3.0 From 72acae952da2fbc44411300986eec2e9ee922b8c Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 27 Jan 2021 12:04:33 +0100 Subject: [PATCH 53/63] Fixing changelog --- changelogs/changelog.yaml | 4 +--- changelogs/fragments/260-extending-s3_bucket_info-module.yml | 3 +++ 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 changelogs/fragments/260-extending-s3_bucket_info-module.yml diff --git a/changelogs/changelog.yaml b/changelogs/changelog.yaml index 882a31a01e8..72a5ee46f29 100644 --- a/changelogs/changelog.yaml +++ b/changelogs/changelog.yaml @@ -775,7 +775,7 @@ releases: behavior (https://github.com/ansible-collections/community.aws/pull/318). minor_changes: - ec2_vpc_igw - Add AWSRetry decorators to improve reliability (https://github.com/ansible-collections/community.aws/pull/318). - - ec2_vpc_igw - Add ``purge_tags`` parameter so that tags can be added without + - ec2_vpc_igw - Add ``purge_tagsminor_changes:`` parameter so that tags can be added without purging existing tags to match the collection standard tagging behaviour (https://github.com/ansible-collections/community.aws/pull/318). - ec2_vpc_igw_info - Add AWSRetry decorators to improve reliability (https://github.com/ansible-collections/community.aws/pull/318). - ec2_vpc_igw_info - Add ``convert_tags`` parameter so that tags can be returned @@ -785,8 +785,6 @@ releases: - redshift - add support for setting tags. - s3_lifecycle - Add support for intelligent tiering and deep archive storage classes (https://github.com/ansible-collections/community.aws/issues/270) - - aws_s3_bucket_info - new module options ``name:``, ``name_filter``, ``bucket_facts`` and ``transform_location`` - (https://github.com/ansible-collections/community.aws/pull/260). fragments: - 244-rds_instance-no_log.yml - 264-fix-elemt-type-for-containers-in-ecs_taskdefinition.yml diff --git a/changelogs/fragments/260-extending-s3_bucket_info-module.yml b/changelogs/fragments/260-extending-s3_bucket_info-module.yml new file mode 100644 index 00000000000..a1b36dfc8d7 --- /dev/null +++ b/changelogs/fragments/260-extending-s3_bucket_info-module.yml @@ -0,0 +1,3 @@ +minor_changes: + - aws_s3_bucket_info - new module options ``name``, ``name_filter``, ``bucket_facts`` and ``transform_location`` + (https://github.com/ansible-collections/community.aws/pull/260). From bc9653ff09e4ce4433d26ad9844e04cbdeb2be66 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 27 Jan 2021 13:24:39 +0100 Subject: [PATCH 54/63] Code cleanup --- plugins/modules/aws_s3_bucket_info.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 0fb2fdfcc77..c7215580511 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -27,12 +27,13 @@ - Name of bucket to query. type: str default: "" + version_added: 1.4.0 name_filter: description: - Get information only about buckets name containing a string. type: str default: "" - version_added: 1.3.0 + version_added: 1.4.0 bucket_facts: description: - Retrieve requested S3 bucket detailed information @@ -104,7 +105,7 @@ type: bool default: False type: dict - version_added: 1.3.0 + version_added: 1.4.0 transform_location: description: - S3 bucket location for default us-east-1 is normally reported as 'null' @@ -179,9 +180,10 @@ except ImportError: pass # Handled by AnsibleAWSModule -from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict - from ansible_collections.amazon.aws.plugins.module_utils.core import AnsibleAWSModule +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict +from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict def get_bucket_list(module, connection, name="", name_filter=""): @@ -199,8 +201,8 @@ def get_bucket_list(module, connection, name="", name_filter=""): # Get all buckets try: buckets = camel_dict_to_snake_dict(connection.list_buckets())['buckets'] - except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: - module.fail_json_aws(e, msg="Failed to list buckets") + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as err_code: + module.fail_json_aws(err_code, msg="Failed to list buckets") # Filter buckets if requested if name_filter: @@ -266,7 +268,7 @@ def get_bucket_details(connection, name, requested_facts, transform_location): return(all_facts) -@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) +@AWSRetry.jittered_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_location(name, connection, transform_location=False): """ Get bucket location and optionally transform 'null' to 'us-east-1' @@ -288,7 +290,7 @@ def get_bucket_location(name, connection, transform_location=False): return(data) -@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) +@AWSRetry.jittered_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_tagging(name, connection): """ Get bucket tags and transform them using `boto3_tag_list_to_ansible_dict` function @@ -307,7 +309,7 @@ def get_bucket_tagging(name, connection): return(data) -@AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) +@AWSRetry.jittered_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_property(name, connection, get_api_name): """ Get bucket property From 9c3e6704c8fd99ed077b10dbf5fdf2fd6ec8a823 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 27 Jan 2021 14:03:59 +0100 Subject: [PATCH 55/63] Update integration tests --- .../targets/aws_s3_bucket_info/tasks/main.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 17b8d422976..29ddb9a5d02 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -12,8 +12,10 @@ name: "{{ item }}" state: present tags: - integration: "verified" - facts: "defined" + "lowercase spaced": "hello cruel world" + "Title Case": "Hello Cruel World" + CamelCase: "SimpleCamelCase" + snake_case: "simple_snake_case" register: output loop: "{{ testing_buckets }}" @@ -78,8 +80,14 @@ assert: that: - item.bucket_acl.Owner is defined - - item.bucket_tagging.facts is defined - - item.bucket_tagging.integration is defined + - item.bucket_tagging.snake_case is defined + - item.bucket_tagging.CamelCase is defined + - item.bucket_tagging["lowercase spaced"] is defined + - item.bucket_tagging["Title Case"] is defined + - item.bucket_tagging.snake_case == 'simple_snake_case' + - item.bucket_tagging.CamelCase == 'SimpleCamelCase' + - item.bucket_tagging["lowercase spaced"] == 'hello cruel world' + - item.bucket_tagging["Title Case"] == 'Hello Cruel World' loop: "{{ bucket_list.buckets }}" loop_control: label: "{{ item.name }}" From 22ea4fc6250b7f04d81a21361ba4617d8927fc8b Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Wed, 27 Jan 2021 14:10:20 +0100 Subject: [PATCH 56/63] Add S3 bucket location check to integration test --- tests/integration/targets/aws_s3_bucket_info/tasks/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml index 29ddb9a5d02..5afdf8d7841 100644 --- a/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml +++ b/tests/integration/targets/aws_s3_bucket_info/tasks/main.yml @@ -88,6 +88,7 @@ - item.bucket_tagging.CamelCase == 'SimpleCamelCase' - item.bucket_tagging["lowercase spaced"] == 'hello cruel world' - item.bucket_tagging["Title Case"] == 'Hello Cruel World' + - item.bucket_location.LocationConstraint == aws_region loop: "{{ bucket_list.buckets }}" loop_control: label: "{{ item.name }}" From dec9b0522ae4921b0bb443cd654f62697df8aaca Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Tue, 2 Feb 2021 14:27:27 +0100 Subject: [PATCH 57/63] Documentation update - fix wording and typos in DOCUMENTATION - complete rewrite of RETURN --- plugins/modules/aws_s3_bucket_info.py | 300 ++++++++++++++++++++++---- 1 file changed, 260 insertions(+), 40 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index c7215580511..428607af57c 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -13,12 +13,12 @@ module: aws_s3_bucket_info version_added: 1.0.0 author: "Gerben Geijteman (@hyperized)" -short_description: Lists S3 buckets in AWS +short_description: lists S3 buckets in AWS requirements: - boto3 >= 1.4.4 - python >= 2.6 description: - - Lists S3 buckets in AWS + - Lists S3 buckets and details about those buckets. - This module was called C(aws_s3_bucket_facts) before Ansible 2.9, returning C(ansible_facts). Note that the M(community.aws.aws_s3_bucket_info) module no longer returns C(ansible_facts)! options: @@ -30,95 +30,89 @@ version_added: 1.4.0 name_filter: description: - - Get information only about buckets name containing a string. + - Limits buckets to only buckets who's name contain the string in I(name_filter). type: str default: "" version_added: 1.4.0 bucket_facts: description: - Retrieve requested S3 bucket detailed information - - Each bucket_X option executes one API call, hence many options=true will case slower module execution - - You can limit buckets by using I(name) or I(name_filter) option + - Each bucket_X option executes one API call, hence many options being set to C(true) will cause slower module execution. + - You can limit buckets by using the I(name) or I(name_filter) option. suboptions: bucket_accelerate_configuration: - description: Retrive S3 accelerate configuration + description: Retrive S3 accelerate configuration. type: bool default: False bucket_location: - description: Retrive S3 bucket location + description: Retrive S3 bucket location. type: bool default: False bucket_replication: - description: Retrive S3 bucket replication + description: Retrive S3 bucket replication. type: bool default: False bucket_acl: - description: Retrive S3 bucket ACLs + description: Retrive S3 bucket ACLs. type: bool default: False bucket_logging: - description: Retrive S3 bucket logging + description: Retrive S3 bucket logging. type: bool default: False bucket_request_payment: - description: Retrive S3 bucket request payment + description: Retrive S3 bucket request payment. type: bool default: False bucket_tagging: - description: Retrive S3 bucket tagging + description: Retrive S3 bucket tagging. type: bool default: False bucket_cors: - description: Retrive S3 bucket CORS configuration + description: Retrive S3 bucket CORS configuration. type: bool default: False bucket_notification_configuration: - description: Retrive S3 bucket notification configuration + description: Retrive S3 bucket notification configuration. type: bool default: False bucket_encryption: - description: Retrive S3 bucket encryption + description: Retrive S3 bucket encryption. type: bool default: False bucket_ownership_controls: - description: Retrive S3 ownership controls + description: Retrive S3 ownership controls. type: bool default: False bucket_website: - description: Retrive S3 bucket website + description: Retrive S3 bucket website. type: bool default: False bucket_policy: - description: Retrive S3 bucket policy + description: Retrive S3 bucket policy. type: bool default: False bucket_policy_status: - description: Retrive S3 bucket policy status + description: Retrive S3 bucket policy status. type: bool default: False bucket_lifecycle_configuration: - description: Retrive S3 bucket lifecycle configuration + description: Retrive S3 bucket lifecycle configuration. type: bool default: False public_access_block: - description: Retrive S3 bucket public access block + description: Retrive S3 bucket public access block. type: bool default: False type: dict version_added: 1.4.0 transform_location: description: - - S3 bucket location for default us-east-1 is normally reported as 'null' - - setting this option to 'true' will return 'us-east-1' instead - - affects only queries with I(bucket_facts) > I(bucket_location) = true + - S3 bucket location for default us-east-1 is normally reported as C(null). + - Setting this option to C(true) will return C(us-east-1) instead. + - Affects only queries with I(bucket_facts=true) and I(bucket_location=true). type: bool - default: False - version_added: 1.4.0 - -extends_documentation_fragment: -- amazon.aws.aws -- amazon.aws.ec2 - + default: FalseBucket public access block configuration ''' EXAMPLES = ''' @@ -162,17 +156,243 @@ ''' RETURN = ''' -buckets: +bucket_list: description: "List of buckets" returned: always - sample: - - creation_date: '2017-07-06 15:05:12 +00:00' - name: my_bucket - # bucket facts if requested - bucket_location: dictionary data - bucket_cors: dictionary data - # ...etc - type: list + type: complex + contains: + name: + description: Bucket name. + returned: always + type: str + sample: a-testing-bucket-name + creation_date: + description: Bucket creation date timestamp. + returned: always + type: str + sample: "2021-01-21T12:44:10+00:00" + public_access_block: + description: Bucket public access block configuration. + returned: when I(bucket_facts=true) and I(public_access_block=true) + type: complex + contains: + PublicAccessBlockConfiguration: + description: PublicAccessBlockConfiguration data. + returned: when PublicAccessBlockConfiguration is defined for the bucket + type: complex + contains: + BlockPublicAcls: + description: BlockPublicAcls setting value. + type: bool + sample: true + BlockPublicPolicy: + description: BlockPublicPolicy setting value. + type: bool + sample: true + IgnorePublicAcls: + description: IgnorePublicAcls setting value. + type: bool + sample: true + RestrictPublicBuckets: + description: RestrictPublicBuckets setting value. + type: bool + sample: true + bucket_name_filter: + description: String used to limit buckets. See I(name_filter). + returned: when I(name_filter) is defined + type: str + sample: filter-by-this-string + bucket_acl: + description: Bucket ACL configuration. + returned: when I(bucket_facts=true) and I(bucket_acl=true) + type: complex + contains: + Grants: + description: List of ACL grants. + type: list + sample: [] + Owner: + description: Bucket owner information. + type: complex + contains: + DisplayName: + description: Bucket owner user display name. + returned: str + sample: username + ID: + description: Bucket owner user ID. + returned: str + sample: 123894e509349etc + bucket_cors: + description: Bucket CORS configuration. + returned: when I(bucket_facts=true) and I(bucket_cors=true) + type: complex + contains: + CORSRules: + description: Bucket CORS configuration. + returned: when CORS rules are defined for the bucket + type: list + sample: [] + bucket_encryption: + description: Bucket encryption configuration. + returned: when I(bucket_facts=true) and I(bucket_encryption=true) + type: complex + contains: + ServerSideEncryptionConfiguration: + description: ServerSideEncryptionConfiguration configuration. + returned: when encryption is enabled on the bucket + type: complex + contains: + Rules: + description: List of applied encryptio rules. + returned: when encryption is enabled on the bucket + type: list + sample: { "ApplyServerSideEncryptionByDefault": { "SSEAlgorithm": "AES256" }, "BucketKeyEnabled": False } + bucket_lifecycle_configuration: + description: Bucket lifecycle configuration settings. + returned: when I(bucket_facts=true) and I(bucket_lifecycle_configuration=true) + type: complex + contains: + Rules: + description: List of lifecycle management rules. + returned: when lifecycle configuration is present + type: list + sample: [{ "Status": "Enabled", "ID": "example-rule" }] + bucket_location: + description: Bucket location. + returned: when I(bucket_facts=true) and I(bucket_location=true) + type: complex + contains: + LocationConstraint: + description: AWS region. + returned: always + type: str + sample: us-east-2 + bucket_logging: + description: Server access logging configuration. + returned: when I(bucket_facts=true) and I(bucket_logging=true) + type: complex + contains: + LoggingEnabled: + description: Server access logging configuration. + returned: when server access logging is defined for the bucket + type: complex + contains: + TargetBucket: + description: Target bucket name. + returned: always + type: str + sample: logging-bucket-name + TargetPrefix: + description: Prefix in target bucket. + returned: always + type: str + sample: "" + bucket_notification_configuration: + description: Bucket notification settings. + returned: when I(bucket_facts=true) and I(bucket_notification_configuration=true) + type: complex + contains: + TopicConfigurations: + description: List of notification events configurations. + returned: when at least one notification is configured + type: list + sample: [] + bucket_ownership_controls: + description: Preffered object ownership settings. + returned: when I(bucket_facts=true) and I(bucket_ownership_controls=true) + type: complex + contains: + OwnershipControls: + description: Object ownership settings. + returned: when ownership controls are defined for the bucket + type: complex + contains: + Rules: + description: List of ownership rules. + returned: when ownership rule is defined + type: list + sample: [{ "ObjectOwnership:": "ObjectWriter" }] + bucket_policy: + description: Bucket policy contents. + returned: when I(bucket_facts=true) and I(bucket_policy=true) + type: str + sample: '{"Version":"2012-10-17","Statement":[{"Sid":"AddCannedAcl","Effect":"Allow",..}}]}' + bucket_policy_status: + description: Status of bucket policy. + returned: when I(bucket_facts=true) and I(bucket_policy_status=true) + type: complex + contains: + PolicyStatus: + description: Status of bucket policy. + returned: when bucket policy is present + type: complex + contains: + IsPublic: + description: Report bucket policy public status. + returned: when bucket policy is present + type: bool + sample: True + bucket_replication: + description: Replication configuration settings. + returned: when I(bucket_facts=true) and I(bucket_replication=true) + type: complex + contains: + Role: + description: IAM role used for replication. + returned: when replication rule is defined + type: str + sample: "arn:aws:iam::123:role/example-role" + Rules: + description: List of replication rules. + returned: when replication rule is defined + type: list + sample: [{ "ID": "rule-1", "Filter": "{}" }] + bucket_request_payment: + description: Requester pays setting. + returned: when I(bucket_facts=true) and I(bucket_request_payment=true) + type: complex + contains: + Payer: + description: Current payer. + returned: always + type: str + sample: BucketOwner + bucket_tagging: + description: Bucket tags. + returned: when I(bucket_facts=true) and I(bucket_tagging=true) + type: dict + sample: { "Tag1": "Value1", "Tag2": "Value2" } + bucket_website: + description: Static website hosting. + returned: when I(bucket_facts=true) and I(bucket_website=true) + type: complex + contains: + ErrorDocument: + description: Object serving as HTTP error page. + returned: when static website hosting is enabled + type: dict + sample: { "Key": "error.html" } + IndexDocument: + description: Object serving as HTTP index page. + returned: when static website hosting is enabled + type: dict + sample: { "Suffix": "error.html" } + RedirectAllRequestsTo: + description: Website redict settings. + returned: when redirect requests is configured + type: complex + contains: + HostName: + description: Hostname to redirect. + returned: always + type: str + sample: www.example.com + Protocol: + description: Protocol used for redirect. + returned: always + type: str + sample: https ''' try: From 2f6f32ec0958fdbb9ea2765a7d1cea995a3bc0b6 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Wed, 3 Feb 2021 10:08:43 +0100 Subject: [PATCH 58/63] Update plugins/modules/aws_s3_bucket_info.py --- plugins/modules/aws_s3_bucket_info.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 428607af57c..ee328aeac94 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -112,7 +112,11 @@ - Setting this option to C(true) will return C(us-east-1) instead. - Affects only queries with I(bucket_facts=true) and I(bucket_location=true). type: bool - default: FalseBucket public access block configuration + default: False +version_added: 1.4.0 +extends_documentation_fragment: +- amazon.aws.aws +- amazon.aws.ec2 ''' EXAMPLES = ''' From 202025461637a9e08ffbc6a4f2c3a24c194f6d3a Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Wed, 3 Feb 2021 10:09:36 +0100 Subject: [PATCH 59/63] Update plugins/modules/aws_s3_bucket_info.py --- plugins/modules/aws_s3_bucket_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index ee328aeac94..471affb8e68 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -113,7 +113,7 @@ - Affects only queries with I(bucket_facts=true) and I(bucket_location=true). type: bool default: False -version_added: 1.4.0 + version_added: 1.4.0 extends_documentation_fragment: - amazon.aws.aws - amazon.aws.ec2 From 9bbc1ab758460f736e5672dec0f7409ecc6cb102 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Tue, 9 Feb 2021 10:29:05 +0100 Subject: [PATCH 60/63] Fixing RETURN indentation --- plugins/modules/aws_s3_bucket_info.py | 290 +++++++++++++------------- 1 file changed, 145 insertions(+), 145 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index 471affb8e68..f0bbc560862 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -252,151 +252,151 @@ returned: when encryption is enabled on the bucket type: list sample: { "ApplyServerSideEncryptionByDefault": { "SSEAlgorithm": "AES256" }, "BucketKeyEnabled": False } - bucket_lifecycle_configuration: - description: Bucket lifecycle configuration settings. - returned: when I(bucket_facts=true) and I(bucket_lifecycle_configuration=true) - type: complex - contains: - Rules: - description: List of lifecycle management rules. - returned: when lifecycle configuration is present - type: list - sample: [{ "Status": "Enabled", "ID": "example-rule" }] - bucket_location: - description: Bucket location. - returned: when I(bucket_facts=true) and I(bucket_location=true) - type: complex - contains: - LocationConstraint: - description: AWS region. - returned: always - type: str - sample: us-east-2 - bucket_logging: - description: Server access logging configuration. - returned: when I(bucket_facts=true) and I(bucket_logging=true) - type: complex - contains: - LoggingEnabled: - description: Server access logging configuration. - returned: when server access logging is defined for the bucket - type: complex - contains: - TargetBucket: - description: Target bucket name. - returned: always - type: str - sample: logging-bucket-name - TargetPrefix: - description: Prefix in target bucket. - returned: always - type: str - sample: "" - bucket_notification_configuration: - description: Bucket notification settings. - returned: when I(bucket_facts=true) and I(bucket_notification_configuration=true) - type: complex - contains: - TopicConfigurations: - description: List of notification events configurations. - returned: when at least one notification is configured - type: list - sample: [] - bucket_ownership_controls: - description: Preffered object ownership settings. - returned: when I(bucket_facts=true) and I(bucket_ownership_controls=true) - type: complex - contains: - OwnershipControls: - description: Object ownership settings. - returned: when ownership controls are defined for the bucket - type: complex - contains: - Rules: - description: List of ownership rules. - returned: when ownership rule is defined - type: list - sample: [{ "ObjectOwnership:": "ObjectWriter" }] - bucket_policy: - description: Bucket policy contents. - returned: when I(bucket_facts=true) and I(bucket_policy=true) - type: str - sample: '{"Version":"2012-10-17","Statement":[{"Sid":"AddCannedAcl","Effect":"Allow",..}}]}' - bucket_policy_status: - description: Status of bucket policy. - returned: when I(bucket_facts=true) and I(bucket_policy_status=true) - type: complex - contains: - PolicyStatus: - description: Status of bucket policy. - returned: when bucket policy is present - type: complex - contains: - IsPublic: - description: Report bucket policy public status. - returned: when bucket policy is present - type: bool - sample: True - bucket_replication: - description: Replication configuration settings. - returned: when I(bucket_facts=true) and I(bucket_replication=true) - type: complex - contains: - Role: - description: IAM role used for replication. - returned: when replication rule is defined - type: str - sample: "arn:aws:iam::123:role/example-role" - Rules: - description: List of replication rules. - returned: when replication rule is defined - type: list - sample: [{ "ID": "rule-1", "Filter": "{}" }] - bucket_request_payment: - description: Requester pays setting. - returned: when I(bucket_facts=true) and I(bucket_request_payment=true) - type: complex - contains: - Payer: - description: Current payer. - returned: always - type: str - sample: BucketOwner - bucket_tagging: - description: Bucket tags. - returned: when I(bucket_facts=true) and I(bucket_tagging=true) - type: dict - sample: { "Tag1": "Value1", "Tag2": "Value2" } - bucket_website: - description: Static website hosting. - returned: when I(bucket_facts=true) and I(bucket_website=true) - type: complex - contains: - ErrorDocument: - description: Object serving as HTTP error page. - returned: when static website hosting is enabled - type: dict - sample: { "Key": "error.html" } - IndexDocument: - description: Object serving as HTTP index page. - returned: when static website hosting is enabled - type: dict - sample: { "Suffix": "error.html" } - RedirectAllRequestsTo: - description: Website redict settings. - returned: when redirect requests is configured - type: complex - contains: - HostName: - description: Hostname to redirect. - returned: always - type: str - sample: www.example.com - Protocol: - description: Protocol used for redirect. - returned: always - type: str - sample: https + bucket_lifecycle_configuration: + description: Bucket lifecycle configuration settings. + returned: when I(bucket_facts=true) and I(bucket_lifecycle_configuration=true) + type: complex + contains: + Rules: + description: List of lifecycle management rules. + returned: when lifecycle configuration is present + type: list + sample: [{ "Status": "Enabled", "ID": "example-rule" }] + bucket_location: + description: Bucket location. + returned: when I(bucket_facts=true) and I(bucket_location=true) + type: complex + contains: + LocationConstraint: + description: AWS region. + returned: always + type: str + sample: us-east-2 + bucket_logging: + description: Server access logging configuration. + returned: when I(bucket_facts=true) and I(bucket_logging=true) + type: complex + contains: + LoggingEnabled: + description: Server access logging configuration. + returned: when server access logging is defined for the bucket + type: complex + contains: + TargetBucket: + description: Target bucket name. + returned: always + type: str + sample: logging-bucket-name + TargetPrefix: + description: Prefix in target bucket. + returned: always + type: str + sample: "" + bucket_notification_configuration: + description: Bucket notification settings. + returned: when I(bucket_facts=true) and I(bucket_notification_configuration=true) + type: complex + contains: + TopicConfigurations: + description: List of notification events configurations. + returned: when at least one notification is configured + type: list + sample: [] + bucket_ownership_controls: + description: Preffered object ownership settings. + returned: when I(bucket_facts=true) and I(bucket_ownership_controls=true) + type: complex + contains: + OwnershipControls: + description: Object ownership settings. + returned: when ownership controls are defined for the bucket + type: complex + contains: + Rules: + description: List of ownership rules. + returned: when ownership rule is defined + type: list + sample: [{ "ObjectOwnership:": "ObjectWriter" }] + bucket_policy: + description: Bucket policy contents. + returned: when I(bucket_facts=true) and I(bucket_policy=true) + type: str + sample: '{"Version":"2012-10-17","Statement":[{"Sid":"AddCannedAcl","Effect":"Allow",..}}]}' + bucket_policy_status: + description: Status of bucket policy. + returned: when I(bucket_facts=true) and I(bucket_policy_status=true) + type: complex + contains: + PolicyStatus: + description: Status of bucket policy. + returned: when bucket policy is present + type: complex + contains: + IsPublic: + description: Report bucket policy public status. + returned: when bucket policy is present + type: bool + sample: True + bucket_replication: + description: Replication configuration settings. + returned: when I(bucket_facts=true) and I(bucket_replication=true) + type: complex + contains: + Role: + description: IAM role used for replication. + returned: when replication rule is defined + type: str + sample: "arn:aws:iam::123:role/example-role" + Rules: + description: List of replication rules. + returned: when replication rule is defined + type: list + sample: [{ "ID": "rule-1", "Filter": "{}" }] + bucket_request_payment: + description: Requester pays setting. + returned: when I(bucket_facts=true) and I(bucket_request_payment=true) + type: complex + contains: + Payer: + description: Current payer. + returned: always + type: str + sample: BucketOwner + bucket_tagging: + description: Bucket tags. + returned: when I(bucket_facts=true) and I(bucket_tagging=true) + type: dict + sample: { "Tag1": "Value1", "Tag2": "Value2" } + bucket_website: + description: Static website hosting. + returned: when I(bucket_facts=true) and I(bucket_website=true) + type: complex + contains: + ErrorDocument: + description: Object serving as HTTP error page. + returned: when static website hosting is enabled + type: dict + sample: { "Key": "error.html" } + IndexDocument: + description: Object serving as HTTP index page. + returned: when static website hosting is enabled + type: dict + sample: { "Suffix": "error.html" } + RedirectAllRequestsTo: + description: Website redict settings. + returned: when redirect requests is configured + type: complex + contains: + HostName: + description: Hostname to redirect. + returned: always + type: str + sample: www.example.com + Protocol: + description: Protocol used for redirect. + returned: always + type: str + sample: https ''' try: From 3c623097182b9ad2b0f102e9270b17ee7bc55339 Mon Sep 17 00:00:00 2001 From: Milan Zink Date: Tue, 9 Feb 2021 12:21:22 +0100 Subject: [PATCH 61/63] Fixing missing type: in RETURN --- plugins/modules/aws_s3_bucket_info.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/modules/aws_s3_bucket_info.py b/plugins/modules/aws_s3_bucket_info.py index f0bbc560862..05d92310013 100644 --- a/plugins/modules/aws_s3_bucket_info.py +++ b/plugins/modules/aws_s3_bucket_info.py @@ -221,11 +221,13 @@ contains: DisplayName: description: Bucket owner user display name. - returned: str + returned: always + type: str sample: username ID: description: Bucket owner user ID. - returned: str + returned: always + type: str sample: 123894e509349etc bucket_cors: description: Bucket CORS configuration. From f83b88b0cd8a50f38744c9e10fe89e9a9f4d34b4 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Tue, 9 Feb 2021 13:40:38 +0100 Subject: [PATCH 62/63] Revert changes to changelogs/changelog.yaml --- changelogs/changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelogs/changelog.yaml b/changelogs/changelog.yaml index 72a5ee46f29..3fb51ebee6d 100644 --- a/changelogs/changelog.yaml +++ b/changelogs/changelog.yaml @@ -775,7 +775,7 @@ releases: behavior (https://github.com/ansible-collections/community.aws/pull/318). minor_changes: - ec2_vpc_igw - Add AWSRetry decorators to improve reliability (https://github.com/ansible-collections/community.aws/pull/318). - - ec2_vpc_igw - Add ``purge_tagsminor_changes:`` parameter so that tags can be added without + - ec2_vpc_igw - Add ``purge_tags:`` parameter so that tags can be added without purging existing tags to match the collection standard tagging behaviour (https://github.com/ansible-collections/community.aws/pull/318). - ec2_vpc_igw_info - Add AWSRetry decorators to improve reliability (https://github.com/ansible-collections/community.aws/pull/318). - ec2_vpc_igw_info - Add ``convert_tags`` parameter so that tags can be returned From 9ab066caf52cb42113fe8b63f2a6a11197c21971 Mon Sep 17 00:00:00 2001 From: Mark Chappell Date: Tue, 9 Feb 2021 13:41:09 +0100 Subject: [PATCH 63/63] Revert changes to changelogs/changelog.yaml --- changelogs/changelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelogs/changelog.yaml b/changelogs/changelog.yaml index 3fb51ebee6d..25102ab8cc6 100644 --- a/changelogs/changelog.yaml +++ b/changelogs/changelog.yaml @@ -775,7 +775,7 @@ releases: behavior (https://github.com/ansible-collections/community.aws/pull/318). minor_changes: - ec2_vpc_igw - Add AWSRetry decorators to improve reliability (https://github.com/ansible-collections/community.aws/pull/318). - - ec2_vpc_igw - Add ``purge_tags:`` parameter so that tags can be added without + - ec2_vpc_igw - Add ``purge_tags`` parameter so that tags can be added without purging existing tags to match the collection standard tagging behaviour (https://github.com/ansible-collections/community.aws/pull/318). - ec2_vpc_igw_info - Add AWSRetry decorators to improve reliability (https://github.com/ansible-collections/community.aws/pull/318). - ec2_vpc_igw_info - Add ``convert_tags`` parameter so that tags can be returned