From d3624757acd80de0093d5e33d1b25d8d6e1faf93 Mon Sep 17 00:00:00 2001 From: flowerysong Date: Thu, 11 Jun 2020 13:25:04 -0400 Subject: [PATCH] Move module_utils out of subdir (#26) * Move module_utils out of subdir and update imports * Use relative imports * Always get dict transformations from their canonical home * Split imports by line to match tremble's other refactoring * Make import order more consistent * Temporarily restore legacy module_utils files --- plugins/lookup/aws_account_attribute.py | 11 +- plugins/lookup/aws_ssm.py | 14 +- plugins/module_utils/acm.py | 217 +++++ plugins/module_utils/batch.py | 106 +++ plugins/module_utils/cloudfront_facts.py | 238 ++++++ plugins/module_utils/core.py | 340 ++++++++ plugins/module_utils/direct_connect.py | 89 ++ plugins/module_utils/ec2.py | 21 +- plugins/module_utils/elb_utils.py | 111 +++ plugins/module_utils/elbv2.py | 895 ++++++++++++++++++++ plugins/module_utils/iam.py | 49 ++ plugins/module_utils/rds.py | 235 +++++ plugins/module_utils/s3.py | 50 ++ plugins/module_utils/urls.py | 213 +++++ plugins/module_utils/waf.py | 224 +++++ plugins/module_utils/waiters.py | 405 +++++++++ plugins/modules/aws_az_info.py | 9 +- plugins/modules/aws_caller_info.py | 9 +- plugins/modules/aws_s3.py | 17 +- plugins/modules/cloudformation.py | 11 +- plugins/modules/cloudformation_info.py | 13 +- plugins/modules/ec2.py | 9 +- plugins/modules/ec2_ami.py | 15 +- plugins/modules/ec2_ami_info.py | 14 +- plugins/modules/ec2_elb_lb.py | 11 +- plugins/modules/ec2_eni.py | 12 +- plugins/modules/ec2_eni_info.py | 7 +- plugins/modules/ec2_group.py | 32 +- plugins/modules/ec2_group_info.py | 10 +- plugins/modules/ec2_key.py | 7 +- plugins/modules/ec2_snapshot.py | 6 +- plugins/modules/ec2_snapshot_info.py | 9 +- plugins/modules/ec2_tag.py | 9 +- plugins/modules/ec2_tag_info.py | 6 +- plugins/modules/ec2_vol.py | 10 +- plugins/modules/ec2_vol_info.py | 16 +- plugins/modules/ec2_vpc_dhcp_option.py | 8 +- plugins/modules/ec2_vpc_dhcp_option_info.py | 7 +- plugins/modules/ec2_vpc_net.py | 20 +- plugins/modules/ec2_vpc_net_info.py | 15 +- plugins/modules/ec2_vpc_subnet.py | 18 +- plugins/modules/ec2_vpc_subnet_info.py | 9 +- plugins/modules/s3_bucket.py | 22 +- 43 files changed, 3385 insertions(+), 164 deletions(-) create mode 100644 plugins/module_utils/acm.py create mode 100644 plugins/module_utils/batch.py create mode 100644 plugins/module_utils/cloudfront_facts.py create mode 100644 plugins/module_utils/core.py create mode 100644 plugins/module_utils/direct_connect.py create mode 100644 plugins/module_utils/elb_utils.py create mode 100644 plugins/module_utils/elbv2.py create mode 100644 plugins/module_utils/iam.py create mode 100644 plugins/module_utils/rds.py create mode 100644 plugins/module_utils/s3.py create mode 100644 plugins/module_utils/urls.py create mode 100644 plugins/module_utils/waf.py create mode 100644 plugins/module_utils/waiters.py diff --git a/plugins/lookup/aws_account_attribute.py b/plugins/lookup/aws_account_attribute.py index d8d4ff8f5b2..4c612033567 100644 --- a/plugins/lookup/aws_account_attribute.py +++ b/plugins/lookup/aws_account_attribute.py @@ -52,6 +52,8 @@ (or all attributes if one is not specified). """ +import os + from ansible.errors import AnsibleError try: @@ -60,12 +62,13 @@ except ImportError: raise AnsibleError("The lookup aws_account_attribute requires boto3 and botocore.") -from ansible.plugins import AnsiblePlugin -from ansible.plugins.lookup import LookupBase -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_conn, get_aws_connection_info from ansible.module_utils._text import to_native from ansible.module_utils.six import string_types -import os +from ansible.plugins import AnsiblePlugin +from ansible.plugins.lookup import LookupBase + +from ..module_utils.ec2 import boto3_conn +from ..module_utils.ec2 import get_aws_connection_info def _boto3_conn(region, credentials): diff --git a/plugins/lookup/aws_ssm.py b/plugins/lookup/aws_ssm.py index b988562f338..fe2798e5b4d 100644 --- a/plugins/lookup/aws_ssm.py +++ b/plugins/lookup/aws_ssm.py @@ -105,12 +105,6 @@ ''' -from ansible.module_utils._text import to_native -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO3, boto3_tag_list_to_ansible_dict -from ansible.errors import AnsibleError -from ansible.plugins.lookup import LookupBase -from ansible.utils.display import Display - try: from botocore.exceptions import ClientError import botocore @@ -118,6 +112,14 @@ except ImportError: pass # will be captured by imported HAS_BOTO3 +from ansible.errors import AnsibleError +from ansible.module_utils._text import to_native +from ansible.plugins.lookup import LookupBase +from ansible.utils.display import Display + +from ..module_utils.ec2 import HAS_BOTO3 +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict + display = Display() diff --git a/plugins/module_utils/acm.py b/plugins/module_utils/acm.py new file mode 100644 index 00000000000..12b4e42ab8d --- /dev/null +++ b/plugins/module_utils/acm.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2019 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . +# +# Author: +# - Matthew Davis +# on behalf of Telstra Corporation Limited +# +# Common functionality to be used by the modules: +# - acm + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +""" +Common Amazon Certificate Manager facts shared between modules +""" +import traceback + +try: + import botocore + from botocore.exceptions import BotoCoreError, ClientError +except ImportError: + pass + +from ansible.module_utils._text import to_bytes +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from .ec2 import AWSRetry +from .ec2 import ansible_dict_to_boto3_tag_list +from .ec2 import boto3_conn +from .ec2 import boto3_tag_list_to_ansible_dict +from .ec2 import get_aws_connection_info + + +class ACMServiceManager(object): + """Handles ACM Facts Services""" + + def __init__(self, module): + self.module = module + + region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True) + self.client = module.client('acm') + + @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['RequestInProgressException']) + def delete_certificate_with_backoff(self, client, arn): + client.delete_certificate(CertificateArn=arn) + + def delete_certificate(self, client, module, arn): + module.debug("Attempting to delete certificate %s" % arn) + try: + self.delete_certificate_with_backoff(client, arn) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg="Couldn't delete certificate %s" % arn) + module.debug("Successfully deleted certificate %s" % arn) + + @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['RequestInProgressException']) + def list_certificates_with_backoff(self, client, statuses=None): + paginator = client.get_paginator('list_certificates') + kwargs = dict() + if statuses: + kwargs['CertificateStatuses'] = statuses + return paginator.paginate(**kwargs).build_full_result()['CertificateSummaryList'] + + @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException', 'RequestInProgressException']) + def get_certificate_with_backoff(self, client, certificate_arn): + response = client.get_certificate(CertificateArn=certificate_arn) + # strip out response metadata + return {'Certificate': response['Certificate'], + 'CertificateChain': response['CertificateChain']} + + @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException', 'RequestInProgressException']) + def describe_certificate_with_backoff(self, client, certificate_arn): + return client.describe_certificate(CertificateArn=certificate_arn)['Certificate'] + + @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException', 'RequestInProgressException']) + def list_certificate_tags_with_backoff(self, client, certificate_arn): + return client.list_tags_for_certificate(CertificateArn=certificate_arn)['Tags'] + + # Returns a list of certificates + # if domain_name is specified, returns only certificates with that domain + # if an ARN is specified, returns only that certificate + # only_tags is a dict, e.g. {'key':'value'}. If specified this function will return + # only certificates which contain all those tags (key exists, value matches). + def get_certificates(self, client, module, domain_name=None, statuses=None, arn=None, only_tags=None): + try: + all_certificates = self.list_certificates_with_backoff(client=client, statuses=statuses) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg="Couldn't obtain certificates") + if domain_name: + certificates = [cert for cert in all_certificates + if cert['DomainName'] == domain_name] + else: + certificates = all_certificates + + if arn: + # still return a list, not just one item + certificates = [c for c in certificates if c['CertificateArn'] == arn] + + results = [] + for certificate in certificates: + try: + cert_data = self.describe_certificate_with_backoff(client, certificate['CertificateArn']) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg="Couldn't obtain certificate metadata for domain %s" % certificate['DomainName']) + + # in some states, ACM resources do not have a corresponding cert + if cert_data['Status'] not in ['PENDING_VALIDATION', 'VALIDATION_TIMED_OUT', 'FAILED']: + try: + cert_data.update(self.get_certificate_with_backoff(client, certificate['CertificateArn'])) + except (BotoCoreError, ClientError, KeyError) as e: + module.fail_json_aws(e, msg="Couldn't obtain certificate data for domain %s" % certificate['DomainName']) + cert_data = camel_dict_to_snake_dict(cert_data) + try: + tags = self.list_certificate_tags_with_backoff(client, certificate['CertificateArn']) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg="Couldn't obtain tags for domain %s" % certificate['DomainName']) + + cert_data['tags'] = boto3_tag_list_to_ansible_dict(tags) + results.append(cert_data) + + if only_tags: + for tag_key in only_tags: + try: + results = [c for c in results if ('tags' in c) and (tag_key in c['tags']) and (c['tags'][tag_key] == only_tags[tag_key])] + except (TypeError, AttributeError) as e: + for c in results: + if 'tags' not in c: + module.debug("cert is %s" % str(c)) + module.fail_json(msg="ACM tag filtering err", exception=e) + + return results + + # returns the domain name of a certificate (encoded in the public cert) + # for a given ARN + # A cert with that ARN must already exist + def get_domain_of_cert(self, client, module, arn): + if arn is None: + module.fail(msg="Internal error with ACM domain fetching, no certificate ARN specified") + try: + cert_data = self.describe_certificate_with_backoff(client=client, certificate_arn=arn) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg="Couldn't obtain certificate data for arn %s" % arn) + return cert_data['DomainName'] + + @AWSRetry.backoff(tries=5, delay=5, backoff=2.0) + def import_certificate_with_backoff(self, client, certificate, private_key, certificate_chain, arn): + if certificate_chain: + if arn: + ret = client.import_certificate(Certificate=to_bytes(certificate), + PrivateKey=to_bytes(private_key), + CertificateChain=to_bytes(certificate_chain), + CertificateArn=arn) + else: + ret = client.import_certificate(Certificate=to_bytes(certificate), + PrivateKey=to_bytes(private_key), + CertificateChain=to_bytes(certificate_chain)) + else: + if arn: + ret = client.import_certificate(Certificate=to_bytes(certificate), + PrivateKey=to_bytes(private_key), + CertificateArn=arn) + else: + ret = client.import_certificate(Certificate=to_bytes(certificate), + PrivateKey=to_bytes(private_key)) + return ret['CertificateArn'] + + # Tags are a normal Ansible style dict + # {'Key':'Value'} + @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException', 'RequestInProgressException']) + def tag_certificate_with_backoff(self, client, arn, tags): + aws_tags = ansible_dict_to_boto3_tag_list(tags) + client.add_tags_to_certificate(CertificateArn=arn, Tags=aws_tags) + + def import_certificate(self, client, module, certificate, private_key, arn=None, certificate_chain=None, tags=None): + + original_arn = arn + + # upload cert + try: + arn = self.import_certificate_with_backoff(client, certificate, private_key, certificate_chain, arn) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg="Couldn't upload new certificate") + + if original_arn and (arn != original_arn): + # I'm not sure whether the API guarentees that the ARN will not change + # I'm failing just in case. + # If I'm wrong, I'll catch it in the integration tests. + module.fail_json(msg="ARN changed with ACM update, from %s to %s" % (original_arn, arn)) + + # tag that cert + try: + self.tag_certificate_with_backoff(client, arn, tags) + except (BotoCoreError, ClientError) as e: + module.debug("Attempting to delete the cert we just created, arn=%s" % arn) + try: + self.delete_certificate_with_backoff(client, arn) + except Exception as f: + module.warn("Certificate %s exists, and is not tagged. So Ansible will not see it on the next run.") + module.fail_json_aws(e, msg="Couldn't tag certificate %s, couldn't delete it either" % arn) + module.fail_json_aws(e, msg="Couldn't tag certificate %s" % arn) + + return arn diff --git a/plugins/module_utils/batch.py b/plugins/module_utils/batch.py new file mode 100644 index 00000000000..4fe4f09ee05 --- /dev/null +++ b/plugins/module_utils/batch.py @@ -0,0 +1,106 @@ +# Copyright (c) 2017 Ansible Project +# +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +""" +This module adds shared support for Batch modules. +""" + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +try: + from botocore.exceptions import ClientError +except ImportError: + pass + +from ansible.module_utils.common.dict_transformations import snake_dict_to_camel_dict + +from .ec2 import boto3_conn +from .ec2 import get_aws_connection_info + + +class AWSConnection(object): + """ + Create the connection object and client objects as required. + """ + + def __init__(self, ansible_obj, resources, boto3=True): + + ansible_obj.deprecate("The 'ansible.module_utils.aws.batch.AWSConnection' class is deprecated please use 'AnsibleAWSModule.client()'", + version='2.14') + + self.region, self.endpoint, aws_connect_kwargs = get_aws_connection_info(ansible_obj, boto3=boto3) + + self.resource_client = dict() + if not resources: + resources = ['batch'] + + resources.append('iam') + + for resource in resources: + aws_connect_kwargs.update(dict(region=self.region, + endpoint=self.endpoint, + conn_type='client', + resource=resource + )) + self.resource_client[resource] = boto3_conn(ansible_obj, **aws_connect_kwargs) + + # if region is not provided, then get default profile/session region + if not self.region: + self.region = self.resource_client['batch'].meta.region_name + + # set account ID + try: + self.account_id = self.resource_client['iam'].get_user()['User']['Arn'].split(':')[4] + except (ClientError, ValueError, KeyError, IndexError): + self.account_id = '' + + def client(self, resource='batch'): + return self.resource_client[resource] + + +def cc(key): + """ + Changes python key into Camel case equivalent. For example, 'compute_environment_name' becomes + 'computeEnvironmentName'. + + :param key: + :return: + """ + components = key.split('_') + return components[0] + "".join([token.capitalize() for token in components[1:]]) + + +def set_api_params(module, module_params): + """ + Sets module parameters to those expected by the boto3 API. + :param module: + :param module_params: + :return: + """ + api_params = dict((k, v) for k, v in dict(module.params).items() if k in module_params and v is not None) + return snake_dict_to_camel_dict(api_params) diff --git a/plugins/module_utils/cloudfront_facts.py b/plugins/module_utils/cloudfront_facts.py new file mode 100644 index 00000000000..fa9cc720dae --- /dev/null +++ b/plugins/module_utils/cloudfront_facts.py @@ -0,0 +1,238 @@ +# -*- coding: utf-8 -*- +# +# Copyright (c) 2017 Willem van Ketwich +# +# This module is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This software is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this software. If not, see . +# +# Author: +# - Willem van Ketwich +# +# Common functionality to be used by the modules: +# - cloudfront_distribution +# - cloudfront_invalidation +# - cloudfront_origin_access_identity +""" +Common cloudfront facts shared between modules +""" + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +try: + import botocore +except ImportError: + pass + +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from .ec2 import boto3_conn +from .ec2 import boto3_tag_list_to_ansible_dict +from .ec2 import get_aws_connection_info + + +class CloudFrontFactsServiceManager(object): + """Handles CloudFront Facts Services""" + + def __init__(self, module): + self.module = module + + region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True) + self.client = boto3_conn(module, conn_type='client', + resource='cloudfront', region=region, + endpoint=ec2_url, **aws_connect_kwargs) + + def get_distribution(self, distribution_id): + try: + return self.client.get_distribution(Id=distribution_id) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error describing distribution") + + def get_distribution_config(self, distribution_id): + try: + return self.client.get_distribution_config(Id=distribution_id) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error describing distribution configuration") + + def get_origin_access_identity(self, origin_access_identity_id): + try: + return self.client.get_cloud_front_origin_access_identity(Id=origin_access_identity_id) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error describing origin access identity") + + def get_origin_access_identity_config(self, origin_access_identity_id): + try: + return self.client.get_cloud_front_origin_access_identity_config(Id=origin_access_identity_id) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error describing origin access identity configuration") + + def get_invalidation(self, distribution_id, invalidation_id): + try: + return self.client.get_invalidation(DistributionId=distribution_id, Id=invalidation_id) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error describing invalidation") + + def get_streaming_distribution(self, distribution_id): + try: + return self.client.get_streaming_distribution(Id=distribution_id) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error describing streaming distribution") + + def get_streaming_distribution_config(self, distribution_id): + try: + return self.client.get_streaming_distribution_config(Id=distribution_id) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error describing streaming distribution") + + def list_origin_access_identities(self): + try: + paginator = self.client.get_paginator('list_cloud_front_origin_access_identities') + result = paginator.paginate().build_full_result().get('CloudFrontOriginAccessIdentityList', {}) + return result.get('Items', []) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error listing cloud front origin access identities") + + def list_distributions(self, keyed=True): + try: + paginator = self.client.get_paginator('list_distributions') + result = paginator.paginate().build_full_result().get('DistributionList', {}) + distribution_list = result.get('Items', []) + if not keyed: + return distribution_list + return self.keyed_list_helper(distribution_list) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error listing distributions") + + def list_distributions_by_web_acl_id(self, web_acl_id): + try: + result = self.client.list_distributions_by_web_acl_id(WebAclId=web_acl_id) + distribution_list = result.get('DistributionList', {}).get('Items', []) + return self.keyed_list_helper(distribution_list) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error listing distributions by web acl id") + + def list_invalidations(self, distribution_id): + try: + paginator = self.client.get_paginator('list_invalidations') + result = paginator.paginate(DistributionId=distribution_id).build_full_result() + return result.get('InvalidationList', {}).get('Items', []) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error listing invalidations") + + def list_streaming_distributions(self, keyed=True): + try: + paginator = self.client.get_paginator('list_streaming_distributions') + result = paginator.paginate().build_full_result() + streaming_distribution_list = result.get('StreamingDistributionList', {}).get('Items', []) + if not keyed: + return streaming_distribution_list + return self.keyed_list_helper(streaming_distribution_list) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error listing streaming distributions") + + def summary(self): + summary_dict = {} + summary_dict.update(self.summary_get_distribution_list(False)) + summary_dict.update(self.summary_get_distribution_list(True)) + summary_dict.update(self.summary_get_origin_access_identity_list()) + return summary_dict + + def summary_get_origin_access_identity_list(self): + try: + origin_access_identity_list = {'origin_access_identities': []} + origin_access_identities = self.list_origin_access_identities() + for origin_access_identity in origin_access_identities: + oai_id = origin_access_identity['Id'] + oai_full_response = self.get_origin_access_identity(oai_id) + oai_summary = {'Id': oai_id, 'ETag': oai_full_response['ETag']} + origin_access_identity_list['origin_access_identities'].append(oai_summary) + return origin_access_identity_list + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error generating summary of origin access identities") + + def summary_get_distribution_list(self, streaming=False): + try: + list_name = 'streaming_distributions' if streaming else 'distributions' + key_list = ['Id', 'ARN', 'Status', 'LastModifiedTime', 'DomainName', 'Comment', 'PriceClass', 'Enabled'] + distribution_list = {list_name: []} + distributions = self.list_streaming_distributions(False) if streaming else self.list_distributions(False) + for dist in distributions: + temp_distribution = {} + for key_name in key_list: + temp_distribution[key_name] = dist[key_name] + temp_distribution['Aliases'] = [alias for alias in dist['Aliases'].get('Items', [])] + temp_distribution['ETag'] = self.get_etag_from_distribution_id(dist['Id'], streaming) + if not streaming: + temp_distribution['WebACLId'] = dist['WebACLId'] + invalidation_ids = self.get_list_of_invalidation_ids_from_distribution_id(dist['Id']) + if invalidation_ids: + temp_distribution['Invalidations'] = invalidation_ids + resource_tags = self.client.list_tags_for_resource(Resource=dist['ARN']) + temp_distribution['Tags'] = boto3_tag_list_to_ansible_dict(resource_tags['Tags'].get('Items', [])) + distribution_list[list_name].append(temp_distribution) + return distribution_list + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error generating summary of distributions") + except Exception as e: + self.module.fail_json_aws(e, msg="Error generating summary of distributions") + + def get_etag_from_distribution_id(self, distribution_id, streaming): + distribution = {} + if not streaming: + distribution = self.get_distribution(distribution_id) + else: + distribution = self.get_streaming_distribution(distribution_id) + return distribution['ETag'] + + def get_list_of_invalidation_ids_from_distribution_id(self, distribution_id): + try: + invalidation_ids = [] + invalidations = self.list_invalidations(distribution_id) + for invalidation in invalidations: + invalidation_ids.append(invalidation['Id']) + return invalidation_ids + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error getting list of invalidation ids") + + def get_distribution_id_from_domain_name(self, domain_name): + try: + distribution_id = "" + distributions = self.list_distributions(False) + distributions += self.list_streaming_distributions(False) + for dist in distributions: + if 'Items' in dist['Aliases']: + for alias in dist['Aliases']['Items']: + if str(alias).lower() == domain_name.lower(): + distribution_id = dist['Id'] + break + return distribution_id + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error getting distribution id from domain name") + + def get_aliases_from_distribution_id(self, distribution_id): + try: + distribution = self.get_distribution(distribution_id) + return distribution['DistributionConfig']['Aliases'].get('Items', []) + except botocore.exceptions.ClientError as e: + self.module.fail_json_aws(e, msg="Error getting list of aliases from distribution_id") + + def keyed_list_helper(self, list_to_key): + keyed_list = dict() + for item in list_to_key: + distribution_id = item['Id'] + if 'Items' in item['Aliases']: + aliases = item['Aliases']['Items'] + for alias in aliases: + keyed_list.update({alias: item}) + keyed_list.update({distribution_id: item}) + return keyed_list diff --git a/plugins/module_utils/core.py b/plugins/module_utils/core.py new file mode 100644 index 00000000000..7244d2c805e --- /dev/null +++ b/plugins/module_utils/core.py @@ -0,0 +1,340 @@ +# +# Copyright 2017 Michael De La Rue | Ansible +# +# This file is part of Ansible +# +# Ansible is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Ansible. If not, see . + +"""This module adds shared support for generic Amazon AWS modules + +**This code is not yet ready for use in user modules. As of 2017** +**and through to 2018, the interface is likely to change** +**aggressively as the exact correct interface for ansible AWS modules** +**is identified. In particular, until this notice goes away or is** +**changed, methods may disappear from the interface. Please don't** +**publish modules using this except directly to the main Ansible** +**development repository.** + +In order to use this module, include it as part of a custom +module as shown below. + + from ansible.module_utils.aws import AnsibleAWSModule + module = AnsibleAWSModule(argument_spec=dictionary, supports_check_mode=boolean + mutually_exclusive=list1, required_together=list2) + +The 'AnsibleAWSModule' module provides similar, but more restricted, +interfaces to the normal Ansible module. It also includes the +additional methods for connecting to AWS using the standard module arguments + + m.resource('lambda') # - get an AWS connection as a boto3 resource. + +or + + m.client('sts') # - get an AWS connection as a boto3 client. + +To make use of AWSRetry easier, it can now be wrapped around any call from a +module-created client. To add retries to a client, create a client: + + m.client('ec2', retry_decorator=AWSRetry.jittered_backoff(retries=10)) + +Any calls from that client can be made to use the decorator passed at call-time +using the `aws_retry` argument. By default, no retries are used. + + ec2 = m.client('ec2', retry_decorator=AWSRetry.jittered_backoff(retries=10)) + ec2.describe_instances(InstanceIds=['i-123456789'], aws_retry=True) + +The call will be retried the specified number of times, so the calling functions +don't need to be wrapped in the backoff decorator. +""" + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import re +import logging +import traceback +from functools import wraps +from distutils.version import LooseVersion + +try: + from cStringIO import StringIO +except ImportError: + # Python 3 + from io import StringIO + +from ansible.module_utils.basic import AnsibleModule, missing_required_lib +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict +from ansible.module_utils._text import to_native + +from .ec2 import HAS_BOTO3 +from .ec2 import boto3_conn +from .ec2 import ec2_argument_spec +from .ec2 import get_aws_connection_info +from .ec2 import get_aws_region + +# We will also export HAS_BOTO3 so end user modules can use it. +__all__ = ('AnsibleAWSModule', 'HAS_BOTO3', 'is_boto3_error_code') + + +class AnsibleAWSModule(object): + """An ansible module class for AWS modules + + AnsibleAWSModule provides an a class for building modules which + connect to Amazon Web Services. The interface is currently more + restricted than the basic module class with the aim that later the + basic module class can be reduced. If you find that any key + feature is missing please contact the author/Ansible AWS team + (available on #ansible-aws on IRC) to request the additional + features needed. + """ + default_settings = { + "default_args": True, + "check_boto3": True, + "auto_retry": True, + "module_class": AnsibleModule + } + + def __init__(self, **kwargs): + local_settings = {} + for key in AnsibleAWSModule.default_settings: + try: + local_settings[key] = kwargs.pop(key) + except KeyError: + local_settings[key] = AnsibleAWSModule.default_settings[key] + self.settings = local_settings + + if local_settings["default_args"]: + # ec2_argument_spec contains the region so we use that; there's a patch coming which + # will add it to aws_argument_spec so if that's accepted then later we should change + # over + argument_spec_full = ec2_argument_spec() + try: + argument_spec_full.update(kwargs["argument_spec"]) + except (TypeError, NameError): + pass + kwargs["argument_spec"] = argument_spec_full + + self._module = AnsibleAWSModule.default_settings["module_class"](**kwargs) + + if local_settings["check_boto3"] and not HAS_BOTO3: + self._module.fail_json( + msg=missing_required_lib('botocore or boto3')) + + self.check_mode = self._module.check_mode + self._diff = self._module._diff + self._name = self._module._name + + self._botocore_endpoint_log_stream = StringIO() + self.logger = None + if self.params.get('debug_botocore_endpoint_logs'): + self.logger = logging.getLogger('botocore.endpoint') + self.logger.setLevel(logging.DEBUG) + self.logger.addHandler(logging.StreamHandler(self._botocore_endpoint_log_stream)) + + @property + def params(self): + return self._module.params + + def _get_resource_action_list(self): + actions = [] + for ln in self._botocore_endpoint_log_stream.getvalue().split('\n'): + ln = ln.strip() + if not ln: + continue + found_operational_request = re.search(r"OperationModel\(name=.*?\)", ln) + if found_operational_request: + operation_request = found_operational_request.group(0)[20:-1] + resource = re.search(r"https://.*?\.", ln).group(0)[8:-1] + actions.append("{0}:{1}".format(resource, operation_request)) + return list(set(actions)) + + def exit_json(self, *args, **kwargs): + if self.params.get('debug_botocore_endpoint_logs'): + kwargs['resource_actions'] = self._get_resource_action_list() + return self._module.exit_json(*args, **kwargs) + + def fail_json(self, *args, **kwargs): + if self.params.get('debug_botocore_endpoint_logs'): + kwargs['resource_actions'] = self._get_resource_action_list() + return self._module.fail_json(*args, **kwargs) + + def debug(self, *args, **kwargs): + return self._module.debug(*args, **kwargs) + + def warn(self, *args, **kwargs): + return self._module.warn(*args, **kwargs) + + def deprecate(self, *args, **kwargs): + return self._module.deprecate(*args, **kwargs) + + def boolean(self, *args, **kwargs): + return self._module.boolean(*args, **kwargs) + + def md5(self, *args, **kwargs): + return self._module.md5(*args, **kwargs) + + def client(self, service, retry_decorator=None): + region, ec2_url, aws_connect_kwargs = get_aws_connection_info(self, boto3=True) + conn = boto3_conn(self, conn_type='client', resource=service, + region=region, endpoint=ec2_url, **aws_connect_kwargs) + return conn if retry_decorator is None else _RetryingBotoClientWrapper(conn, retry_decorator) + + def resource(self, service): + region, ec2_url, aws_connect_kwargs = get_aws_connection_info(self, boto3=True) + return boto3_conn(self, conn_type='resource', resource=service, + region=region, endpoint=ec2_url, **aws_connect_kwargs) + + @property + def region(self, boto3=True): + return get_aws_region(self, boto3) + + def fail_json_aws(self, exception, msg=None): + """call fail_json with processed exception + + function for converting exceptions thrown by AWS SDK modules, + botocore, boto3 and boto, into nice error messages. + """ + last_traceback = traceback.format_exc() + + # to_native is trusted to handle exceptions that str() could + # convert to text. + try: + except_msg = to_native(exception.message) + except AttributeError: + except_msg = to_native(exception) + + if msg is not None: + message = '{0}: {1}'.format(msg, except_msg) + else: + message = except_msg + + try: + response = exception.response + except AttributeError: + response = None + + failure = dict( + msg=message, + exception=last_traceback, + **self._gather_versions() + ) + + if response is not None: + failure.update(**camel_dict_to_snake_dict(response)) + + self.fail_json(**failure) + + def _gather_versions(self): + """Gather AWS SDK (boto3 and botocore) dependency versions + + Returns {'boto3_version': str, 'botocore_version': str} + Returns {} if neither are installed + """ + if not HAS_BOTO3: + return {} + import boto3 + import botocore + return dict(boto3_version=boto3.__version__, + botocore_version=botocore.__version__) + + def boto3_at_least(self, desired): + """Check if the available boto3 version is greater than or equal to a desired version. + + Usage: + if module.params.get('assign_ipv6_address') and not module.boto3_at_least('1.4.4'): + # conditionally fail on old boto3 versions if a specific feature is not supported + module.fail_json(msg="Boto3 can't deal with EC2 IPv6 addresses before version 1.4.4.") + """ + existing = self._gather_versions() + return LooseVersion(existing['boto3_version']) >= LooseVersion(desired) + + def botocore_at_least(self, desired): + """Check if the available botocore version is greater than or equal to a desired version. + + Usage: + if not module.botocore_at_least('1.2.3'): + module.fail_json(msg='The Serverless Elastic Load Compute Service is not in botocore before v1.2.3') + if not module.botocore_at_least('1.5.3'): + module.warn('Botocore did not include waiters for Service X before 1.5.3. ' + 'To wait until Service X resources are fully available, update botocore.') + """ + existing = self._gather_versions() + return LooseVersion(existing['botocore_version']) >= LooseVersion(desired) + + +class _RetryingBotoClientWrapper(object): + __never_wait = ( + 'get_paginator', 'can_paginate', + 'get_waiter', 'generate_presigned_url', + ) + + def __init__(self, client, retry): + self.client = client + self.retry = retry + + def _create_optional_retry_wrapper_function(self, unwrapped): + retrying_wrapper = self.retry(unwrapped) + + @wraps(unwrapped) + def deciding_wrapper(aws_retry=False, *args, **kwargs): + if aws_retry: + return retrying_wrapper(*args, **kwargs) + else: + return unwrapped(*args, **kwargs) + return deciding_wrapper + + def __getattr__(self, name): + unwrapped = getattr(self.client, name) + if name in self.__never_wait: + return unwrapped + elif callable(unwrapped): + wrapped = self._create_optional_retry_wrapper_function(unwrapped) + setattr(self, name, wrapped) + return wrapped + else: + return unwrapped + + +def is_boto3_error_code(code, e=None): + """Check if the botocore exception is raised by a specific error code. + + Returns ClientError if the error code matches, a dummy exception if it does not have an error code or does not match + + Example: + try: + ec2.describe_instances(InstanceIds=['potato']) + except is_boto3_error_code('InvalidInstanceID.Malformed'): + # handle the error for that code case + except botocore.exceptions.ClientError as e: + # handle the generic error case for all other codes + """ + from botocore.exceptions import ClientError + if e is None: + import sys + dummy, e, dummy = sys.exc_info() + if isinstance(e, ClientError) and e.response['Error']['Code'] == code: + return ClientError + return type('NeverEverRaisedException', (Exception,), {}) + + +def get_boto3_client_method_parameters(client, method_name, required=False): + op = client.meta.method_to_api_mapping.get(method_name) + input_shape = client._service_model.operation_model(op).input_shape + if not input_shape: + parameters = [] + elif required: + parameters = list(input_shape.required_members) + else: + parameters = list(input_shape.members.keys()) + return parameters diff --git a/plugins/module_utils/direct_connect.py b/plugins/module_utils/direct_connect.py new file mode 100644 index 00000000000..24794330f2f --- /dev/null +++ b/plugins/module_utils/direct_connect.py @@ -0,0 +1,89 @@ +# Copyright (c) 2017 Ansible Project +# +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +""" +This module adds shared support for Direct Connect modules. +""" + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import traceback + +try: + import botocore +except ImportError: + pass + +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + + +class DirectConnectError(Exception): + def __init__(self, msg, last_traceback=None, exception=None): + self.msg = msg + self.last_traceback = last_traceback + self.exception = exception + + +def delete_connection(client, connection_id): + try: + client.delete_connection(connectionId=connection_id) + except botocore.exceptions.ClientError as e: + raise DirectConnectError(msg="Failed to delete DirectConnection {0}.".format(connection_id), + last_traceback=traceback.format_exc(), + exception=e) + + +def associate_connection_and_lag(client, connection_id, lag_id): + try: + client.associate_connection_with_lag(connectionId=connection_id, + lagId=lag_id) + except botocore.exceptions.ClientError as e: + raise DirectConnectError(msg="Failed to associate Direct Connect connection {0}" + " with link aggregation group {1}.".format(connection_id, lag_id), + last_traceback=traceback.format_exc(), + exception=e) + + +def disassociate_connection_and_lag(client, connection_id, lag_id): + try: + client.disassociate_connection_from_lag(connectionId=connection_id, + lagId=lag_id) + except botocore.exceptions.ClientError as e: + raise DirectConnectError(msg="Failed to disassociate Direct Connect connection {0}" + " from link aggregation group {1}.".format(connection_id, lag_id), + last_traceback=traceback.format_exc(), + exception=e) + + +def delete_virtual_interface(client, virtual_interface): + try: + client.delete_virtual_interface(virtualInterfaceId=virtual_interface) + except botocore.exceptions.ClientError as e: + raise DirectConnectError(msg="Could not delete virtual interface {0}".format(virtual_interface), + last_traceback=traceback.format_exc(), + exception=e) diff --git a/plugins/module_utils/ec2.py b/plugins/module_utils/ec2.py index 790b1612641..3e3f3c6a71d 100644 --- a/plugins/module_utils/ec2.py +++ b/plugins/module_utils/ec2.py @@ -34,15 +34,20 @@ import sys import traceback +from ansible.module_utils._text import to_native +from ansible.module_utils._text import to_text from ansible.module_utils.ansible_release import __version__ -from ansible.module_utils.basic import missing_required_lib, env_fallback -from ansible.module_utils._text import to_native, to_text -from ansible_collections.amazon.aws.plugins.module_utils.cloud import CloudRetry -from ansible.module_utils.six import string_types, binary_type, text_type -from ansible.module_utils.common.dict_transformations import ( - camel_dict_to_snake_dict, snake_dict_to_camel_dict, - _camel_to_snake, _snake_to_camel, -) +from ansible.module_utils.basic import env_fallback +from ansible.module_utils.basic import missing_required_lib +from ansible.module_utils.common.dict_transformations import _camel_to_snake +from ansible.module_utils.common.dict_transformations import _snake_to_camel +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict +from ansible.module_utils.common.dict_transformations import snake_dict_to_camel_dict +from ansible.module_utils.six import binary_type +from ansible.module_utils.six import string_types +from ansible.module_utils.six import text_type + +from .cloud import CloudRetry BOTO_IMP_ERR = None try: diff --git a/plugins/module_utils/elb_utils.py b/plugins/module_utils/elb_utils.py new file mode 100644 index 00000000000..5ab8b349057 --- /dev/null +++ b/plugins/module_utils/elb_utils.py @@ -0,0 +1,111 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +try: + from botocore.exceptions import BotoCoreError, ClientError +except ImportError: + pass + +from .ec2 import AWSRetry + + +def get_elb(connection, module, elb_name): + """ + Get an ELB based on name. If not found, return None. + + :param connection: AWS boto3 elbv2 connection + :param module: Ansible module + :param elb_name: Name of load balancer to get + :return: boto3 ELB dict or None if not found + """ + try: + return _get_elb(connection, module, elb_name) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e) + + +@AWSRetry.jittered_backoff() +def _get_elb(connection, module, elb_name): + """ + Get an ELB based on name using AWSRetry. If not found, return None. + + :param connection: AWS boto3 elbv2 connection + :param module: Ansible module + :param elb_name: Name of load balancer to get + :return: boto3 ELB dict or None if not found + """ + + try: + load_balancer_paginator = connection.get_paginator('describe_load_balancers') + return (load_balancer_paginator.paginate(Names=[elb_name]).build_full_result())['LoadBalancers'][0] + except (BotoCoreError, ClientError) as e: + if e.response['Error']['Code'] == 'LoadBalancerNotFound': + return None + else: + raise e + + +def get_elb_listener(connection, module, elb_arn, listener_port): + """ + Get an ELB listener based on the port provided. If not found, return None. + + :param connection: AWS boto3 elbv2 connection + :param module: Ansible module + :param elb_arn: ARN of the ELB to look at + :param listener_port: Port of the listener to look for + :return: boto3 ELB listener dict or None if not found + """ + + try: + listener_paginator = connection.get_paginator('describe_listeners') + listeners = (AWSRetry.jittered_backoff()(listener_paginator.paginate)(LoadBalancerArn=elb_arn).build_full_result())['Listeners'] + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e) + + l = None + + for listener in listeners: + if listener['Port'] == listener_port: + l = listener + break + + return l + + +def get_elb_listener_rules(connection, module, listener_arn): + """ + Get rules for a particular ELB listener using the listener ARN. + + :param connection: AWS boto3 elbv2 connection + :param module: Ansible module + :param listener_arn: ARN of the ELB listener + :return: boto3 ELB rules list + """ + + try: + return AWSRetry.jittered_backoff()(connection.describe_rules)(ListenerArn=listener_arn)['Rules'] + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e) + + +def convert_tg_name_to_arn(connection, module, tg_name): + """ + Get ARN of a target group using the target group's name + + :param connection: AWS boto3 elbv2 connection + :param module: Ansible module + :param tg_name: Name of the target group + :return: target group ARN string + """ + + try: + response = AWSRetry.jittered_backoff()(connection.describe_target_groups)(Names=[tg_name]) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e) + + tg_arn = response['TargetGroups'][0]['TargetGroupArn'] + + return tg_arn diff --git a/plugins/module_utils/elbv2.py b/plugins/module_utils/elbv2.py new file mode 100644 index 00000000000..8ae12e01172 --- /dev/null +++ b/plugins/module_utils/elbv2.py @@ -0,0 +1,895 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import traceback +from copy import deepcopy + +try: + from botocore.exceptions import BotoCoreError, ClientError +except ImportError: + pass + +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from .ec2 import AWSRetry +from .ec2 import ansible_dict_to_boto3_tag_list +from .ec2 import boto3_tag_list_to_ansible_dict +from .ec2 import get_ec2_security_group_ids_from_names +from .elb_utils import convert_tg_name_to_arn +from .elb_utils import get_elb +from .elb_utils import get_elb_listener + + +class ElasticLoadBalancerV2(object): + + def __init__(self, connection, module): + + self.connection = connection + self.module = module + self.changed = False + self.new_load_balancer = False + self.scheme = module.params.get("scheme") + self.name = module.params.get("name") + self.subnet_mappings = module.params.get("subnet_mappings") + self.subnets = module.params.get("subnets") + self.deletion_protection = module.params.get("deletion_protection") + self.wait = module.params.get("wait") + + if module.params.get("tags") is not None: + self.tags = ansible_dict_to_boto3_tag_list(module.params.get("tags")) + else: + self.tags = None + self.purge_tags = module.params.get("purge_tags") + + self.elb = get_elb(connection, module, self.name) + if self.elb is not None: + self.elb_attributes = self.get_elb_attributes() + self.elb['tags'] = self.get_elb_tags() + else: + self.elb_attributes = None + + def wait_for_status(self, elb_arn): + """ + Wait for load balancer to reach 'active' status + + :param elb_arn: The load balancer ARN + :return: + """ + + try: + waiter = self.connection.get_waiter('load_balancer_available') + waiter.wait(LoadBalancerArns=[elb_arn]) + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + def get_elb_attributes(self): + """ + Get load balancer attributes + + :return: + """ + + try: + attr_list = AWSRetry.jittered_backoff()( + self.connection.describe_load_balancer_attributes + )(LoadBalancerArn=self.elb['LoadBalancerArn'])['Attributes'] + + elb_attributes = boto3_tag_list_to_ansible_dict(attr_list) + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + # Replace '.' with '_' in attribute key names to make it more Ansibley + return dict((k.replace('.', '_'), v) for k, v in elb_attributes.items()) + + def update_elb_attributes(self): + """ + Update the elb_attributes parameter + :return: + """ + self.elb_attributes = self.get_elb_attributes() + + def get_elb_tags(self): + """ + Get load balancer tags + + :return: + """ + + try: + return AWSRetry.jittered_backoff()( + self.connection.describe_tags + )(ResourceArns=[self.elb['LoadBalancerArn']])['TagDescriptions'][0]['Tags'] + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + def delete_tags(self, tags_to_delete): + """ + Delete elb tags + + :return: + """ + + try: + AWSRetry.jittered_backoff()( + self.connection.remove_tags + )(ResourceArns=[self.elb['LoadBalancerArn']], TagKeys=tags_to_delete) + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + self.changed = True + + def modify_tags(self): + """ + Modify elb tags + + :return: + """ + + try: + AWSRetry.jittered_backoff()( + self.connection.add_tags + )(ResourceArns=[self.elb['LoadBalancerArn']], Tags=self.tags) + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + self.changed = True + + def delete(self): + """ + Delete elb + :return: + """ + + try: + AWSRetry.jittered_backoff()( + self.connection.delete_load_balancer + )(LoadBalancerArn=self.elb['LoadBalancerArn']) + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + self.changed = True + + def compare_subnets(self): + """ + Compare user subnets with current ELB subnets + + :return: bool True if they match otherwise False + """ + + subnet_mapping_id_list = [] + subnet_mappings = [] + + # Check if we're dealing with subnets or subnet_mappings + if self.subnets is not None: + # Convert subnets to subnet_mappings format for comparison + for subnet in self.subnets: + subnet_mappings.append({'SubnetId': subnet}) + + if self.subnet_mappings is not None: + # Use this directly since we're comparing as a mapping + subnet_mappings = self.subnet_mappings + + # Build a subnet_mapping style struture of what's currently + # on the load balancer + for subnet in self.elb['AvailabilityZones']: + this_mapping = {'SubnetId': subnet['SubnetId']} + for address in subnet.get('LoadBalancerAddresses', []): + if 'AllocationId' in address: + this_mapping['AllocationId'] = address['AllocationId'] + break + + subnet_mapping_id_list.append(this_mapping) + + return set(frozenset(mapping.items()) for mapping in subnet_mapping_id_list) == set(frozenset(mapping.items()) for mapping in subnet_mappings) + + def modify_subnets(self): + """ + Modify elb subnets to match module parameters + :return: + """ + + try: + AWSRetry.jittered_backoff()( + self.connection.set_subnets + )(LoadBalancerArn=self.elb['LoadBalancerArn'], Subnets=self.subnets) + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + self.changed = True + + def update(self): + """ + Update the elb from AWS + :return: + """ + + self.elb = get_elb(self.connection, self.module, self.module.params.get("name")) + self.elb['tags'] = self.get_elb_tags() + + +class ApplicationLoadBalancer(ElasticLoadBalancerV2): + + def __init__(self, connection, connection_ec2, module): + """ + + :param connection: boto3 connection + :param module: Ansible module + """ + super(ApplicationLoadBalancer, self).__init__(connection, module) + + self.connection_ec2 = connection_ec2 + + # Ansible module parameters specific to ALBs + self.type = 'application' + if module.params.get('security_groups') is not None: + try: + self.security_groups = AWSRetry.jittered_backoff()( + get_ec2_security_group_ids_from_names + )(module.params.get('security_groups'), self.connection_ec2, boto3=True) + except ValueError as e: + self.module.fail_json(msg=str(e), exception=traceback.format_exc()) + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + else: + self.security_groups = module.params.get('security_groups') + self.access_logs_enabled = module.params.get("access_logs_enabled") + self.access_logs_s3_bucket = module.params.get("access_logs_s3_bucket") + self.access_logs_s3_prefix = module.params.get("access_logs_s3_prefix") + self.idle_timeout = module.params.get("idle_timeout") + self.http2 = module.params.get("http2") + + if self.elb is not None and self.elb['Type'] != 'application': + self.module.fail_json(msg="The load balancer type you are trying to manage is not application. Try elb_network_lb module instead.") + + def create_elb(self): + """ + Create a load balancer + :return: + """ + + # Required parameters + params = dict() + params['Name'] = self.name + params['Type'] = self.type + + # Other parameters + if self.subnets is not None: + params['Subnets'] = self.subnets + if self.subnet_mappings is not None: + params['SubnetMappings'] = self.subnet_mappings + if self.security_groups is not None: + params['SecurityGroups'] = self.security_groups + params['Scheme'] = self.scheme + if self.tags: + params['Tags'] = self.tags + + try: + self.elb = AWSRetry.jittered_backoff()(self.connection.create_load_balancer)(**params)['LoadBalancers'][0] + self.changed = True + self.new_load_balancer = True + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + if self.wait: + self.wait_for_status(self.elb['LoadBalancerArn']) + + def modify_elb_attributes(self): + """ + Update Application ELB attributes if required + + :return: + """ + + update_attributes = [] + + if self.access_logs_enabled is not None and str(self.access_logs_enabled).lower() != self.elb_attributes['access_logs_s3_enabled']: + update_attributes.append({'Key': 'access_logs.s3.enabled', 'Value': str(self.access_logs_enabled).lower()}) + if self.access_logs_s3_bucket is not None and self.access_logs_s3_bucket != self.elb_attributes['access_logs_s3_bucket']: + update_attributes.append({'Key': 'access_logs.s3.bucket', 'Value': self.access_logs_s3_bucket}) + if self.access_logs_s3_prefix is not None and self.access_logs_s3_prefix != self.elb_attributes['access_logs_s3_prefix']: + update_attributes.append({'Key': 'access_logs.s3.prefix', 'Value': self.access_logs_s3_prefix}) + if self.deletion_protection is not None and str(self.deletion_protection).lower() != self.elb_attributes['deletion_protection_enabled']: + update_attributes.append({'Key': 'deletion_protection.enabled', 'Value': str(self.deletion_protection).lower()}) + if self.idle_timeout is not None and str(self.idle_timeout) != self.elb_attributes['idle_timeout_timeout_seconds']: + update_attributes.append({'Key': 'idle_timeout.timeout_seconds', 'Value': str(self.idle_timeout)}) + if self.http2 is not None and str(self.http2).lower() != self.elb_attributes['routing_http2_enabled']: + update_attributes.append({'Key': 'routing.http2.enabled', 'Value': str(self.http2).lower()}) + + if update_attributes: + try: + AWSRetry.jittered_backoff()( + self.connection.modify_load_balancer_attributes + )(LoadBalancerArn=self.elb['LoadBalancerArn'], Attributes=update_attributes) + self.changed = True + except (BotoCoreError, ClientError) as e: + # Something went wrong setting attributes. If this ELB was created during this task, delete it to leave a consistent state + if self.new_load_balancer: + AWSRetry.jittered_backoff()(self.connection.delete_load_balancer)(LoadBalancerArn=self.elb['LoadBalancerArn']) + self.module.fail_json_aws(e) + + def compare_security_groups(self): + """ + Compare user security groups with current ELB security groups + + :return: bool True if they match otherwise False + """ + + if set(self.elb['SecurityGroups']) != set(self.security_groups): + return False + else: + return True + + def modify_security_groups(self): + """ + Modify elb security groups to match module parameters + :return: + """ + + try: + AWSRetry.jittered_backoff()( + self.connection.set_security_groups + )(LoadBalancerArn=self.elb['LoadBalancerArn'], SecurityGroups=self.security_groups) + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + self.changed = True + + +class NetworkLoadBalancer(ElasticLoadBalancerV2): + + def __init__(self, connection, connection_ec2, module): + + """ + + :param connection: boto3 connection + :param module: Ansible module + """ + super(NetworkLoadBalancer, self).__init__(connection, module) + + self.connection_ec2 = connection_ec2 + + # Ansible module parameters specific to NLBs + self.type = 'network' + self.cross_zone_load_balancing = module.params.get('cross_zone_load_balancing') + + if self.elb is not None and self.elb['Type'] != 'network': + self.module.fail_json(msg="The load balancer type you are trying to manage is not network. Try elb_application_lb module instead.") + + def create_elb(self): + """ + Create a load balancer + :return: + """ + + # Required parameters + params = dict() + params['Name'] = self.name + params['Type'] = self.type + + # Other parameters + if self.subnets is not None: + params['Subnets'] = self.subnets + if self.subnet_mappings is not None: + params['SubnetMappings'] = self.subnet_mappings + params['Scheme'] = self.scheme + if self.tags: + params['Tags'] = self.tags + + try: + self.elb = AWSRetry.jittered_backoff()(self.connection.create_load_balancer)(**params)['LoadBalancers'][0] + self.changed = True + self.new_load_balancer = True + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + if self.wait: + self.wait_for_status(self.elb['LoadBalancerArn']) + + def modify_elb_attributes(self): + """ + Update Network ELB attributes if required + + :return: + """ + + update_attributes = [] + + if self.cross_zone_load_balancing is not None and str(self.cross_zone_load_balancing).lower() != \ + self.elb_attributes['load_balancing_cross_zone_enabled']: + update_attributes.append({'Key': 'load_balancing.cross_zone.enabled', 'Value': str(self.cross_zone_load_balancing).lower()}) + if self.deletion_protection is not None and str(self.deletion_protection).lower() != self.elb_attributes['deletion_protection_enabled']: + update_attributes.append({'Key': 'deletion_protection.enabled', 'Value': str(self.deletion_protection).lower()}) + + if update_attributes: + try: + AWSRetry.jittered_backoff()( + self.connection.modify_load_balancer_attributes + )(LoadBalancerArn=self.elb['LoadBalancerArn'], Attributes=update_attributes) + self.changed = True + except (BotoCoreError, ClientError) as e: + # Something went wrong setting attributes. If this ELB was created during this task, delete it to leave a consistent state + if self.new_load_balancer: + AWSRetry.jittered_backoff()(self.connection.delete_load_balancer)(LoadBalancerArn=self.elb['LoadBalancerArn']) + self.module.fail_json_aws(e) + + def modify_subnets(self): + """ + Modify elb subnets to match module parameters (unsupported for NLB) + :return: + """ + + self.module.fail_json(msg='Modifying subnets and elastic IPs is not supported for Network Load Balancer') + + +class ELBListeners(object): + + def __init__(self, connection, module, elb_arn): + + self.connection = connection + self.module = module + self.elb_arn = elb_arn + listeners = module.params.get("listeners") + if listeners is not None: + # Remove suboption argspec defaults of None from each listener + listeners = [dict((x, listener_dict[x]) for x in listener_dict if listener_dict[x] is not None) for listener_dict in listeners] + self.listeners = self._ensure_listeners_default_action_has_arn(listeners) + self.current_listeners = self._get_elb_listeners() + self.purge_listeners = module.params.get("purge_listeners") + self.changed = False + + def update(self): + """ + Update the listeners for the ELB + + :return: + """ + self.current_listeners = self._get_elb_listeners() + + def _get_elb_listeners(self): + """ + Get ELB listeners + + :return: + """ + + try: + listener_paginator = self.connection.get_paginator('describe_listeners') + return (AWSRetry.jittered_backoff()(listener_paginator.paginate)(LoadBalancerArn=self.elb_arn).build_full_result())['Listeners'] + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + def _ensure_listeners_default_action_has_arn(self, listeners): + """ + If a listener DefaultAction has been passed with a Target Group Name instead of ARN, lookup the ARN and + replace the name. + + :param listeners: a list of listener dicts + :return: the same list of dicts ensuring that each listener DefaultActions dict has TargetGroupArn key. If a TargetGroupName key exists, it is removed. + """ + + if not listeners: + listeners = [] + + fixed_listeners = [] + for listener in listeners: + fixed_actions = [] + for action in listener['DefaultActions']: + if 'TargetGroupName' in action: + action['TargetGroupArn'] = convert_tg_name_to_arn(self.connection, + self.module, + action['TargetGroupName']) + del action['TargetGroupName'] + fixed_actions.append(action) + listener['DefaultActions'] = fixed_actions + fixed_listeners.append(listener) + + return fixed_listeners + + def compare_listeners(self): + """ + + :return: + """ + listeners_to_modify = [] + listeners_to_delete = [] + listeners_to_add = deepcopy(self.listeners) + + # Check each current listener port to see if it's been passed to the module + for current_listener in self.current_listeners: + current_listener_passed_to_module = False + for new_listener in self.listeners[:]: + new_listener['Port'] = int(new_listener['Port']) + if current_listener['Port'] == new_listener['Port']: + current_listener_passed_to_module = True + # Remove what we match so that what is left can be marked as 'to be added' + listeners_to_add.remove(new_listener) + modified_listener = self._compare_listener(current_listener, new_listener) + if modified_listener: + modified_listener['Port'] = current_listener['Port'] + modified_listener['ListenerArn'] = current_listener['ListenerArn'] + listeners_to_modify.append(modified_listener) + break + + # If the current listener was not matched against passed listeners and purge is True, mark for removal + if not current_listener_passed_to_module and self.purge_listeners: + listeners_to_delete.append(current_listener['ListenerArn']) + + return listeners_to_add, listeners_to_modify, listeners_to_delete + + def _compare_listener(self, current_listener, new_listener): + """ + Compare two listeners. + + :param current_listener: + :param new_listener: + :return: + """ + + modified_listener = {} + + # Port + if current_listener['Port'] != new_listener['Port']: + modified_listener['Port'] = new_listener['Port'] + + # Protocol + if current_listener['Protocol'] != new_listener['Protocol']: + modified_listener['Protocol'] = new_listener['Protocol'] + + # If Protocol is HTTPS, check additional attributes + if current_listener['Protocol'] == 'HTTPS' and new_listener['Protocol'] == 'HTTPS': + # Cert + if current_listener['SslPolicy'] != new_listener['SslPolicy']: + modified_listener['SslPolicy'] = new_listener['SslPolicy'] + if current_listener['Certificates'][0]['CertificateArn'] != new_listener['Certificates'][0]['CertificateArn']: + modified_listener['Certificates'] = [] + modified_listener['Certificates'].append({}) + modified_listener['Certificates'][0]['CertificateArn'] = new_listener['Certificates'][0]['CertificateArn'] + elif current_listener['Protocol'] != 'HTTPS' and new_listener['Protocol'] == 'HTTPS': + modified_listener['SslPolicy'] = new_listener['SslPolicy'] + modified_listener['Certificates'] = [] + modified_listener['Certificates'].append({}) + modified_listener['Certificates'][0]['CertificateArn'] = new_listener['Certificates'][0]['CertificateArn'] + + # Default action + + # Check proper rule format on current listener + if len(current_listener['DefaultActions']) > 1: + for action in current_listener['DefaultActions']: + if 'Order' not in action: + self.module.fail_json(msg="'Order' key not found in actions. " + "installed version of botocore does not support " + "multiple actions, please upgrade botocore to version " + "1.10.30 or higher") + + # If the lengths of the actions are the same, we'll have to verify that the + # contents of those actions are the same + if len(current_listener['DefaultActions']) == len(new_listener['DefaultActions']): + # if actions have just one element, compare the contents and then update if + # they're different + if len(current_listener['DefaultActions']) == 1 and len(new_listener['DefaultActions']) == 1: + if current_listener['DefaultActions'] != new_listener['DefaultActions']: + modified_listener['DefaultActions'] = new_listener['DefaultActions'] + # if actions have multiple elements, we'll have to order them first before comparing. + # multiple actions will have an 'Order' key for this purpose + else: + current_actions_sorted = sorted(current_listener['DefaultActions'], key=lambda x: x['Order']) + new_actions_sorted = sorted(new_listener['DefaultActions'], key=lambda x: x['Order']) + + # the AWS api won't return the client secret, so we'll have to remove it + # or the module will always see the new and current actions as different + # and try to apply the same config + new_actions_sorted_no_secret = [] + for action in new_actions_sorted: + # the secret is currently only defined in the oidc config + if action['Type'] == 'authenticate-oidc': + action['AuthenticateOidcConfig'].pop('ClientSecret') + new_actions_sorted_no_secret.append(action) + else: + new_actions_sorted_no_secret.append(action) + + if current_actions_sorted != new_actions_sorted_no_secret: + modified_listener['DefaultActions'] = new_listener['DefaultActions'] + # If the action lengths are different, then replace with the new actions + else: + modified_listener['DefaultActions'] = new_listener['DefaultActions'] + + if modified_listener: + return modified_listener + else: + return None + + +class ELBListener(object): + + def __init__(self, connection, module, listener, elb_arn): + """ + + :param connection: + :param module: + :param listener: + :param elb_arn: + """ + + self.connection = connection + self.module = module + self.listener = listener + self.elb_arn = elb_arn + + def add(self): + + try: + # Rules is not a valid parameter for create_listener + if 'Rules' in self.listener: + self.listener.pop('Rules') + AWSRetry.jittered_backoff()(self.connection.create_listener)(LoadBalancerArn=self.elb_arn, **self.listener) + except (BotoCoreError, ClientError) as e: + if '"Order", must be one of: Type, TargetGroupArn' in str(e): + self.module.fail_json(msg="installed version of botocore does not support " + "multiple actions, please upgrade botocore to version " + "1.10.30 or higher") + else: + self.module.fail_json_aws(e) + + def modify(self): + + try: + # Rules is not a valid parameter for modify_listener + if 'Rules' in self.listener: + self.listener.pop('Rules') + AWSRetry.jittered_backoff()(self.connection.modify_listener)(**self.listener) + except (BotoCoreError, ClientError) as e: + if '"Order", must be one of: Type, TargetGroupArn' in str(e): + self.module.fail_json(msg="installed version of botocore does not support " + "multiple actions, please upgrade botocore to version " + "1.10.30 or higher") + else: + self.module.fail_json_aws(e) + + def delete(self): + + try: + AWSRetry.jittered_backoff()(self.connection.delete_listener)(ListenerArn=self.listener) + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + +class ELBListenerRules(object): + + def __init__(self, connection, module, elb_arn, listener_rules, listener_port): + + self.connection = connection + self.module = module + self.elb_arn = elb_arn + self.rules = self._ensure_rules_action_has_arn(listener_rules) + self.changed = False + + # Get listener based on port so we can use ARN + self.current_listener = get_elb_listener(connection, module, elb_arn, listener_port) + self.listener_arn = self.current_listener['ListenerArn'] + self.rules_to_add = deepcopy(self.rules) + self.rules_to_modify = [] + self.rules_to_delete = [] + + # If the listener exists (i.e. has an ARN) get rules for the listener + if 'ListenerArn' in self.current_listener: + self.current_rules = self._get_elb_listener_rules() + else: + self.current_rules = [] + + def _ensure_rules_action_has_arn(self, rules): + """ + If a rule Action has been passed with a Target Group Name instead of ARN, lookup the ARN and + replace the name. + + :param rules: a list of rule dicts + :return: the same list of dicts ensuring that each rule Actions dict has TargetGroupArn key. If a TargetGroupName key exists, it is removed. + """ + + fixed_rules = [] + for rule in rules: + fixed_actions = [] + for action in rule['Actions']: + if 'TargetGroupName' in action: + action['TargetGroupArn'] = convert_tg_name_to_arn(self.connection, self.module, action['TargetGroupName']) + del action['TargetGroupName'] + fixed_actions.append(action) + rule['Actions'] = fixed_actions + fixed_rules.append(rule) + + return fixed_rules + + def _get_elb_listener_rules(self): + + try: + return AWSRetry.jittered_backoff()(self.connection.describe_rules)(ListenerArn=self.current_listener['ListenerArn'])['Rules'] + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + def _compare_condition(self, current_conditions, condition): + """ + + :param current_conditions: + :param condition: + :return: + """ + + condition_found = False + + for current_condition in current_conditions: + if current_condition.get('SourceIpConfig'): + if (current_condition['Field'] == condition['Field'] and + current_condition['SourceIpConfig']['Values'][0] == condition['SourceIpConfig']['Values'][0]): + condition_found = True + break + elif current_condition['Field'] == condition['Field'] and sorted(current_condition['Values']) == sorted(condition['Values']): + condition_found = True + break + + return condition_found + + def _compare_rule(self, current_rule, new_rule): + """ + + :return: + """ + + modified_rule = {} + + # Priority + if int(current_rule['Priority']) != int(new_rule['Priority']): + modified_rule['Priority'] = new_rule['Priority'] + + # Actions + + # Check proper rule format on current listener + if len(current_rule['Actions']) > 1: + for action in current_rule['Actions']: + if 'Order' not in action: + self.module.fail_json(msg="'Order' key not found in actions. " + "installed version of botocore does not support " + "multiple actions, please upgrade botocore to version " + "1.10.30 or higher") + + # If the lengths of the actions are the same, we'll have to verify that the + # contents of those actions are the same + if len(current_rule['Actions']) == len(new_rule['Actions']): + # if actions have just one element, compare the contents and then update if + # they're different + if len(current_rule['Actions']) == 1 and len(new_rule['Actions']) == 1: + if current_rule['Actions'] != new_rule['Actions']: + modified_rule['Actions'] = new_rule['Actions'] + # if actions have multiple elements, we'll have to order them first before comparing. + # multiple actions will have an 'Order' key for this purpose + else: + current_actions_sorted = sorted(current_rule['Actions'], key=lambda x: x['Order']) + new_actions_sorted = sorted(new_rule['Actions'], key=lambda x: x['Order']) + + # the AWS api won't return the client secret, so we'll have to remove it + # or the module will always see the new and current actions as different + # and try to apply the same config + new_actions_sorted_no_secret = [] + for action in new_actions_sorted: + # the secret is currently only defined in the oidc config + if action['Type'] == 'authenticate-oidc': + action['AuthenticateOidcConfig'].pop('ClientSecret') + new_actions_sorted_no_secret.append(action) + else: + new_actions_sorted_no_secret.append(action) + + if current_actions_sorted != new_actions_sorted_no_secret: + modified_rule['Actions'] = new_rule['Actions'] + # If the action lengths are different, then replace with the new actions + else: + modified_rule['Actions'] = new_rule['Actions'] + + # Conditions + modified_conditions = [] + for condition in new_rule['Conditions']: + if not self._compare_condition(current_rule['Conditions'], condition): + modified_conditions.append(condition) + + if modified_conditions: + modified_rule['Conditions'] = modified_conditions + + return modified_rule + + def compare_rules(self): + """ + + :return: + """ + + rules_to_modify = [] + rules_to_delete = [] + rules_to_add = deepcopy(self.rules) + + for current_rule in self.current_rules: + current_rule_passed_to_module = False + for new_rule in self.rules[:]: + if current_rule['Priority'] == str(new_rule['Priority']): + current_rule_passed_to_module = True + # Remove what we match so that what is left can be marked as 'to be added' + rules_to_add.remove(new_rule) + modified_rule = self._compare_rule(current_rule, new_rule) + if modified_rule: + modified_rule['Priority'] = int(current_rule['Priority']) + modified_rule['RuleArn'] = current_rule['RuleArn'] + modified_rule['Actions'] = new_rule['Actions'] + modified_rule['Conditions'] = new_rule['Conditions'] + rules_to_modify.append(modified_rule) + break + + # If the current rule was not matched against passed rules, mark for removal + if not current_rule_passed_to_module and not current_rule['IsDefault']: + rules_to_delete.append(current_rule['RuleArn']) + + return rules_to_add, rules_to_modify, rules_to_delete + + +class ELBListenerRule(object): + + def __init__(self, connection, module, rule, listener_arn): + + self.connection = connection + self.module = module + self.rule = rule + self.listener_arn = listener_arn + self.changed = False + + def create(self): + """ + Create a listener rule + + :return: + """ + + try: + self.rule['ListenerArn'] = self.listener_arn + self.rule['Priority'] = int(self.rule['Priority']) + AWSRetry.jittered_backoff()(self.connection.create_rule)(**self.rule) + except (BotoCoreError, ClientError) as e: + if '"Order", must be one of: Type, TargetGroupArn' in str(e): + self.module.fail_json(msg="installed version of botocore does not support " + "multiple actions, please upgrade botocore to version " + "1.10.30 or higher") + else: + self.module.fail_json_aws(e) + + self.changed = True + + def modify(self): + """ + Modify a listener rule + + :return: + """ + + try: + del self.rule['Priority'] + AWSRetry.jittered_backoff()(self.connection.modify_rule)(**self.rule) + except (BotoCoreError, ClientError) as e: + if '"Order", must be one of: Type, TargetGroupArn' in str(e): + self.module.fail_json(msg="installed version of botocore does not support " + "multiple actions, please upgrade botocore to version " + "1.10.30 or higher") + else: + self.module.fail_json_aws(e) + + self.changed = True + + def delete(self): + """ + Delete a listener rule + + :return: + """ + + try: + AWSRetry.jittered_backoff()(self.connection.delete_rule)(RuleArn=self.rule['RuleArn']) + except (BotoCoreError, ClientError) as e: + self.module.fail_json_aws(e) + + self.changed = True diff --git a/plugins/module_utils/iam.py b/plugins/module_utils/iam.py new file mode 100644 index 00000000000..4c39422e413 --- /dev/null +++ b/plugins/module_utils/iam.py @@ -0,0 +1,49 @@ +# Copyright (c) 2017 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import traceback + +try: + from botocore.exceptions import ClientError, NoCredentialsError +except ImportError: + pass + +from ansible.module_utils._text import to_native + + +def get_aws_account_id(module): + """ Given AnsibleAWSModule instance, get the active AWS account ID + + get_account_id tries too find out the account that we are working + on. It's not guaranteed that this will be easy so we try in + several different ways. Giving either IAM or STS privilages to + the account should be enough to permit this. + """ + account_id = None + try: + sts_client = module.client('sts') + account_id = sts_client.get_caller_identity().get('Account') + # non-STS sessions may also get NoCredentialsError from this STS call, so + # we must catch that too and try the IAM version + except (ClientError, NoCredentialsError): + try: + iam_client = module.client('iam') + account_id = iam_client.get_user()['User']['Arn'].split(':')[4] + except ClientError as e: + if (e.response['Error']['Code'] == 'AccessDenied'): + except_msg = to_native(e) + # don't match on `arn:aws` because of China region `arn:aws-cn` and similar + account_id = except_msg.search(r"arn:\w+:iam::([0-9]{12,32}):\w+/").group(1) + if account_id is None: + module.fail_json_aws(e, msg="Could not get AWS account information") + except Exception as e: + module.fail_json( + msg="Failed to get AWS account information, Try allowing sts:GetCallerIdentity or iam:GetUser permissions.", + exception=traceback.format_exc() + ) + if not account_id: + module.fail_json(msg="Failed while determining AWS account ID. Try allowing sts:GetCallerIdentity or iam:GetUser permissions.") + return to_native(account_id) diff --git a/plugins/module_utils/rds.py b/plugins/module_utils/rds.py new file mode 100644 index 00000000000..49b782e0181 --- /dev/null +++ b/plugins/module_utils/rds.py @@ -0,0 +1,235 @@ +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +from collections import namedtuple +from time import sleep + +try: + from botocore.exceptions import BotoCoreError, ClientError, WaiterError +except ImportError: + pass + +from ansible.module_utils._text import to_text +from ansible.module_utils.common.dict_transformations import snake_dict_to_camel_dict + +from .ec2 import AWSRetry +from .ec2 import ansible_dict_to_boto3_tag_list +from .ec2 import boto3_tag_list_to_ansible_dict +from .ec2 import compare_aws_tags +from .waiters import get_waiter + +Boto3ClientMethod = namedtuple('Boto3ClientMethod', ['name', 'waiter', 'operation_description', 'cluster', 'instance']) +# Whitelist boto3 client methods for cluster and instance resources +cluster_method_names = [ + 'create_db_cluster', 'restore_db_cluster_from_db_snapshot', 'restore_db_cluster_from_s3', + 'restore_db_cluster_to_point_in_time', 'modify_db_cluster', 'delete_db_cluster', 'add_tags_to_resource', + 'remove_tags_from_resource', 'list_tags_for_resource', 'promote_read_replica_db_cluster' +] +instance_method_names = [ + 'create_db_instance', 'restore_db_instance_to_point_in_time', 'restore_db_instance_from_s3', + 'restore_db_instance_from_db_snapshot', 'create_db_instance_read_replica', 'modify_db_instance', + 'delete_db_instance', 'add_tags_to_resource', 'remove_tags_from_resource', 'list_tags_for_resource', + 'promote_read_replica', 'stop_db_instance', 'start_db_instance', 'reboot_db_instance' +] + + +def get_rds_method_attribute(method_name, module): + readable_op = method_name.replace('_', ' ').replace('db', 'DB') + if method_name in cluster_method_names and 'new_db_cluster_identifier' in module.params: + cluster = True + instance = False + if method_name == 'delete_db_cluster': + waiter = 'cluster_deleted' + else: + waiter = 'cluster_available' + elif method_name in instance_method_names and 'new_db_instance_identifier' in module.params: + cluster = False + instance = True + if method_name == 'delete_db_instance': + waiter = 'db_instance_deleted' + elif method_name == 'stop_db_instance': + waiter = 'db_instance_stopped' + else: + waiter = 'db_instance_available' + else: + raise NotImplementedError("method {0} hasn't been added to the list of accepted methods to use a waiter in module_utils/aws/rds.py".format(method_name)) + + return Boto3ClientMethod(name=method_name, waiter=waiter, operation_description=readable_op, cluster=cluster, instance=instance) + + +def get_final_identifier(method_name, module): + apply_immediately = module.params['apply_immediately'] + if get_rds_method_attribute(method_name, module).cluster: + identifier = module.params['db_cluster_identifier'] + updated_identifier = module.params['new_db_cluster_identifier'] + elif get_rds_method_attribute(method_name, module).instance: + identifier = module.params['db_instance_identifier'] + updated_identifier = module.params['new_db_instance_identifier'] + else: + raise NotImplementedError("method {0} hasn't been added to the list of accepted methods in module_utils/aws/rds.py".format(method_name)) + if not module.check_mode and updated_identifier and apply_immediately: + identifier = updated_identifier + return identifier + + +def handle_errors(module, exception, method_name, parameters): + + if not isinstance(exception, ClientError): + module.fail_json_aws(exception, msg="Unexpected failure for method {0} with parameters {1}".format(method_name, parameters)) + + changed = True + error_code = exception.response['Error']['Code'] + if method_name == 'modify_db_instance' and error_code == 'InvalidParameterCombination': + if 'No modifications were requested' in to_text(exception): + changed = False + elif 'ModifyDbCluster API' in to_text(exception): + module.fail_json_aws(exception, msg='It appears you are trying to modify attributes that are managed at the cluster level. Please see rds_cluster') + else: + module.fail_json_aws(exception, msg='Unable to {0}'.format(get_rds_method_attribute(method_name, module).operation_description)) + elif method_name == 'promote_read_replica' and error_code == 'InvalidDBInstanceState': + if 'DB Instance is not a read replica' in to_text(exception): + changed = False + else: + module.fail_json_aws(exception, msg='Unable to {0}'.format(get_rds_method_attribute(method_name, module).operation_description)) + elif method_name == 'create_db_instance' and exception.response['Error']['Code'] == 'InvalidParameterValue': + accepted_engines = [ + 'aurora', 'aurora-mysql', 'aurora-postgresql', 'mariadb', 'mysql', 'oracle-ee', 'oracle-se', + 'oracle-se1', 'oracle-se2', 'postgres', 'sqlserver-ee', 'sqlserver-ex', 'sqlserver-se', 'sqlserver-web' + ] + if parameters.get('Engine') not in accepted_engines: + module.fail_json_aws(exception, msg='DB engine {0} should be one of {1}'.format(parameters.get('Engine'), accepted_engines)) + else: + module.fail_json_aws(exception, msg='Unable to {0}'.format(get_rds_method_attribute(method_name, module).operation_description)) + else: + module.fail_json_aws(exception, msg='Unable to {0}'.format(get_rds_method_attribute(method_name, module).operation_description)) + + return changed + + +def call_method(client, module, method_name, parameters): + result = {} + changed = True + if not module.check_mode: + wait = module.params['wait'] + # TODO: stabilize by adding get_rds_method_attribute(method_name).extra_retry_codes + method = getattr(client, method_name) + try: + if method_name == 'modify_db_instance': + # check if instance is in an available state first, if possible + if wait: + wait_for_status(client, module, module.params['db_instance_identifier'], method_name) + result = AWSRetry.jittered_backoff(catch_extra_error_codes=['InvalidDBInstanceState'])(method)(**parameters) + else: + result = AWSRetry.jittered_backoff()(method)(**parameters) + except (BotoCoreError, ClientError) as e: + changed = handle_errors(module, e, method_name, parameters) + + if wait and changed: + identifier = get_final_identifier(method_name, module) + wait_for_status(client, module, identifier, method_name) + return result, changed + + +def wait_for_instance_status(client, module, db_instance_id, waiter_name): + def wait(client, db_instance_id, waiter_name, extra_retry_codes): + retry = AWSRetry.jittered_backoff(catch_extra_error_codes=extra_retry_codes) + try: + waiter = client.get_waiter(waiter_name) + except ValueError: + # using a waiter in ansible.module_utils.aws.waiters + waiter = get_waiter(client, waiter_name) + waiter.wait(WaiterConfig={'Delay': 60, 'MaxAttempts': 60}, DBInstanceIdentifier=db_instance_id) + + waiter_expected_status = { + 'db_instance_deleted': 'deleted', + 'db_instance_stopped': 'stopped', + } + expected_status = waiter_expected_status.get(waiter_name, 'available') + if expected_status == 'available': + extra_retry_codes = ['DBInstanceNotFound'] + else: + extra_retry_codes = [] + for attempt_to_wait in range(0, 10): + try: + wait(client, db_instance_id, waiter_name, extra_retry_codes) + break + except WaiterError as e: + # Instance may be renamed and AWSRetry doesn't handle WaiterError + if e.last_response.get('Error', {}).get('Code') == 'DBInstanceNotFound': + sleep(10) + continue + module.fail_json_aws(e, msg='Error while waiting for DB instance {0} to be {1}'.format(db_instance_id, expected_status)) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg='Unexpected error while waiting for DB instance {0} to be {1}'.format( + db_instance_id, expected_status) + ) + + +def wait_for_cluster_status(client, module, db_cluster_id, waiter_name): + try: + waiter = get_waiter(client, waiter_name).wait(DBClusterIdentifier=db_cluster_id) + except WaiterError as e: + if waiter_name == 'cluster_deleted': + msg = "Failed to wait for DB cluster {0} to be deleted".format(db_cluster_id) + else: + msg = "Failed to wait for DB cluster {0} to be available".format(db_cluster_id) + module.fail_json_aws(e, msg=msg) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg="Failed with an unexpected error while waiting for the DB cluster {0}".format(db_cluster_id)) + + +def wait_for_status(client, module, identifier, method_name): + waiter_name = get_rds_method_attribute(method_name, module).waiter + if get_rds_method_attribute(method_name, module).cluster: + wait_for_cluster_status(client, module, identifier, waiter_name) + elif get_rds_method_attribute(method_name, module).instance: + wait_for_instance_status(client, module, identifier, waiter_name) + else: + raise NotImplementedError("method {0} hasn't been added to the whitelist of handled methods".format(method_name)) + + +def get_tags(client, module, cluster_arn): + try: + return boto3_tag_list_to_ansible_dict( + client.list_tags_for_resource(ResourceName=cluster_arn)['TagList'] + ) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg="Unable to describe tags") + + +def arg_spec_to_rds_params(options_dict): + tags = options_dict.pop('tags') + has_processor_features = False + if 'processor_features' in options_dict: + has_processor_features = True + processor_features = options_dict.pop('processor_features') + camel_options = snake_dict_to_camel_dict(options_dict, capitalize_first=True) + for key in list(camel_options.keys()): + for old, new in (('Db', 'DB'), ('Iam', 'IAM'), ('Az', 'AZ')): + if old in key: + camel_options[key.replace(old, new)] = camel_options.pop(key) + camel_options['Tags'] = tags + if has_processor_features: + camel_options['ProcessorFeatures'] = processor_features + return camel_options + + +def ensure_tags(client, module, resource_arn, existing_tags, tags, purge_tags): + if tags is None: + return False + tags_to_add, tags_to_remove = compare_aws_tags(existing_tags, tags, purge_tags) + changed = bool(tags_to_add or tags_to_remove) + if tags_to_add: + call_method( + client, module, method_name='add_tags_to_resource', + parameters={'ResourceName': resource_arn, 'Tags': ansible_dict_to_boto3_tag_list(tags_to_add)} + ) + if tags_to_remove: + call_method( + client, module, method_name='remove_tags_from_resource', + parameters={'ResourceName': resource_arn, 'TagKeys': tags_to_remove} + ) + return changed diff --git a/plugins/module_utils/s3.py b/plugins/module_utils/s3.py new file mode 100644 index 00000000000..2185869d499 --- /dev/null +++ b/plugins/module_utils/s3.py @@ -0,0 +1,50 @@ +# Copyright (c) 2018 Red Hat, Inc. +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +try: + from botocore.exceptions import BotoCoreError, ClientError +except ImportError: + pass # Handled by the calling module + +HAS_MD5 = True +try: + from hashlib import md5 +except ImportError: + try: + from md5 import md5 + except ImportError: + HAS_MD5 = False + + +def calculate_etag(module, filename, etag, s3, bucket, obj, version=None): + if not HAS_MD5: + return None + + if '-' in etag: + # Multi-part ETag; a hash of the hashes of each part. + parts = int(etag[1:-1].split('-')[1]) + digests = [] + + s3_kwargs = dict( + Bucket=bucket, + Key=obj, + ) + if version: + s3_kwargs['VersionId'] = version + + with open(filename, 'rb') as f: + for part_num in range(1, parts + 1): + s3_kwargs['PartNumber'] = part_num + try: + head = s3.head_object(**s3_kwargs) + except (BotoCoreError, ClientError) as e: + module.fail_json_aws(e, msg="Failed to get head object") + digests.append(md5(f.read(int(head['ContentLength'])))) + + digest_squared = md5(b''.join(m.digest() for m in digests)) + return '"{0}-{1}"'.format(digest_squared.hexdigest(), len(digests)) + else: # Compute the MD5 sum normally + return '"{0}"'.format(module.md5(filename)) diff --git a/plugins/module_utils/urls.py b/plugins/module_utils/urls.py new file mode 100644 index 00000000000..387d1f5ca62 --- /dev/null +++ b/plugins/module_utils/urls.py @@ -0,0 +1,213 @@ +# Copyright: (c) 2018, Aaron Haaf +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +import datetime +import hashlib +import hmac +import operator + +try: + from boto3 import session +except ImportError: + pass + +from ansible.module_utils.six.moves.urllib.parse import urlencode +from ansible.module_utils.urls import open_url + +from .ec2 import HAS_BOTO3 +from .ec2 import boto3_conn +from .ec2 import get_aws_connection_info + + +def hexdigest(s): + """ + Returns the sha256 hexdigest of a string after encoding. + """ + + return hashlib.sha256(s.encode("utf-8")).hexdigest() + + +def format_querystring(params=None): + """ + Returns properly url-encoded query string from the provided params dict. + + It's specially sorted for cannonical requests + """ + + if not params: + return "" + + # Query string values must be URL-encoded (space=%20). The parameters must be sorted by name. + return urlencode(sorted(params.items(), operator.itemgetter(0))) + + +# Key derivation functions. See: +# http://docs.aws.amazon.com/general/latest/gr/signature-v4-examples.html#signature-v4-examples-python +def sign(key, msg): + ''' + Return digest for key applied to msg + ''' + + return hmac.new(key, msg.encode("utf-8"), hashlib.sha256).digest() + + +def get_signature_key(key, dateStamp, regionName, serviceName): + ''' + Returns signature key for AWS resource + ''' + + kDate = sign(("AWS4" + key).encode("utf-8"), dateStamp) + kRegion = sign(kDate, regionName) + kService = sign(kRegion, serviceName) + kSigning = sign(kService, "aws4_request") + return kSigning + + +def get_aws_credentials_object(module): + ''' + Returns aws_access_key_id, aws_secret_access_key, session_token for a module. + ''' + + if not HAS_BOTO3: + module.fail_json("get_aws_credentials_object requires boto3") + + dummy, dummy, boto_params = get_aws_connection_info(module, boto3=True) + s = session.Session(**boto_params) + + return s.get_credentials() + + +# Reference: https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html +def signed_request( + module=None, + method="GET", service=None, host=None, uri=None, + query=None, body="", headers=None, + session_in_header=True, session_in_query=False +): + """Generate a SigV4 request to an AWS resource for a module + + This is used if you wish to authenticate with AWS credentials to a secure endpoint like an elastisearch domain. + + Returns :class:`HTTPResponse` object. + + Example: + result = signed_request( + module=this, + service="es", + host="search-recipes1-xxxxxxxxx.us-west-2.es.amazonaws.com", + ) + + :kwarg host: endpoint to talk to + :kwarg service: AWS id of service (like `ec2` or `es`) + :kwarg module: An AnsibleAWSModule to gather connection info from + + :kwarg body: (optional) Payload to send + :kwarg method: (optional) HTTP verb to use + :kwarg query: (optional) dict of query params to handle + :kwarg uri: (optional) Resource path without query parameters + + :kwarg session_in_header: (optional) Add the session token to the headers + :kwarg session_in_query: (optional) Add the session token to the query parameters + + :returns: HTTPResponse + """ + + if not HAS_BOTO3: + module.fail_json("A sigv4 signed_request requires boto3") + + # "Constants" + + t = datetime.datetime.utcnow() + amz_date = t.strftime("%Y%m%dT%H%M%SZ") + datestamp = t.strftime("%Y%m%d") # Date w/o time, used in credential scope + algorithm = "AWS4-HMAC-SHA256" + + # AWS stuff + + region, dummy, dummy = get_aws_connection_info(module, boto3=True) + credentials = get_aws_credentials_object(module) + access_key = credentials.access_key + secret_key = credentials.secret_key + session_token = credentials.token + + if not access_key: + module.fail_json(msg="aws_access_key_id is missing") + if not secret_key: + module.fail_json(msg="aws_secret_access_key is missing") + + credential_scope = "/".join([datestamp, region, service, "aws4_request"]) + + # Argument Defaults + + uri = uri or "/" + query_string = format_querystring(query) if query else "" + + headers = headers or dict() + query = query or dict() + + headers.update({ + "host": host, + "x-amz-date": amz_date, + }) + + # Handle adding of session_token if present + if session_token: + if session_in_header: + headers["X-Amz-Security-Token"] = session_token + if session_in_query: + query["X-Amz-Security-Token"] = session_token + + if method == "GET": + body = "" + + # Derived data + + body_hash = hexdigest(body) + signed_headers = ";".join(sorted(headers.keys())) + + # Setup Cannonical request to generate auth token + + cannonical_headers = "\n".join([ + key.lower().strip() + ":" + value for key, value in headers.items() + ]) + "\n" # Note additional trailing newline + + cannonical_request = "\n".join([ + method, + uri, + query_string, + cannonical_headers, + signed_headers, + body_hash, + ]) + + string_to_sign = "\n".join([algorithm, amz_date, credential_scope, hexdigest(cannonical_request)]) + + # Sign the Cannonical request + + signing_key = get_signature_key(secret_key, datestamp, region, service) + signature = hmac.new(signing_key, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() + + # Make auth header with that info + + authorization_header = "{0} Credential={1}/{2}, SignedHeaders={3}, Signature={4}".format( + algorithm, access_key, credential_scope, signed_headers, signature + ) + + # PERFORM THE REQUEST! + + url = "https://" + host + uri + + if query_string != "": + url = url + "?" + query_string + + final_headers = { + "x-amz-date": amz_date, + "Authorization": authorization_header, + } + + final_headers.update(headers) + + return open_url(url, method=method, data=body, headers=final_headers) diff --git a/plugins/module_utils/waf.py b/plugins/module_utils/waf.py new file mode 100644 index 00000000000..3ecc645c460 --- /dev/null +++ b/plugins/module_utils/waf.py @@ -0,0 +1,224 @@ +# Copyright (c) 2017 Will Thames +# +# This code is part of Ansible, but is an independent component. +# This particular file snippet, and this file snippet only, is BSD licensed. +# Modules you write using this snippet, which is embedded dynamically by Ansible +# still belong to the author of the module, and may assign their own license +# to the complete work. +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE +# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +""" +This module adds shared support for Web Application Firewall modules +""" + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +try: + import botocore +except ImportError: + pass # caught by imported HAS_BOTO3 + +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from .ec2 import AWSRetry +from .waiters import get_waiter + + +MATCH_LOOKUP = { + 'byte': { + 'method': 'byte_match_set', + 'conditionset': 'ByteMatchSet', + 'conditiontuple': 'ByteMatchTuple', + 'type': 'ByteMatch' + }, + 'geo': { + 'method': 'geo_match_set', + 'conditionset': 'GeoMatchSet', + 'conditiontuple': 'GeoMatchConstraint', + 'type': 'GeoMatch' + }, + 'ip': { + 'method': 'ip_set', + 'conditionset': 'IPSet', + 'conditiontuple': 'IPSetDescriptor', + 'type': 'IPMatch' + }, + 'regex': { + 'method': 'regex_match_set', + 'conditionset': 'RegexMatchSet', + 'conditiontuple': 'RegexMatchTuple', + 'type': 'RegexMatch' + }, + 'size': { + 'method': 'size_constraint_set', + 'conditionset': 'SizeConstraintSet', + 'conditiontuple': 'SizeConstraint', + 'type': 'SizeConstraint' + }, + 'sql': { + 'method': 'sql_injection_match_set', + 'conditionset': 'SqlInjectionMatchSet', + 'conditiontuple': 'SqlInjectionMatchTuple', + 'type': 'SqlInjectionMatch', + }, + 'xss': { + 'method': 'xss_match_set', + 'conditionset': 'XssMatchSet', + 'conditiontuple': 'XssMatchTuple', + 'type': 'XssMatch' + }, +} + + +@AWSRetry.backoff(tries=5, delay=5, backoff=2.0) +def get_rule_with_backoff(client, rule_id): + return client.get_rule(RuleId=rule_id)['Rule'] + + +@AWSRetry.backoff(tries=5, delay=5, backoff=2.0) +def get_byte_match_set_with_backoff(client, byte_match_set_id): + return client.get_byte_match_set(ByteMatchSetId=byte_match_set_id)['ByteMatchSet'] + + +@AWSRetry.backoff(tries=5, delay=5, backoff=2.0) +def get_ip_set_with_backoff(client, ip_set_id): + return client.get_ip_set(IPSetId=ip_set_id)['IPSet'] + + +@AWSRetry.backoff(tries=5, delay=5, backoff=2.0) +def get_size_constraint_set_with_backoff(client, size_constraint_set_id): + return client.get_size_constraint_set(SizeConstraintSetId=size_constraint_set_id)['SizeConstraintSet'] + + +@AWSRetry.backoff(tries=5, delay=5, backoff=2.0) +def get_sql_injection_match_set_with_backoff(client, sql_injection_match_set_id): + return client.get_sql_injection_match_set(SqlInjectionMatchSetId=sql_injection_match_set_id)['SqlInjectionMatchSet'] + + +@AWSRetry.backoff(tries=5, delay=5, backoff=2.0) +def get_xss_match_set_with_backoff(client, xss_match_set_id): + return client.get_xss_match_set(XssMatchSetId=xss_match_set_id)['XssMatchSet'] + + +def get_rule(client, module, rule_id): + try: + rule = get_rule_with_backoff(client, rule_id) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Couldn't obtain waf rule") + + match_sets = { + 'ByteMatch': get_byte_match_set_with_backoff, + 'IPMatch': get_ip_set_with_backoff, + 'SizeConstraint': get_size_constraint_set_with_backoff, + 'SqlInjectionMatch': get_sql_injection_match_set_with_backoff, + 'XssMatch': get_xss_match_set_with_backoff + } + if 'Predicates' in rule: + for predicate in rule['Predicates']: + if predicate['Type'] in match_sets: + predicate.update(match_sets[predicate['Type']](client, predicate['DataId'])) + # replaced by Id from the relevant MatchSet + del(predicate['DataId']) + return rule + + +@AWSRetry.backoff(tries=5, delay=5, backoff=2.0) +def get_web_acl_with_backoff(client, web_acl_id): + return client.get_web_acl(WebACLId=web_acl_id)['WebACL'] + + +def get_web_acl(client, module, web_acl_id): + try: + web_acl = get_web_acl_with_backoff(client, web_acl_id) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Couldn't obtain web acl") + + if web_acl: + try: + for rule in web_acl['Rules']: + rule.update(get_rule(client, module, rule['RuleId'])) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Couldn't obtain web acl rule") + return camel_dict_to_snake_dict(web_acl) + + +@AWSRetry.backoff(tries=5, delay=5, backoff=2.0) +def list_rules_with_backoff(client): + paginator = client.get_paginator('list_rules') + return paginator.paginate().build_full_result()['Rules'] + + +@AWSRetry.backoff(tries=5, delay=5, backoff=2.0) +def list_regional_rules_with_backoff(client): + resp = client.list_rules() + rules = [] + while resp: + rules += resp['Rules'] + resp = client.list_rules(NextMarker=resp['NextMarker']) if 'NextMarker' in resp else None + return rules + + +@AWSRetry.backoff(tries=5, delay=5, backoff=2.0) +def list_web_acls_with_backoff(client): + paginator = client.get_paginator('list_web_acls') + return paginator.paginate().build_full_result()['WebACLs'] + + +@AWSRetry.backoff(tries=5, delay=5, backoff=2.0) +def list_regional_web_acls_with_backoff(client): + resp = client.list_web_acls() + acls = [] + while resp: + acls += resp['WebACLs'] + resp = client.list_web_acls(NextMarker=resp['NextMarker']) if 'NextMarker' in resp else None + return acls + + +def list_web_acls(client, module): + try: + if client.__class__.__name__ == 'WAF': + return list_web_acls_with_backoff(client) + elif client.__class__.__name__ == 'WAFRegional': + return list_regional_web_acls_with_backoff(client) + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Couldn't obtain web acls") + + +def get_change_token(client, module): + try: + token = client.get_change_token() + return token['ChangeToken'] + except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e: + module.fail_json_aws(e, msg="Couldn't obtain change token") + + +@AWSRetry.backoff(tries=10, delay=2, backoff=2.0, catch_extra_error_codes=['WAFStaleDataException']) +def run_func_with_change_token_backoff(client, module, params, func, wait=False): + params['ChangeToken'] = get_change_token(client, module) + result = func(**params) + if wait: + get_waiter( + client, 'change_token_in_sync', + ).wait( + ChangeToken=result['ChangeToken'] + ) + return result diff --git a/plugins/module_utils/waiters.py b/plugins/module_utils/waiters.py new file mode 100644 index 00000000000..25db598bcb3 --- /dev/null +++ b/plugins/module_utils/waiters.py @@ -0,0 +1,405 @@ +# Copyright: (c) 2018, Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +try: + import botocore.waiter as core_waiter +except ImportError: + pass # caught by HAS_BOTO3 + + +ec2_data = { + "version": 2, + "waiters": { + "InternetGatewayExists": { + "delay": 5, + "maxAttempts": 40, + "operation": "DescribeInternetGateways", + "acceptors": [ + { + "matcher": "path", + "expected": True, + "argument": "length(InternetGateways) > `0`", + "state": "success" + }, + { + "matcher": "error", + "expected": "InvalidInternetGatewayID.NotFound", + "state": "retry" + }, + ] + }, + "RouteTableExists": { + "delay": 5, + "maxAttempts": 40, + "operation": "DescribeRouteTables", + "acceptors": [ + { + "matcher": "path", + "expected": True, + "argument": "length(RouteTables[]) > `0`", + "state": "success" + }, + { + "matcher": "error", + "expected": "InvalidRouteTableID.NotFound", + "state": "retry" + }, + ] + }, + "SecurityGroupExists": { + "delay": 5, + "maxAttempts": 40, + "operation": "DescribeSecurityGroups", + "acceptors": [ + { + "matcher": "path", + "expected": True, + "argument": "length(SecurityGroups[]) > `0`", + "state": "success" + }, + { + "matcher": "error", + "expected": "InvalidGroup.NotFound", + "state": "retry" + }, + ] + }, + "SubnetExists": { + "delay": 5, + "maxAttempts": 40, + "operation": "DescribeSubnets", + "acceptors": [ + { + "matcher": "path", + "expected": True, + "argument": "length(Subnets[]) > `0`", + "state": "success" + }, + { + "matcher": "error", + "expected": "InvalidSubnetID.NotFound", + "state": "retry" + }, + ] + }, + "SubnetHasMapPublic": { + "delay": 5, + "maxAttempts": 40, + "operation": "DescribeSubnets", + "acceptors": [ + { + "matcher": "pathAll", + "expected": True, + "argument": "Subnets[].MapPublicIpOnLaunch", + "state": "success" + }, + ] + }, + "SubnetNoMapPublic": { + "delay": 5, + "maxAttempts": 40, + "operation": "DescribeSubnets", + "acceptors": [ + { + "matcher": "pathAll", + "expected": False, + "argument": "Subnets[].MapPublicIpOnLaunch", + "state": "success" + }, + ] + }, + "SubnetHasAssignIpv6": { + "delay": 5, + "maxAttempts": 40, + "operation": "DescribeSubnets", + "acceptors": [ + { + "matcher": "pathAll", + "expected": True, + "argument": "Subnets[].AssignIpv6AddressOnCreation", + "state": "success" + }, + ] + }, + "SubnetNoAssignIpv6": { + "delay": 5, + "maxAttempts": 40, + "operation": "DescribeSubnets", + "acceptors": [ + { + "matcher": "pathAll", + "expected": False, + "argument": "Subnets[].AssignIpv6AddressOnCreation", + "state": "success" + }, + ] + }, + "SubnetDeleted": { + "delay": 5, + "maxAttempts": 40, + "operation": "DescribeSubnets", + "acceptors": [ + { + "matcher": "path", + "expected": True, + "argument": "length(Subnets[]) > `0`", + "state": "retry" + }, + { + "matcher": "error", + "expected": "InvalidSubnetID.NotFound", + "state": "success" + }, + ] + }, + "VpnGatewayExists": { + "delay": 5, + "maxAttempts": 40, + "operation": "DescribeVpnGateways", + "acceptors": [ + { + "matcher": "path", + "expected": True, + "argument": "length(VpnGateways[]) > `0`", + "state": "success" + }, + { + "matcher": "error", + "expected": "InvalidVpnGatewayID.NotFound", + "state": "retry" + }, + ] + }, + "VpnGatewayDetached": { + "delay": 5, + "maxAttempts": 40, + "operation": "DescribeVpnGateways", + "acceptors": [ + { + "matcher": "path", + "expected": True, + "argument": "VpnGateways[0].State == 'available'", + "state": "success" + }, + ] + }, + } +} + + +waf_data = { + "version": 2, + "waiters": { + "ChangeTokenInSync": { + "delay": 20, + "maxAttempts": 60, + "operation": "GetChangeTokenStatus", + "acceptors": [ + { + "matcher": "path", + "expected": True, + "argument": "ChangeTokenStatus == 'INSYNC'", + "state": "success" + }, + { + "matcher": "error", + "expected": "WAFInternalErrorException", + "state": "retry" + } + ] + } + } +} + +eks_data = { + "version": 2, + "waiters": { + "ClusterActive": { + "delay": 20, + "maxAttempts": 60, + "operation": "DescribeCluster", + "acceptors": [ + { + "state": "success", + "matcher": "path", + "argument": "cluster.status", + "expected": "ACTIVE" + }, + { + "state": "retry", + "matcher": "error", + "expected": "ResourceNotFoundException" + } + ] + }, + "ClusterDeleted": { + "delay": 20, + "maxAttempts": 60, + "operation": "DescribeCluster", + "acceptors": [ + { + "state": "retry", + "matcher": "path", + "argument": "cluster.status != 'DELETED'", + "expected": True + }, + { + "state": "success", + "matcher": "error", + "expected": "ResourceNotFoundException" + } + ] + } + } +} + + +rds_data = { + "version": 2, + "waiters": { + "DBInstanceStopped": { + "delay": 20, + "maxAttempts": 60, + "operation": "DescribeDBInstances", + "acceptors": [ + { + "state": "success", + "matcher": "pathAll", + "argument": "DBInstances[].DBInstanceStatus", + "expected": "stopped" + }, + ] + } + } +} + + +def ec2_model(name): + ec2_models = core_waiter.WaiterModel(waiter_config=ec2_data) + return ec2_models.get_waiter(name) + + +def waf_model(name): + waf_models = core_waiter.WaiterModel(waiter_config=waf_data) + return waf_models.get_waiter(name) + + +def eks_model(name): + eks_models = core_waiter.WaiterModel(waiter_config=eks_data) + return eks_models.get_waiter(name) + + +def rds_model(name): + rds_models = core_waiter.WaiterModel(waiter_config=rds_data) + return rds_models.get_waiter(name) + + +waiters_by_name = { + ('EC2', 'internet_gateway_exists'): lambda ec2: core_waiter.Waiter( + 'internet_gateway_exists', + ec2_model('InternetGatewayExists'), + core_waiter.NormalizedOperationMethod( + ec2.describe_internet_gateways + )), + ('EC2', 'route_table_exists'): lambda ec2: core_waiter.Waiter( + 'route_table_exists', + ec2_model('RouteTableExists'), + core_waiter.NormalizedOperationMethod( + ec2.describe_route_tables + )), + ('EC2', 'security_group_exists'): lambda ec2: core_waiter.Waiter( + 'security_group_exists', + ec2_model('SecurityGroupExists'), + core_waiter.NormalizedOperationMethod( + ec2.describe_security_groups + )), + ('EC2', 'subnet_exists'): lambda ec2: core_waiter.Waiter( + 'subnet_exists', + ec2_model('SubnetExists'), + core_waiter.NormalizedOperationMethod( + ec2.describe_subnets + )), + ('EC2', 'subnet_has_map_public'): lambda ec2: core_waiter.Waiter( + 'subnet_has_map_public', + ec2_model('SubnetHasMapPublic'), + core_waiter.NormalizedOperationMethod( + ec2.describe_subnets + )), + ('EC2', 'subnet_no_map_public'): lambda ec2: core_waiter.Waiter( + 'subnet_no_map_public', + ec2_model('SubnetNoMapPublic'), + core_waiter.NormalizedOperationMethod( + ec2.describe_subnets + )), + ('EC2', 'subnet_has_assign_ipv6'): lambda ec2: core_waiter.Waiter( + 'subnet_has_assign_ipv6', + ec2_model('SubnetHasAssignIpv6'), + core_waiter.NormalizedOperationMethod( + ec2.describe_subnets + )), + ('EC2', 'subnet_no_assign_ipv6'): lambda ec2: core_waiter.Waiter( + 'subnet_no_assign_ipv6', + ec2_model('SubnetNoAssignIpv6'), + core_waiter.NormalizedOperationMethod( + ec2.describe_subnets + )), + ('EC2', 'subnet_deleted'): lambda ec2: core_waiter.Waiter( + 'subnet_deleted', + ec2_model('SubnetDeleted'), + core_waiter.NormalizedOperationMethod( + ec2.describe_subnets + )), + ('EC2', 'vpn_gateway_exists'): lambda ec2: core_waiter.Waiter( + 'vpn_gateway_exists', + ec2_model('VpnGatewayExists'), + core_waiter.NormalizedOperationMethod( + ec2.describe_vpn_gateways + )), + ('EC2', 'vpn_gateway_detached'): lambda ec2: core_waiter.Waiter( + 'vpn_gateway_detached', + ec2_model('VpnGatewayDetached'), + core_waiter.NormalizedOperationMethod( + ec2.describe_vpn_gateways + )), + ('WAF', 'change_token_in_sync'): lambda waf: core_waiter.Waiter( + 'change_token_in_sync', + waf_model('ChangeTokenInSync'), + core_waiter.NormalizedOperationMethod( + waf.get_change_token_status + )), + ('WAFRegional', 'change_token_in_sync'): lambda waf: core_waiter.Waiter( + 'change_token_in_sync', + waf_model('ChangeTokenInSync'), + core_waiter.NormalizedOperationMethod( + waf.get_change_token_status + )), + ('EKS', 'cluster_active'): lambda eks: core_waiter.Waiter( + 'cluster_active', + eks_model('ClusterActive'), + core_waiter.NormalizedOperationMethod( + eks.describe_cluster + )), + ('EKS', 'cluster_deleted'): lambda eks: core_waiter.Waiter( + 'cluster_deleted', + eks_model('ClusterDeleted'), + core_waiter.NormalizedOperationMethod( + eks.describe_cluster + )), + ('RDS', 'db_instance_stopped'): lambda rds: core_waiter.Waiter( + 'db_instance_stopped', + rds_model('DBInstanceStopped'), + core_waiter.NormalizedOperationMethod( + rds.describe_db_instances + )), +} + + +def get_waiter(client, waiter_name): + try: + return waiters_by_name[(client.__class__.__name__, waiter_name)](client) + except KeyError: + raise NotImplementedError("Waiter {0} could not be found for client {1}. Available waiters: {2}".format( + waiter_name, type(client), ', '.join(repr(k) for k in waiters_by_name.keys()))) diff --git a/plugins/modules/aws_az_info.py b/plugins/modules/aws_az_info.py index 7e870107db0..098be65b046 100644 --- a/plugins/modules/aws_az_info.py +++ b/plugins/modules/aws_az_info.py @@ -65,14 +65,17 @@ ]" ''' -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry, ansible_dict_to_boto3_filter_list, camel_dict_to_snake_dict - try: from botocore.exceptions import ClientError, BotoCoreError except ImportError: pass # Handled by AnsibleAWSModule +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import AWSRetry +from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list + def main(): argument_spec = dict( diff --git a/plugins/modules/aws_caller_info.py b/plugins/modules/aws_caller_info.py index 6e49d6438f2..590efd5643e 100644 --- a/plugins/modules/aws_caller_info.py +++ b/plugins/modules/aws_caller_info.py @@ -59,13 +59,14 @@ sample: 123456789012:my-federated-user-name ''' -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import camel_dict_to_snake_dict - try: from botocore.exceptions import BotoCoreError, ClientError except ImportError: - pass # caught by AnsibleAWSModule + pass # Handled by AnsibleAWSModule + +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from ..module_utils.core import AnsibleAWSModule def main(): diff --git a/plugins/modules/aws_s3.py b/plugins/modules/aws_s3.py index 93e0fabcdf6..e32f0c99ace 100644 --- a/plugins/modules/aws_s3.py +++ b/plugins/modules/aws_s3.py @@ -276,17 +276,22 @@ import mimetypes import os -from ansible.module_utils.six.moves.urllib.parse import urlparse from ssl import SSLError -from ansible.module_utils.basic import to_text, to_native -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.aws.s3 import calculate_etag, HAS_MD5 -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_aws_connection_info, boto3_conn try: import botocore except ImportError: - pass # will be detected by imported AnsibleAWSModule + pass # Handled by AnsibleAWSModule + +from ansible.module_utils.basic import to_text +from ansible.module_utils.basic import to_native +from ansible.module_utils.six.moves.urllib.parse import urlparse + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import boto3_conn +from ..module_utils.ec2 import get_aws_connection_info +from ..module_utils.s3 import HAS_MD5 +from ..module_utils.s3 import calculate_etag IGNORE_S3_DROP_IN_EXCEPTIONS = ['XNotImplemented', 'NotImplemented'] diff --git a/plugins/modules/cloudformation.py b/plugins/modules/cloudformation.py index 2e2946fbf13..7549e95b342 100644 --- a/plugins/modules/cloudformation.py +++ b/plugins/modules/cloudformation.py @@ -319,8 +319,8 @@ import json import time -import uuid import traceback +import uuid from hashlib import sha1 try: @@ -331,10 +331,11 @@ from ansible.module_utils._text import to_bytes from ansible.module_utils._text import to_native -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_tag_list -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto_exception + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import AWSRetry +from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list +from ..module_utils.ec2 import boto_exception def get_stack_events(cfn, stack_name, events_limit, token_filter=None): diff --git a/plugins/modules/cloudformation_info.py b/plugins/modules/cloudformation_info.py index d27ffa670dc..ea51c490fbe 100644 --- a/plugins/modules/cloudformation_info.py +++ b/plugins/modules/cloudformation_info.py @@ -166,16 +166,19 @@ import json import traceback - from functools import partial -from ansible.module_utils._text import to_native -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (camel_dict_to_snake_dict, AWSRetry, boto3_tag_list_to_ansible_dict) try: import botocore except ImportError: - pass # handled by AnsibleAWSModule + pass # Handled by AnsibleAWSModule + +from ansible.module_utils._text import to_native +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import AWSRetry +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict class CloudFormationServiceManager: diff --git a/plugins/modules/ec2.py b/plugins/modules/ec2.py index c03753a80af..2852a41cd21 100644 --- a/plugins/modules/ec2.py +++ b/plugins/modules/ec2.py @@ -592,10 +592,11 @@ from ansible.module_utils.six import string_types from ansible.module_utils._text import to_bytes from ansible.module_utils._text import to_text -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ec2_connect -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_aws_connection_info + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import HAS_BOTO +from ..module_utils.ec2 import ec2_connect +from ..module_utils.ec2 import get_aws_connection_info def find_running_instances_by_count_tag(module, ec2, vpc, count_tag, zone=None): diff --git a/plugins/modules/ec2_ami.py b/plugins/modules/ec2_ami.py index 98b27db7cc4..f246f7bccda 100644 --- a/plugins/modules/ec2_ami.py +++ b/plugins/modules/ec2_ami.py @@ -347,17 +347,18 @@ ''' import time -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (ansible_dict_to_boto3_tag_list, - boto3_tag_list_to_ansible_dict, - camel_dict_to_snake_dict, - compare_aws_tags, - ) -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule try: import botocore except ImportError: - pass # caught by AnsibleAWSModule + pass # Handled by AnsibleAWSModule + +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict +from ..module_utils.ec2 import compare_aws_tags def get_block_device_mapping(image): diff --git a/plugins/modules/ec2_ami_info.py b/plugins/modules/ec2_ami_info.py index e1b6fdb82da..7af9a8de823 100644 --- a/plugins/modules/ec2_ami_info.py +++ b/plugins/modules/ec2_ami_info.py @@ -201,14 +201,14 @@ try: from botocore.exceptions import ClientError, BotoCoreError except ImportError: - pass # caught by AnsibleAWSModule + pass # Handled by AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (ansible_dict_to_boto3_tag_list, - camel_dict_to_snake_dict, - boto3_tag_list_to_ansible_dict, - ansible_dict_to_boto3_filter_list, - ) +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list +from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict def list_ec2_images(ec2_client, module): diff --git a/plugins/modules/ec2_elb_lb.py b/plugins/modules/ec2_elb_lb.py index d36e2d84253..e0771cfd7a6 100644 --- a/plugins/modules/ec2_elb_lb.py +++ b/plugins/modules/ec2_elb_lb.py @@ -355,11 +355,12 @@ from ansible.module_utils.six import string_types from ansible.module_utils._text import to_native -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AnsibleAWSError -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import connect_to_aws -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_aws_connection_info + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import AnsibleAWSError +from ..module_utils.ec2 import HAS_BOTO +from ..module_utils.ec2 import connect_to_aws +from ..module_utils.ec2 import get_aws_connection_info def _throttleable_operation(max_retries): diff --git a/plugins/modules/ec2_eni.py b/plugins/modules/ec2_eni.py index 00cea5e1b62..487313fdc47 100644 --- a/plugins/modules/ec2_eni.py +++ b/plugins/modules/ec2_eni.py @@ -258,12 +258,12 @@ except ImportError: pass # Taken care of by ec2.HAS_BOTO -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AnsibleAWSError -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import connect_to_aws -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_aws_connection_info -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_ec2_security_group_ids_from_names +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import AnsibleAWSError +from ..module_utils.ec2 import HAS_BOTO +from ..module_utils.ec2 import connect_to_aws +from ..module_utils.ec2 import get_aws_connection_info +from ..module_utils.ec2 import get_ec2_security_group_ids_from_names def get_eni_info(interface): diff --git a/plugins/modules/ec2_eni_info.py b/plugins/modules/ec2_eni_info.py index c818f46d082..443ff611ba9 100644 --- a/plugins/modules/ec2_eni_info.py +++ b/plugins/modules/ec2_eni_info.py @@ -179,9 +179,10 @@ pass # Handled by AnsibleAWSModule from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict def list_eni(connection, module): diff --git a/plugins/modules/ec2_group.py b/plugins/modules/ec2_group.py index 3393ca9611d..34837ec1940 100644 --- a/plugins/modules/ec2_group.py +++ b/plugins/modules/ec2_group.py @@ -395,23 +395,29 @@ from copy import deepcopy from time import sleep from collections import namedtuple -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code -from ansible_collections.amazon.aws.plugins.module_utils.aws.iam import get_aws_account_id -from ansible_collections.amazon.aws.plugins.module_utils.aws.waiters import get_waiter -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry, camel_dict_to_snake_dict, compare_aws_tags -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (ansible_dict_to_boto3_filter_list, - boto3_tag_list_to_ansible_dict, - ansible_dict_to_boto3_tag_list, - ) -from ansible.module_utils.common.network import to_ipv6_subnet, to_subnet -from ansible_collections.ansible.netcommon.plugins.module_utils.compat.ipaddress import ip_network, IPv6Network -from ansible.module_utils._text import to_text -from ansible.module_utils.six import string_types try: from botocore.exceptions import BotoCoreError, ClientError except ImportError: - pass # caught by AnsibleAWSModule + pass # Handled by AnsibleAWSModule + +from ansible.module_utils._text import to_text +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict +from ansible.module_utils.common.network import to_ipv6_subnet +from ansible.module_utils.common.network import to_subnet +from ansible.module_utils.six import string_types +from ansible_collections.ansible.netcommon.plugins.module_utils.compat.ipaddress import IPv6Network +from ansible_collections.ansible.netcommon.plugins.module_utils.compat.ipaddress import ip_network + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.core import is_boto3_error_code +from ..module_utils.ec2 import AWSRetry +from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list +from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict +from ..module_utils.ec2 import compare_aws_tags +from ..module_utils.iam import get_aws_account_id +from ..module_utils.waiters import get_waiter Rule = namedtuple('Rule', ['port_range', 'protocol', 'target', 'target_type', 'description']) diff --git a/plugins/modules/ec2_group_info.py b/plugins/modules/ec2_group_info.py index f556054fc18..7d73fb6c614 100644 --- a/plugins/modules/ec2_group_info.py +++ b/plugins/modules/ec2_group_info.py @@ -95,11 +95,11 @@ except ImportError: pass # caught by AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (boto3_tag_list_to_ansible_dict, - ansible_dict_to_boto3_filter_list, - camel_dict_to_snake_dict, - ) +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict def main(): diff --git a/plugins/modules/ec2_key.py b/plugins/modules/ec2_key.py index d8fb35204a7..15c029cebc2 100644 --- a/plugins/modules/ec2_key.py +++ b/plugins/modules/ec2_key.py @@ -125,14 +125,15 @@ import uuid -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible.module_utils._text import to_bytes - try: from botocore.exceptions import ClientError except ImportError: pass # caught by AnsibleAWSModule +from ansible.module_utils._text import to_bytes + +from ..module_utils.core import AnsibleAWSModule + def extract_key_data(key): diff --git a/plugins/modules/ec2_snapshot.py b/plugins/modules/ec2_snapshot.py index 0b253769f1d..6d7e5c88e5d 100644 --- a/plugins/modules/ec2_snapshot.py +++ b/plugins/modules/ec2_snapshot.py @@ -139,9 +139,9 @@ except ImportError: pass # Taken care of by ec2.HAS_BOTO -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ec2_connect +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import HAS_BOTO +from ..module_utils.ec2 import ec2_connect # Find the most recent snapshot diff --git a/plugins/modules/ec2_snapshot_info.py b/plugins/modules/ec2_snapshot_info.py index 2207bfe8ca9..24cf025cc20 100644 --- a/plugins/modules/ec2_snapshot_info.py +++ b/plugins/modules/ec2_snapshot_info.py @@ -180,10 +180,11 @@ pass # Handled by AnsibleAWSModule from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import is_boto3_error_code -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.core import is_boto3_error_code +from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict def list_ec2_snapshots(connection, module): diff --git a/plugins/modules/ec2_tag.py b/plugins/modules/ec2_tag.py index bd45d7267b0..cd043e48370 100644 --- a/plugins/modules/ec2_tag.py +++ b/plugins/modules/ec2_tag.py @@ -113,14 +113,17 @@ type: dict ''' -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ..module_utils.ec2 import AWSRetry, ansible_dict_to_boto3_tag_list, boto3_tag_list_to_ansible_dict, compare_aws_tags - try: from botocore.exceptions import BotoCoreError, ClientError except Exception: pass # Handled by AnsibleAWSModule +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import AWSRetry +from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict +from ..module_utils.ec2 import compare_aws_tags + def get_tags(ec2, module, resource): filters = [{'Name': 'resource-id', 'Values': [resource]}] diff --git a/plugins/modules/ec2_tag_info.py b/plugins/modules/ec2_tag_info.py index 5762143e787..beb2a23e109 100644 --- a/plugins/modules/ec2_tag_info.py +++ b/plugins/modules/ec2_tag_info.py @@ -51,14 +51,14 @@ type: dict ''' -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict, AWSRetry - try: from botocore.exceptions import BotoCoreError, ClientError except Exception: pass # Handled by AnsibleAWSModule +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict, AWSRetry + @AWSRetry.jittered_backoff() def get_tags(ec2, module, resource): diff --git a/plugins/modules/ec2_vol.py b/plugins/modules/ec2_vol.py index d38acd7d696..53600fd208e 100644 --- a/plugins/modules/ec2_vol.py +++ b/plugins/modules/ec2_vol.py @@ -233,11 +233,11 @@ except ImportError: pass # Taken care of by ec2.HAS_BOTO -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AnsibleAWSError -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import connect_to_aws -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_aws_connection_info +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import AnsibleAWSError +from ..module_utils.ec2 import HAS_BOTO +from ..module_utils.ec2 import connect_to_aws +from ..module_utils.ec2 import get_aws_connection_info def get_volume(module, ec2): diff --git a/plugins/modules/ec2_vol_info.py b/plugins/modules/ec2_vol_info.py index d2440c37363..eb8beee089e 100644 --- a/plugins/modules/ec2_vol_info.py +++ b/plugins/modules/ec2_vol_info.py @@ -59,14 +59,14 @@ try: from botocore.exceptions import ClientError except ImportError: - pass # caught by AnsibleAWSModule - -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (boto3_tag_list_to_ansible_dict, - ansible_dict_to_boto3_filter_list, - camel_dict_to_snake_dict, - ) + pass # Handled by AnsibleAWSModule + +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import AWSRetry +from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict def get_volume_info(volume, region): diff --git a/plugins/modules/ec2_vpc_dhcp_option.py b/plugins/modules/ec2_vpc_dhcp_option.py index 9afdd0a4a2e..322808187e4 100644 --- a/plugins/modules/ec2_vpc_dhcp_option.py +++ b/plugins/modules/ec2_vpc_dhcp_option.py @@ -199,10 +199,10 @@ except ImportError: pass # Taken care of by ec2.HAS_BOTO -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import connect_to_aws -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_aws_connection_info +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import HAS_BOTO +from ..module_utils.ec2 import connect_to_aws +from ..module_utils.ec2 import get_aws_connection_info def get_resource_tags(vpc_conn, resource_id): diff --git a/plugins/modules/ec2_vpc_dhcp_option_info.py b/plugins/modules/ec2_vpc_dhcp_option_info.py index e1153007283..acedcc93d7e 100644 --- a/plugins/modules/ec2_vpc_dhcp_option_info.py +++ b/plugins/modules/ec2_vpc_dhcp_option_info.py @@ -85,9 +85,10 @@ pass # Handled by AnsibleAWSModule from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict def get_dhcp_options_info(dhcp_option): diff --git a/plugins/modules/ec2_vpc_net.py b/plugins/modules/ec2_vpc_net.py index 7224f668080..8cefe22f3ca 100644 --- a/plugins/modules/ec2_vpc_net.py +++ b/plugins/modules/ec2_vpc_net.py @@ -189,22 +189,24 @@ sample: pk_vpc4 ''' +from time import sleep +from time import time + try: import botocore except ImportError: pass # Handled by AnsibleAWSModule -from time import sleep, time -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (AWSRetry, - camel_dict_to_snake_dict, - compare_aws_tags, - ansible_dict_to_boto3_tag_list, - boto3_tag_list_to_ansible_dict, - ) -from ansible.module_utils.six import string_types from ansible.module_utils._text import to_native +from ansible.module_utils.six import string_types from ansible.module_utils.common.network import to_subnet +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import AWSRetry +from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict +from ..module_utils.ec2 import compare_aws_tags def vpc_exists(module, vpc, name, cidr_block, multi): diff --git a/plugins/modules/ec2_vpc_net_info.py b/plugins/modules/ec2_vpc_net_info.py index 4c4670a5e25..991862e1c7d 100644 --- a/plugins/modules/ec2_vpc_net_info.py +++ b/plugins/modules/ec2_vpc_net_info.py @@ -149,18 +149,19 @@ import traceback -from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import is_boto3_error_code -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict - try: import botocore except ImportError: pass # Handled by AnsibleAWSModule +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.core import is_boto3_error_code +from ..module_utils.ec2 import AWSRetry +from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict + @AWSRetry.exponential_backoff() def describe_vpc_attr_with_backoff(connection, vpc_id, vpc_attribute): diff --git a/plugins/modules/ec2_vpc_subnet.py b/plugins/modules/ec2_vpc_subnet.py index 20e713d96f7..6c05d89916c 100644 --- a/plugins/modules/ec2_vpc_subnet.py +++ b/plugins/modules/ec2_vpc_subnet.py @@ -210,15 +210,15 @@ pass # caught by AnsibleAWSModule from ansible.module_utils._text import to_text -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.aws.waiters import get_waiter -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import (ansible_dict_to_boto3_filter_list, - ansible_dict_to_boto3_tag_list, - camel_dict_to_snake_dict, - boto3_tag_list_to_ansible_dict, - compare_aws_tags, - AWSRetry, - ) +from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import AWSRetry +from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list +from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict +from ..module_utils.ec2 import compare_aws_tags +from ..module_utils.waiters import get_waiter def get_subnet_info(subnet): diff --git a/plugins/modules/ec2_vpc_subnet_info.py b/plugins/modules/ec2_vpc_subnet_info.py index ae8a09d8d4d..6ccadf078ed 100644 --- a/plugins/modules/ec2_vpc_subnet_info.py +++ b/plugins/modules/ec2_vpc_subnet_info.py @@ -153,10 +153,11 @@ pass # Handled by AnsibleAWSModule from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import ansible_dict_to_boto3_filter_list -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_tag_list_to_ansible_dict + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.ec2 import AWSRetry +from ..module_utils.ec2 import ansible_dict_to_boto3_filter_list +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict @AWSRetry.exponential_backoff() diff --git a/plugins/modules/s3_bucket.py b/plugins/modules/s3_bucket.py index df0a7915198..5c1215b4222 100644 --- a/plugins/modules/s3_bucket.py +++ b/plugins/modules/s3_bucket.py @@ -159,17 +159,23 @@ import os import time -from ansible.module_utils.six.moves.urllib.parse import urlparse -from ansible.module_utils.six import string_types -from ansible.module_utils.basic import to_text -from ansible_collections.amazon.aws.plugins.module_utils.aws.core import AnsibleAWSModule, is_boto3_error_code -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import compare_policies, boto3_tag_list_to_ansible_dict, ansible_dict_to_boto3_tag_list -from ansible_collections.amazon.aws.plugins.module_utils.ec2 import get_aws_connection_info, boto3_conn, AWSRetry - try: from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError, WaiterError except ImportError: - pass # caught by AnsibleAWSModule + pass # Handled by AnsibleAWSModule + +from ansible.module_utils.basic import to_text +from ansible.module_utils.six import string_types +from ansible.module_utils.six.moves.urllib.parse import urlparse + +from ..module_utils.core import AnsibleAWSModule +from ..module_utils.core import is_boto3_error_code +from ..module_utils.ec2 import AWSRetry +from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list +from ..module_utils.ec2 import boto3_conn +from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict +from ..module_utils.ec2 import compare_policies +from ..module_utils.ec2 import get_aws_connection_info def create_or_update_bucket(s3_client, module, location):