Move module_utils out of subdir (#26)

* Move module_utils out of subdir and update imports

* Use relative imports
* Always get dict transformations from their canonical home
* Split imports by line to match tremble's other refactoring
* Make import order more consistent

* Temporarily restore legacy module_utils files

plugins/lookup/aws_account_attribute.py

@@ -52,6 +52,8 @@
     (or all attributes if one is not specified).
 """
 
+import os
+
 from ansible.errors import AnsibleError
 
 try:
@@ -60,12 +62,13 @@
 except ImportError:
     raise AnsibleError("The lookup aws_account_attribute requires boto3 and botocore.")
 
-from ansible.plugins import AnsiblePlugin
-from ansible.plugins.lookup import LookupBase
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import boto3_conn, get_aws_connection_info
 from ansible.module_utils._text import to_native
 from ansible.module_utils.six import string_types
-import os
+from ansible.plugins import AnsiblePlugin
+from ansible.plugins.lookup import LookupBase
+
+from ..module_utils.ec2 import boto3_conn
+from ..module_utils.ec2 import get_aws_connection_info
 
 
 def _boto3_conn(region, credentials):

plugins/lookup/aws_ssm.py

@@ -105,19 +105,21 @@
 
 '''
 
-from ansible.module_utils._text import to_native
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO3, boto3_tag_list_to_ansible_dict
-from ansible.errors import AnsibleError
-from ansible.plugins.lookup import LookupBase
-from ansible.utils.display import Display
-
 try:
     from botocore.exceptions import ClientError
     import botocore
     import boto3
 except ImportError:
     pass  # will be captured by imported HAS_BOTO3
 
+from ansible.errors import AnsibleError
+from ansible.module_utils._text import to_native
+from ansible.plugins.lookup import LookupBase
+from ansible.utils.display import Display
+
+from ..module_utils.ec2 import HAS_BOTO3
+from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict
+
 display = Display()
 
 

plugins/module_utils/acm.py

@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2019 Ansible Project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+#
+# This module is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This software is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this software.  If not, see <http://www.gnu.org/licenses/>.
+#
+# Author:
+#   - Matthew Davis <[email protected]>
+#     on behalf of Telstra Corporation Limited
+#
+# Common functionality to be used by the modules:
+#   - acm
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+"""
+Common Amazon Certificate Manager facts shared between modules
+"""
+import traceback
+
+try:
+    import botocore
+    from botocore.exceptions import BotoCoreError, ClientError
+except ImportError:
+    pass
+
+from ansible.module_utils._text import to_bytes
+from ansible.module_utils.common.dict_transformations import camel_dict_to_snake_dict
+
+from .ec2 import AWSRetry
+from .ec2 import ansible_dict_to_boto3_tag_list
+from .ec2 import boto3_conn
+from .ec2 import boto3_tag_list_to_ansible_dict
+from .ec2 import get_aws_connection_info
+
+
+class ACMServiceManager(object):
+    """Handles ACM Facts Services"""
+
+    def __init__(self, module):
+        self.module = module
+
+        region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True)
+        self.client = module.client('acm')
+
+    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['RequestInProgressException'])
+    def delete_certificate_with_backoff(self, client, arn):
+        client.delete_certificate(CertificateArn=arn)
+
+    def delete_certificate(self, client, module, arn):
+        module.debug("Attempting to delete certificate %s" % arn)
+        try:
+            self.delete_certificate_with_backoff(client, arn)
+        except (BotoCoreError, ClientError) as e:
+            module.fail_json_aws(e, msg="Couldn't delete certificate %s" % arn)
+        module.debug("Successfully deleted certificate %s" % arn)
+
+    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['RequestInProgressException'])
+    def list_certificates_with_backoff(self, client, statuses=None):
+        paginator = client.get_paginator('list_certificates')
+        kwargs = dict()
+        if statuses:
+            kwargs['CertificateStatuses'] = statuses
+        return paginator.paginate(**kwargs).build_full_result()['CertificateSummaryList']
+
+    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException', 'RequestInProgressException'])
+    def get_certificate_with_backoff(self, client, certificate_arn):
+        response = client.get_certificate(CertificateArn=certificate_arn)
+        # strip out response metadata
+        return {'Certificate': response['Certificate'],
+                'CertificateChain': response['CertificateChain']}
+
+    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException', 'RequestInProgressException'])
+    def describe_certificate_with_backoff(self, client, certificate_arn):
+        return client.describe_certificate(CertificateArn=certificate_arn)['Certificate']
+
+    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException', 'RequestInProgressException'])
+    def list_certificate_tags_with_backoff(self, client, certificate_arn):
+        return client.list_tags_for_certificate(CertificateArn=certificate_arn)['Tags']
+
+    # Returns a list of certificates
+    # if domain_name is specified, returns only certificates with that domain
+    # if an ARN is specified, returns only that certificate
+    # only_tags is a dict, e.g. {'key':'value'}. If specified this function will return
+    # only certificates which contain all those tags (key exists, value matches).
+    def get_certificates(self, client, module, domain_name=None, statuses=None, arn=None, only_tags=None):
+        try:
+            all_certificates = self.list_certificates_with_backoff(client=client, statuses=statuses)
+        except (BotoCoreError, ClientError) as e:
+            module.fail_json_aws(e, msg="Couldn't obtain certificates")
+        if domain_name:
+            certificates = [cert for cert in all_certificates
+                            if cert['DomainName'] == domain_name]
+        else:
+            certificates = all_certificates
+
+        if arn:
+            # still return a list, not just one item
+            certificates = [c for c in certificates if c['CertificateArn'] == arn]
+
+        results = []
+        for certificate in certificates:
+            try:
+                cert_data = self.describe_certificate_with_backoff(client, certificate['CertificateArn'])
+            except (BotoCoreError, ClientError) as e:
+                module.fail_json_aws(e, msg="Couldn't obtain certificate metadata for domain %s" % certificate['DomainName'])
+
+            # in some states, ACM resources do not have a corresponding cert
+            if cert_data['Status'] not in ['PENDING_VALIDATION', 'VALIDATION_TIMED_OUT', 'FAILED']:
+                try:
+                    cert_data.update(self.get_certificate_with_backoff(client, certificate['CertificateArn']))
+                except (BotoCoreError, ClientError, KeyError) as e:
+                    module.fail_json_aws(e, msg="Couldn't obtain certificate data for domain %s" % certificate['DomainName'])
+            cert_data = camel_dict_to_snake_dict(cert_data)
+            try:
+                tags = self.list_certificate_tags_with_backoff(client, certificate['CertificateArn'])
+            except (BotoCoreError, ClientError) as e:
+                module.fail_json_aws(e, msg="Couldn't obtain tags for domain %s" % certificate['DomainName'])
+
+            cert_data['tags'] = boto3_tag_list_to_ansible_dict(tags)
+            results.append(cert_data)
+
+        if only_tags:
+            for tag_key in only_tags:
+                try:
+                    results = [c for c in results if ('tags' in c) and (tag_key in c['tags']) and (c['tags'][tag_key] == only_tags[tag_key])]
+                except (TypeError, AttributeError) as e:
+                    for c in results:
+                        if 'tags' not in c:
+                            module.debug("cert is %s" % str(c))
+                    module.fail_json(msg="ACM tag filtering err", exception=e)
+
+        return results
+
+    # returns the domain name of a certificate (encoded in the public cert)
+    # for a given ARN
+    # A cert with that ARN must already exist
+    def get_domain_of_cert(self, client, module, arn):
+        if arn is None:
+            module.fail(msg="Internal error with ACM domain fetching, no certificate ARN specified")
+        try:
+            cert_data = self.describe_certificate_with_backoff(client=client, certificate_arn=arn)
+        except (BotoCoreError, ClientError) as e:
+            module.fail_json_aws(e, msg="Couldn't obtain certificate data for arn %s" % arn)
+        return cert_data['DomainName']
+
+    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0)
+    def import_certificate_with_backoff(self, client, certificate, private_key, certificate_chain, arn):
+        if certificate_chain:
+            if arn:
+                ret = client.import_certificate(Certificate=to_bytes(certificate),
+                                                PrivateKey=to_bytes(private_key),
+                                                CertificateChain=to_bytes(certificate_chain),
+                                                CertificateArn=arn)
+            else:
+                ret = client.import_certificate(Certificate=to_bytes(certificate),
+                                                PrivateKey=to_bytes(private_key),
+                                                CertificateChain=to_bytes(certificate_chain))
+        else:
+            if arn:
+                ret = client.import_certificate(Certificate=to_bytes(certificate),
+                                                PrivateKey=to_bytes(private_key),
+                                                CertificateArn=arn)
+            else:
+                ret = client.import_certificate(Certificate=to_bytes(certificate),
+                                                PrivateKey=to_bytes(private_key))
+        return ret['CertificateArn']
+
+    # Tags are a normal Ansible style dict
+    # {'Key':'Value'}
+    @AWSRetry.backoff(tries=5, delay=5, backoff=2.0, catch_extra_error_codes=['ResourceNotFoundException', 'RequestInProgressException'])
+    def tag_certificate_with_backoff(self, client, arn, tags):
+        aws_tags = ansible_dict_to_boto3_tag_list(tags)
+        client.add_tags_to_certificate(CertificateArn=arn, Tags=aws_tags)
+
+    def import_certificate(self, client, module, certificate, private_key, arn=None, certificate_chain=None, tags=None):
+
+        original_arn = arn
+
+        # upload cert
+        try:
+            arn = self.import_certificate_with_backoff(client, certificate, private_key, certificate_chain, arn)
+        except (BotoCoreError, ClientError) as e:
+            module.fail_json_aws(e, msg="Couldn't upload new certificate")
+
+        if original_arn and (arn != original_arn):
+            # I'm not sure whether the API guarentees that the ARN will not change
+            # I'm failing just in case.
+            # If I'm wrong, I'll catch it in the integration tests.
+            module.fail_json(msg="ARN changed with ACM update, from %s to %s" % (original_arn, arn))
+
+        # tag that cert
+        try:
+            self.tag_certificate_with_backoff(client, arn, tags)
+        except (BotoCoreError, ClientError) as e:
+            module.debug("Attempting to delete the cert we just created, arn=%s" % arn)
+            try:
+                self.delete_certificate_with_backoff(client, arn)
+            except Exception as f:
+                module.warn("Certificate %s exists, and is not tagged. So Ansible will not see it on the next run.")
+                module.fail_json_aws(e, msg="Couldn't tag certificate %s, couldn't delete it either" % arn)
+            module.fail_json_aws(e, msg="Couldn't tag certificate %s" % arn)
+
+        return arn

plugins/module_utils/batch.py

@@ -0,0 +1,106 @@
+# Copyright (c) 2017 Ansible Project
+#
+# This code is part of Ansible, but is an independent component.
+# This particular file snippet, and this file snippet only, is BSD licensed.
+# Modules you write using this snippet, which is embedded dynamically by Ansible
+# still belong to the author of the module, and may assign their own license
+# to the complete work.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+#    * Redistributions of source code must retain the above copyright
+#      notice, this list of conditions and the following disclaimer.
+#    * Redistributions in binary form must reproduce the above copyright notice,
+#      this list of conditions and the following disclaimer in the documentation
+#      and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+"""
+This module adds shared support for Batch modules.
+"""
+
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+try:
+    from botocore.exceptions import ClientError
+except ImportError:
+    pass
+
+from ansible.module_utils.common.dict_transformations import snake_dict_to_camel_dict
+
+from .ec2 import boto3_conn
+from .ec2 import get_aws_connection_info
+
+
+class AWSConnection(object):
+    """
+    Create the connection object and client objects as required.
+    """
+
+    def __init__(self, ansible_obj, resources, boto3=True):
+
+        ansible_obj.deprecate("The 'ansible.module_utils.aws.batch.AWSConnection' class is deprecated please use 'AnsibleAWSModule.client()'",
+                              version='2.14')
+
+        self.region, self.endpoint, aws_connect_kwargs = get_aws_connection_info(ansible_obj, boto3=boto3)
+
+        self.resource_client = dict()
+        if not resources:
+            resources = ['batch']
+
+        resources.append('iam')
+
+        for resource in resources:
+            aws_connect_kwargs.update(dict(region=self.region,
+                                           endpoint=self.endpoint,
+                                           conn_type='client',
+                                           resource=resource
+                                           ))
+            self.resource_client[resource] = boto3_conn(ansible_obj, **aws_connect_kwargs)
+
+        # if region is not provided, then get default profile/session region
+        if not self.region:
+            self.region = self.resource_client['batch'].meta.region_name
+
+        # set account ID
+        try:
+            self.account_id = self.resource_client['iam'].get_user()['User']['Arn'].split(':')[4]
+        except (ClientError, ValueError, KeyError, IndexError):
+            self.account_id = ''
+
+    def client(self, resource='batch'):
+        return self.resource_client[resource]
+
+
+def cc(key):
+    """
+    Changes python key into Camel case equivalent. For example, 'compute_environment_name' becomes
+    'computeEnvironmentName'.
+
+    :param key:
+    :return:
+    """
+    components = key.split('_')
+    return components[0] + "".join([token.capitalize() for token in components[1:]])
+
+
+def set_api_params(module, module_params):
+    """
+    Sets module parameters to those expected by the boto3 API.
+    :param module:
+    :param module_params:
+    :return:
+    """
+    api_params = dict((k, v) for k, v in dict(module.params).items() if k in module_params and v is not None)
+    return snake_dict_to_camel_dict(api_params)