Refacter lookup plugins (ansible-collections#1225)

Refacter lookup plugins

Depends-On: ansible-collections#1248
SUMMARY

Refacters the lookup plugins to use common code for common boto3/botocore operations

ISSUE TYPE

Feature Pull Request

COMPONENT NAME
plugins/lookup/aws_account_attribute.py
plugins/lookup/aws_secret.py
plugins/lookup/aws_ssm.py
plugins/module_utils/botocore.py
plugins/module_utils/core.py
plugins/module_utils/exceptions.py
plugins/module_utils/modules.py
ADDITIONAL INFORMATION

Reviewed-by: Alina Buzachis <None>
Reviewed-by: Mark Chappell <None>

changelogs/fragments/1225-refacter-lookup.yml

@@ -0,0 +1,36 @@
+minor_changes:
+- aws_secret - the ``aws_secret`` lookup plugin has been renamed ``secretsmanager_secret``, ``aws_secret`` remains as an alias
+  (https://github.com/ansible-collections/amazon.aws/pull/1225).
+- aws_ssm - the ``aws_ssm`` lookup plugin has been renamed ``ssm_parameter``, ``aws_ssm`` remains as an alias
+  (https://github.com/ansible-collections/amazon.aws/pull/1225).
+
+- aws_account_attribute - the ``aws_account_attribute`` lookup plugin has been refactored to use
+  ``AWSLookupBase`` as its base class
+  (https://github.com/ansible-collections/amazon.aws/pull/1225).
+- aws_secret - the ``aws_secret`` lookup plugin has been refactored to use
+  ``AWSLookupBase`` as its base class
+  (https://github.com/ansible-collections/amazon.aws/pull/1225).
+- aws_ssm - the ``aws_ssm`` lookup plugin has been refactored to use
+  ``AWSLookupBase`` as its base class
+  (https://github.com/ansible-collections/amazon.aws/pull/1225).
+
+- amazon.aws lookup plugins - ``aws_profile`` has been renamed to ``profile`` for consistency
+  between modules and plugins, ``aws_profile`` remains as an alias.
+  This change should have no observable effect for users outside the module/plugin documentation
+  (https://github.com/ansible-collections/amazon.aws/pull/1225).
+- amazon.aws lookup plugins - ``aws_access_key`` has been renamed to ``access_key`` for consistency
+  between modules and plugins, ``aws_access_key`` remains as an alias.
+  This change should have no observable effect for users outside the module/plugin documentation
+  (https://github.com/ansible-collections/amazon.aws/pull/1225).
+- amazon.aws lookup plugins - ``aws_secret_key`` has been renamed to ``secret_key`` for consistency
+  between modules and plugins, ``aws_secret_key`` remains as an alias.
+  This change should have no observable effect for users outside the module/plugin documentation
+  (https://github.com/ansible-collections/amazon.aws/pull/1225).
+- amazon.aws lookup plugins - ``aws_security_token`` has been renamed to ``session_token`` for consistency
+  between modules and plugins, ``aws_security_token`` remains as an alias.
+  This change should have no observable effect for users outside the module/plugin documentation
+  (https://github.com/ansible-collections/amazon.aws/pull/1225).
+
+deprecated_features:
+- amazon.aws lookup plugins - the ``boto3_profile`` alias for the ``profile`` option has been deprecated, please use ``profile`` instead
+  (https://github.com/ansible-collections/amazon.aws/pull/1225).

meta/runtime.yml

@@ -126,3 +126,10 @@ plugin_routing:
     execute_lambda:
       # Deprecation for this alias should not *start* prior to 2024-09-01
       redirect: amazon.aws.lambda_execute
+  lookup:
+    aws_ssm:
+      # Deprecation for this alias should not *start* prior to 2024-09-01
+      redirect: amazon.aws.ssm_parameter
+    aws_secret:
+      # Deprecation for this alias should not *start* prior to 2024-09-01
+      redirect: amazon.aws.secretsmanager_secret

plugins/lookup/aws_account_attribute.py

@@ -3,14 +3,10 @@
 from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
-DOCUMENTATION = '''
+DOCUMENTATION = r"""
 name: aws_account_attribute
 author:
   - Sloane Hertel (@s-hertel) <[email protected]>
-extends_documentation_fragment:
-  - amazon.aws.boto3
-  - amazon.aws.aws_credentials
-  - amazon.aws.region.plugins
 short_description: Look up AWS account attributes
 description:
   - Describes attributes of your AWS account. You can specify one of the listed
@@ -26,9 +22,13 @@
       - max-elastic-ips
       - vpc-max-elastic-ips
       - has-ec2-classic
-'''
+extends_documentation_fragment:
+  - amazon.aws.boto3
+  - amazon.aws.common.plugins
+  - amazon.aws.region.plugins
+"""
 
-EXAMPLES = """
+EXAMPLES = r"""
 vars:
   has_ec2_classic: "{{ lookup('aws_account_attribute', attribute='has-ec2-classic') }}"
   # true | false
@@ -42,71 +42,34 @@
 
 """
 
-RETURN = """
+RETURN = r"""
 _raw:
   description:
     Returns a boolean when I(attribute) is check_ec2_classic. Otherwise returns the value(s) of the attribute
     (or all attributes if one is not specified).
 """
 
 try:
-    import boto3
     import botocore
 except ImportError:
-    pass  # will be captured by imported HAS_BOTO3
+    pass  # Handled by AWSLookupBase
 
-from ansible.errors import AnsibleError
+from ansible.errors import AnsibleLookupError
 from ansible.module_utils._text import to_native
-from ansible.module_utils.basic import missing_required_lib
-from ansible.plugins.lookup import LookupBase
-
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import AWSRetry
-from ansible_collections.amazon.aws.plugins.module_utils.ec2 import HAS_BOTO3
 
+from ansible_collections.amazon.aws.plugins.module_utils.retries import AWSRetry
+from ansible_collections.amazon.aws.plugins.plugin_utils.lookup import AWSLookupBase
 
-def _boto3_conn(region, credentials):
-    boto_profile = credentials.pop('aws_profile', None)
 
-    try:
-        connection = boto3.session.Session(profile_name=boto_profile).client('ec2', region, **credentials)
-    except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError):
-        if boto_profile:
-            try:
-                connection = boto3.session.Session(profile_name=boto_profile).client('ec2', region)
-            except (botocore.exceptions.ProfileNotFound, botocore.exceptions.PartialCredentialsError):
-                raise AnsibleError("Insufficient credentials found.")
-        else:
-            raise AnsibleError("Insufficient credentials found.")
-    return connection
-
-
-def _get_credentials(options):
-    credentials = {}
-    credentials['aws_profile'] = options['aws_profile']
-    credentials['aws_secret_access_key'] = options['aws_secret_key']
-    credentials['aws_access_key_id'] = options['aws_access_key']
-    if options['aws_security_token']:
-        credentials['aws_session_token'] = options['aws_security_token']
-
-    return credentials
-
-
-@AWSRetry.jittered_backoff(retries=10)
 def _describe_account_attributes(client, **params):
-    return client.describe_account_attributes(**params)
+    return client.describe_account_attributes(aws_retry=True, **params)
 
 
-class LookupModule(LookupBase):
+class LookupModule(AWSLookupBase):
     def run(self, terms, variables, **kwargs):
+        super(LookupModule, self).run(terms, variables, **kwargs)
 
-        if not HAS_BOTO3:
-            raise AnsibleError(missing_required_lib('botocore and boto3'))
-
-        self.set_options(var_options=variables, direct=kwargs)
-        boto_credentials = _get_credentials(self._options)
-
-        region = self._options['region']
-        client = _boto3_conn(region, boto_credentials)
+        client = self.client('ec2', AWSRetry.jittered_backoff())
 
         attribute = kwargs.get('attribute')
         params = {'AttributeNames': []}
@@ -120,7 +83,7 @@ def run(self, terms, variables, **kwargs):
         try:
             response = _describe_account_attributes(client, **params)['AccountAttributes']
         except (botocore.exceptions.ClientError, botocore.exceptions.BotoCoreError) as e:
-            raise AnsibleError("Failed to describe account attributes: %s" % to_native(e))
+            raise AnsibleLookupError("Failed to describe account attributes: {0}".format(to_native(e)))
 
         if check_ec2_classic:
             attr = response[0]

plugins/lookup/aws_service_ip_ranges.py

@@ -45,7 +45,7 @@
 
 import json
 
-from ansible.errors import AnsibleError
+from ansible.errors import AnsibleLookupError
 from ansible.module_utils.six.moves.urllib.error import HTTPError
 from ansible.module_utils.six.moves.urllib.error import URLError
 from ansible.module_utils._text import to_native
@@ -70,15 +70,15 @@ def run(self, terms, variables, **kwargs):
         except getattr(json.decoder, 'JSONDecodeError', ValueError) as e:
             # on Python 3+, json.decoder.JSONDecodeError is raised for bad
             # JSON. On 2.x it's a ValueError
-            raise AnsibleError("Could not decode AWS IP ranges: %s" % to_native(e))
+            raise AnsibleLookupError("Could not decode AWS IP ranges: {0}".format(to_native(e)))
         except HTTPError as e:
-            raise AnsibleError("Received HTTP error while pulling IP ranges: %s" % to_native(e))
+            raise AnsibleLookupError("Received HTTP error while pulling IP ranges: {0}".format(to_native(e)))
         except SSLValidationError as e:
-            raise AnsibleError("Error validating the server's certificate for: %s" % to_native(e))
+            raise AnsibleLookupError("Error validating the server's certificate for: {0}".format(to_native(e)))
         except URLError as e:
-            raise AnsibleError("Failed look up IP range service: %s" % to_native(e))
+            raise AnsibleLookupError("Failed look up IP range service: {0}".format(to_native(e)))
         except ConnectionError as e:
-            raise AnsibleError("Error connecting to IP range service: %s" % to_native(e))
+            raise AnsibleLookupError("Error connecting to IP range service: {0}".format(to_native(e)))
 
         if 'region' in kwargs:
             region = kwargs['region']